diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index e140b1e..6307f72 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] macros = { path = "../macros" } constants = { path = "../constants" } +markdown = { path = "../markdown" } chrono = { version = "0.4.38", features = ["serde", "alloc"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } @@ -19,6 +20,6 @@ serde = { version = "1.0.199", features = ["derive"] } serde_json = "1.0.128" uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } futures = { version = "0.3.30", features = ["executor"] } -sha2 = "0.10.8" -cron = "0.12.1" +cron = "0.13.0" git2 = "0.19.0" +thiserror = "2.0.3" diff --git a/backend/api_server/src/blog/cron/check_update.rs b/backend/api_server/src/blog/cron/check_update.rs deleted file mode 100644 index ef1eca0..0000000 --- a/backend/api_server/src/blog/cron/check_update.rs +++ /dev/null @@ -1,24 +0,0 @@ -use sqlx::{Pool, Postgres}; - -pub fn task(db_connection: &Pool) { - // get dir - // let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| { - // if let Ok(default_log_path) = env::var("WORK_DIR") { - // if default_log_path.starts_with('~') { - // let home_path = home_dir().unwrap_or_else(|| panic!("Cannot find home directory")); - // home_path - // .join(default_log_path.strip_prefix("~/").unwrap()) - // .into_os_string() - // .into_string() - // .expect( - // "[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead", - // ) - // } else { - // default_log_path - // } - // } else { - // panic!("[fatal] failed to parse env `GIT_WORK_DIR`: {}", e); - // } - // }); - println!("check update"); -} diff --git a/backend/api_server/src/blog/cron/check_update/error.rs b/backend/api_server/src/blog/cron/check_update/error.rs new file mode 100644 index 0000000..0d27c29 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/error.rs @@ -0,0 +1,49 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PostIdxError { + #[error("missing essential field `{}` for file `{}`", field, filename)] + MissingField { field: String, filename: String }, + + #[error( + "invalid field `{}` for file `{}`, <{}>: {}", + field_name, + filename, + err_field, + msg + )] + InvalidField { + field_name: String, + err_field: String, + filename: String, + msg: String, + }, + + #[error( + "database error on writing `{}` to table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBWriteFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, + + #[error( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBReadFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, +} diff --git a/backend/api_server/src/blog/cron/check_update/index.rs b/backend/api_server/src/blog/cron/check_update/index.rs new file mode 100644 index 0000000..8bfe006 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/index.rs @@ -0,0 +1,333 @@ +use std::error::Error; +use std::path::Path; + +use crate::blog::cron::check_update::utils::get_md_file_basename; +use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date}; +use markdown::MdFile; + +use sqlx::{query, PgPool, Pool, Postgres, Row}; +use tokio::runtime::Runtime; +use tracing::{info, warn}; + +async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + let post_id = post.file_id; + let title = match post.meta.get("title") { + None => { + return Err(PostIdxError::MissingField { + field: "title".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid title for `{}`", &post.filename); + "invalid title" + }), + }; + let sub_title = post + .meta + .get("sub_title") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let summary = post + .meta + .get("summary") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let content = post.content.as_str(); + let category = match post.meta.get("category") { + None => { + return Err(PostIdxError::MissingField { + field: "category".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid category for `{}`", &post.filename); + "invalid category" + }), + }; + let last_update = { + if let Some(last_update) = post.meta.get("last_update") { + match parse_date(last_update.as_str().unwrap()) { + Ok(date) => Some(date), + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "last_update".to_string(), + err_field: last_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + } + } else { + None + } + }; + let first_update = match post.meta.get("date") { + None => { + return Err(PostIdxError::MissingField { + field: "date".to_string(), + filename: post.filename.clone(), + }) + } + Some(first_update) => match parse_date(first_update.as_str().unwrap()) { + Ok(date) => date, + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "date".to_string(), + err_field: first_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + }, + }; + let filename = get_md_file_basename(&post.filename); + + if let Err(e) = query( + r##" +INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (post_id) DO +UPDATE SET + title = EXCLUDED.title, + summary = EXCLUDED.summary, + content = EXCLUDED.content, + last_update = EXCLUDED.last_update, + first_update = EXCLUDED.first_update, + sub_title = EXCLUDED.sub_title, + category = EXCLUDED.category, + filename = EXCLUDED.filename; + "##, + ) + .bind(post_id) + .bind(title) + .bind(summary) + .bind(content) + .bind(last_update) + .bind(first_update) + .bind(sub_title) + .bind(category) + .bind(filename) + .execute(db_connection) + .await + { + return Err( + PostIdxError::DBWriteFailure { + data_desc: "post base information".to_string(), + db_table: "Post".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + } + ); + } + Ok(()) +} + +async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Some(tags_string) = post.meta.get("tags").to_owned() { + let tags = match tags_string.as_array().map(|vec| { + vec.iter() + .map(|val| val.as_str().unwrap()) + .collect::>() + }) { + None => { + return Err(PostIdxError::MissingField { + field: "tags".to_string(), + filename: post.filename.clone(), + }) + } + Some(val) => val, + }; + + let post_id = post.file_id; + + // remove old record + if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") + .bind(post_id) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }); + } + + // add new record + for tag in tags { + match query("INSERT INTO Tag VALUES ($1, $2);") + .bind(post_id) + .bind(tag) + .execute(db_connection) + .await + { + Ok(_) => {} + Err(e) => { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }) + } + } + } + } + + Ok(()) +} + +async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { + let base64_form = post.hash.clone(); + match query("SELECT hash FROM Hash WHERE post_id = $1") + .bind(post.file_id) + .fetch_optional(db_connection) + .await + { + Ok(row) => { + let db_record: String = match row { + None => "".to_string(), + Some(row) => row.try_get("hash").unwrap_or("".to_string()), + }; + if base64_form == db_record { + Ok(false) + } else { + if let Err(e) = query( + r#" +INSERT INTO Hash (post_id, hash) VALUES ($1, $2) + ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, + ) + .bind(post.file_id) + .bind(&base64_form) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }); + } + Ok(true) + } + } + Err(e) => Err(PostIdxError::DBReadFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }), + } +} + +async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Err(e) = query( + // todo: sanitization + format!( + r#" +DO +$$ + DECLARE + _post_id UUID := '{}'; + _meta_cols JSONB := '{}'; + _exists BOOLEAN; + _col_name TEXT; + _col_value TEXT; + _sql TEXT; + _insert_columns TEXT := 'post_id'; + _insert_values TEXT := quote_literal(_post_id); + _update_set TEXT := ''; + _blacklist TEXT[] := ARRAY['tags']; + BEGIN + SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; + + FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) + LOOP + IF _col_name = ANY(_blacklist) THEN + CONTINUE; + END IF; + + IF NOT EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'meta' + AND column_name = _col_name + -- may change in the future + AND table_schema = 'public') THEN + _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; + EXECUTE _sql; + END IF; + _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); + _insert_values := _insert_values || ', ' || quote_literal(_col_value); + _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); + END LOOP; + + _update_set := substring(_update_set FROM 3); + + IF _exists THEN + _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); + EXECUTE _sql; + ELSE + _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; + EXECUTE _sql; + END IF; + END; +$$ LANGUAGE plpgsql; + "#, post.file_id.to_string().as_str(), post.meta + ).as_str() + ) + .execute(db_connection) + .await { + Err(PostIdxError::DBWriteFailure { + data_desc: "post meta data".to_string(), + db_table: "Meta".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }) + } else { + Ok(()) + } +} + +pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { + let md_file = MdFile::from_file(path)?; + + base(db_conn, &md_file).await?; + tag(db_conn, &md_file).await?; + meta(db_conn, &md_file).await?; + hash(db_conn, &md_file).await?; + + info!("indexed markdown file: {}", path.display()); + Ok(()) +} + +pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + if let Err(e) = query( + r#" +WITH deleted_post AS ( + DELETE FROM post + WHERE id = (SELECT id FROM post WHERE filename = $1) + RETURNING id +) +DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); + "#, + ) + .bind(file_basename) + .execute(db_conn) + .await + { + Err(Box::from( + format!("failed to drop index: {}", e).to_string(), + )) + } else { + Ok(()) + } + }) +} diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs new file mode 100644 index 0000000..63ca30e --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/mod.rs @@ -0,0 +1,144 @@ +use std::env; +use std::fs::remove_dir_all; +use std::path::{Path, PathBuf}; + +use crate::{ + blog::cron::check_update::{ + index::drop_index, + utils::{expand_path, get_md_file_basename, index_files, search_md}, + }, + utils::git, +}; +use constants::config::general::{default_value, var_name}; +use macros::panic_with_log; + +use git2::{Delta, Repository}; +use sqlx::{Pool, Postgres}; +use tracing::{error, info, warn, Level}; + +mod error; +mod index; +mod utils; + +pub fn task(db_connection: &Pool) { + // init + let git_work_dir = { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_path(default_log_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") + }; + + // clone & check update + + let git_url = match env::var(var_name::BLOG_GIT_URL) { + Ok(url) => url, + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to read `{}`: {}", + var_name::BLOG_GIT_URL, + e + ) + } + }; + let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) + .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); + let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) + .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); + + match Repository::open(&git_work_dir) { + Ok(repo) => { + // prevent git remote url mismatch (clean cache) + match repo.find_remote(&git_remote_name) { + Ok(remote) => { + if remote.url().is_some_and(|url| url != git_url) { + remove_dir_all(&git_work_dir).unwrap(); + warn!("blog content git({}) url mismatch, reinitialized", &git_url); + return; + } + } + Err(e) => { + remove_dir_all(&git_work_dir).unwrap(); + error!( + "unexpected error on indexing blog content git({}), reinitialized: {}", + &git_url, e + ); + } + } + + match git::sync(&git_remote_name, &git_remote_branch, &repo) { + Ok(updates) => { + if !updates.is_empty() { + // incremental update + info!( + "blog content git({}) update detected: {:?}", + &git_url, updates + ); + + let mut update_md: Vec = vec![]; + + for update in updates { + if [Delta::Deleted, Delta::Untracked].contains(&update.status) { + if let Some(path) = &update.old_path { + if drop_index( + &get_md_file_basename( + path.file_name().unwrap().to_str().unwrap(), + ), + db_connection, + ) + .is_ok() + { + info!( + "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", + &git_url, + &path.display() + ); + } else { + error!( + "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", + &git_url, + &path.display() + ) + } + } + } else if let Some(path) = &update.new_path { + if path.extension().is_some_and(|ext| ext == "md") { + update_md.push(path.to_path_buf()); + } + } + } + + index_files(&update_md, db_connection); + } + } + Err(e) => { + error!( + "failed to sync blog content git({}) updates: {}", + &git_url, e + ); + } + } + } + Err(_) => { + match Repository::clone(&git_url, &git_work_dir) { + Ok(_repo) => { + // full repo index + let files = search_md(git_work_dir); + index_files(&files, db_connection); + } + Err(e) => { + error!("failed to clone blog content git({}): {}", &git_url, e); + } + } + } + }; +} diff --git a/backend/api_server/src/blog/cron/check_update/utils.rs b/backend/api_server/src/blog/cron/check_update/utils.rs new file mode 100644 index 0000000..d0da864 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/utils.rs @@ -0,0 +1,86 @@ +use dirs::home_dir; +use std::path::PathBuf; + +use crate::blog::cron::check_update::index::index_file; +use chrono::{DateTime, FixedOffset, ParseResult}; +use constants::config::general::{default_value, var_name}; +use constants::time::DEFAULT_DATE_FORMAT; + +use futures::executor::block_on; +use futures::future::join_all; +use sqlx::{Pool, Postgres}; +use tokio::runtime::Runtime; +use tokio::spawn; +use tracing::{error, warn}; + +pub fn search_md(entry: PathBuf) -> Vec { + if entry.ends_with(".git") { + return vec![]; + } + if entry.is_dir() { + if let Ok(files) = entry.read_dir() { + files + .flat_map(|file| { + let file_path = file.unwrap().path(); + search_md(file_path) + }) + .collect() + } else { + vec![] + } + } else if entry.extension().is_some_and(|ext| ext == "md") { + vec![entry] + } else { + vec![] + } +} + +pub fn index_files(files: &[PathBuf], db_conn: &Pool) { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let files_clone = files.to_owned(); + let handlers = files_clone + .into_iter() + .map(|file: PathBuf| { + let conn_clone = db_conn.clone(); + spawn(async move { + if let Err(e) = index_file(&file, &conn_clone).await { + error!("failed to index markdown file `{}`: {}", file.display(), e); + } + }) + }) + .collect::>(); + + block_on(join_all(handlers)); + }); +} + +pub fn expand_path(path: String) -> String { + if path.starts_with('~') { + home_dir() + .map(|dir| { + dir.join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + }) + .unwrap_or_else(|| { + warn!( + "failed to parse env var `{}`, using default value `{}`", + var_name::WORK_DIR, + default_value::WORK_DIR + ); + default_value::WORK_DIR.to_string() + }) + } else { + path + } +} + +pub fn parse_date(date_str: &str) -> ParseResult> { + DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) +} + +pub fn get_md_file_basename(f_name: &str) -> String { + f_name.replace(".md", "") +} diff --git a/backend/api_server/src/init/misc.rs b/backend/api_server/src/init/misc.rs new file mode 100644 index 0000000..6a08bc2 --- /dev/null +++ b/backend/api_server/src/init/misc.rs @@ -0,0 +1,14 @@ +use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds}; +use tracing::warn; + +pub fn init() { + // git2 option + unsafe { + if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) { + warn!("failed to set git2 server connect timeout: {}", e); + } + if let Err(e) = set_server_timeout_in_milliseconds(60_000) { + warn!("failed to set git2 server timeout: {}", e); + } + } +} diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs index 2332914..507d52a 100644 --- a/backend/api_server/src/init/mod.rs +++ b/backend/api_server/src/init/mod.rs @@ -1,6 +1,7 @@ mod cron; mod database; pub mod logger; +mod misc; mod schema; pub mod server; @@ -24,6 +25,8 @@ pub fn load_env() { } pub async fn start() { + misc::init(); + // connect database let db_pool = match database::init().await { Ok(pool) => { diff --git a/backend/api_server/src/init/schema.rs b/backend/api_server/src/init/schema.rs index 41ccb34..ef9def0 100644 --- a/backend/api_server/src/init/schema.rs +++ b/backend/api_server/src/init/schema.rs @@ -7,11 +7,11 @@ CREATE TABLE IF NOT EXISTS Post post_id UUID PRIMARY KEY, title VARCHAR(255) NOT NULL, category VARCHAR(255) NOT NULL, - header_img TEXT, + filename TEXT UNIQUE NOT NULL, summary TEXT, sub_title TEXT, content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, + last_update TIMESTAMPTZ, first_update TIMESTAMPTZ NOT NULL ); "##; diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index 5cdd2e5..8aab2c7 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -1,6 +1,7 @@ mod blog; mod init; mod scheduler; +mod utils; #[tokio::main] async fn main() { diff --git a/backend/api_server/src/utils/git.rs b/backend/api_server/src/utils/git.rs new file mode 100644 index 0000000..6743058 --- /dev/null +++ b/backend/api_server/src/utils/git.rs @@ -0,0 +1,195 @@ +/// git merge +/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs +/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 +/// +use git2::{ + build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, +}; +use std::path::PathBuf; +use tracing::{error, warn}; + + +#[derive(Debug, Eq, PartialEq)] +pub struct FileDelta { + pub old_path: Option, + pub new_path: Option, + pub status: Delta, +} + +fn do_fetch<'a>( + repo: &'a Repository, + refs: &[&str], + remote: &'a mut Remote, +) -> Result, git2::Error> { + let mut fetch_option = FetchOptions::new(); + + // Always fetch all tags. + // Perform a download and also update tips + fetch_option.download_tags(AutotagOption::All); + remote.fetch(refs, Some(&mut fetch_option), None)?; + + let fetch_head = repo.find_reference("FETCH_HEAD")?; + repo.reference_to_annotated_commit(&fetch_head) +} + +fn fast_forward( + repo: &Repository, + lb: &mut Reference, + rc: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let name = match lb.name() { + Some(s) => s.to_string(), + None => String::from_utf8_lossy(lb.name_bytes()).to_string(), + }; + let msg = format!("fast-forward({}) -> {}", name, rc.id()); + // println!("{}", msg); + lb.set_target(rc.id(), &msg)?; + repo.set_head(&name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + // For some reason, the force is required to make the working directory actually get updated + // I suspect we should be adding some logic to handle dirty working directory states, + // but this is just an example so maybe not. + .force(), + ))?; + Ok(()) +} + +fn normal_merge( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let ancestor = repo + .find_commit(repo.merge_base(local.id(), remote.id())?)? + .tree()?; + let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?; + + if idx.has_conflicts() { + warn!("merge conflicts detected"); + repo.checkout_index(Some(&mut idx), None)?; + return Ok(()); + } + let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; + // now create the merge commit + let msg = format!("merge({}) -> {}", remote.id(), local.id()); + let sig = repo.signature()?; + let local_commit = repo.find_commit(local.id())?; + let remote_commit = repo.find_commit(remote.id())?; + + // Do our merge commit and set the current branch head to that commit. + let _merge_commit = repo.commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &result_tree, + &[&local_commit, &remote_commit], + )?; + // Set working tree to match head. + repo.checkout_head(None)?; + Ok(()) +} + +fn do_merge<'a>( + repo: &'a Repository, + remote_branch: &str, + fetch_commit: AnnotatedCommit<'a>, +) -> Result, git2::Error> { + // 1. do a merge analysis + let analysis = repo.merge_analysis(&[&fetch_commit])?; + + let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; + + // 2. Do the appropriate merge + if analysis.0.is_fast_forward() { + // println!("Doing a fast-forward"); + // do a fast-forward + let ref_name = format!("refs/heads/{}", remote_branch); + match repo.find_reference(&ref_name) { + Ok(mut r) => { + fast_forward(repo, &mut r, &fetch_commit)?; + } + Err(_) => { + // The branch doesn't exist, so set the reference to the + // commit directly. Usually this is because you are pulling + // into an empty repository. + repo.reference( + &ref_name, + fetch_commit.id(), + true, + &format!("({}) -> {}", remote_branch, fetch_commit.id()), + )?; + repo.set_head(&ref_name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + .allow_conflicts(true) + .conflict_style_merge(true) + .force(), + ))?; + } + }; + } else if analysis.0.is_normal() { + // do a normal merge + normal_merge(repo, &head_commit, &fetch_commit)?; + } + // else { + // // already up-to-date + // println!("Nothing to do..."); + // } + + calculate_diff(repo, &head_commit, &fetch_commit) +} + +fn calculate_diff( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result, git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?; + + let mut updated_files: Vec = Vec::new(); + + if let Err(e) = diff.foreach( + &mut |d, _| { + // let delta = d.status(); + // + // let n_path = d.new_file().path(); + // let o_path = d.old_file().path(); + // + // if let Some(path) = n_path { + // updated_files.insert(path.to_owned()); + // } + // if let Some(path) = o_path { + // updated_files.insert(path.to_owned()); + // } + updated_files.push(FileDelta { + old_path: d.old_file().path().map(PathBuf::from), + new_path: d.new_file().path().map(PathBuf::from), + status: d.status(), + }); + true // Continue + }, + None, + None, + None, + ) { + error!("error at calculating diff: {:?}", e); + } + + Ok(updated_files.into_iter().collect()) +} + +pub fn sync( + remote_name: &str, + remote_branch: &str, + repo: &Repository, +) -> Result, git2::Error> { + let mut remote = repo.find_remote(remote_name)?; + let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; + do_merge(repo, remote_branch, fetch_commit) +} diff --git a/backend/api_server/src/utils/mod.rs b/backend/api_server/src/utils/mod.rs new file mode 100644 index 0000000..c2bf1c3 --- /dev/null +++ b/backend/api_server/src/utils/mod.rs @@ -0,0 +1 @@ +pub mod git; diff --git a/backend/constants/src/config.rs b/backend/constants/src/config.rs index 4584402..bf6ba77 100644 --- a/backend/constants/src/config.rs +++ b/backend/constants/src/config.rs @@ -35,9 +35,14 @@ pub mod general { pub mod var_name { pub const LOG_DIR: &str = "LOG_DIR"; pub const WORK_DIR: &str = "WORK_DIR"; + pub const BLOG_GIT_URL: &str = "BLOG_GIT_URL"; + pub const BLOG_GIT_REMOTE_BRANCH: &str = "BLOG_GIT_REMOTE_BRANCH"; + pub const BLOG_GIT_REMOTE_NAME: &str = "BLOG_GIT_REMOTE_NAME"; } pub mod default_value { pub const WORK_DIR: &str = "~/.nocturne"; + pub const BLOG_GIT_REMOTE_BRANCH: &str = "master"; + pub const BLOG_GIT_REMOTE_NAME: &str = "origin"; } } diff --git a/backend/constants/src/time.rs b/backend/constants/src/time.rs index 89e7f55..d7b29ac 100644 --- a/backend/constants/src/time.rs +++ b/backend/constants/src/time.rs @@ -1 +1,2 @@ pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; +pub const DEFAULT_DATE_FORMAT: &str = "%Y-%m-%d%zT%H%M%S"; diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index 0d0b372..a7d8aef 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -12,3 +12,4 @@ regex = "1.10.4" yaml-rust2 = "0.9.0" sha2 = "0.10.8" base64 = "0.22.1" +thiserror = "2.0.3" diff --git a/backend/markdown/src/error.rs b/backend/markdown/src/error.rs new file mode 100644 index 0000000..a93585f --- /dev/null +++ b/backend/markdown/src/error.rs @@ -0,0 +1,16 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("missing front-matter field: id")] + MissingID, + + #[error("invalid file id: {}", msg)] + InvalidID { msg: String }, + + #[error("invalid front-matter: {}", msg)] + InvalidFrontMatter { msg: String }, + + #[error("fs error: {}", msg)] + FSError { msg: String }, +} diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index b1e3562..518d162 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -1,35 +1,48 @@ +mod error; mod utils; -use std::error::Error; use std::fs; use std::path::Path; use std::str::FromStr; +use crate::error::Error; +use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + use base64::{engine::general_purpose, Engine}; use serde::Serialize; use serde_json::Value; use sha2::{Digest, Sha256}; use uuid::Uuid; -use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; - #[derive(Default, Debug, Serialize)] pub struct MdFile { pub file_id: Uuid, - pub file_name: String, + pub filename: String, pub meta: Value, pub content: String, pub hash: String, } impl MdFile { - pub async fn from_file(file_path: &str) -> Result> { - let raw_file = fs::read_to_string(file_path)?; + pub fn from_file(file_path: &Path) -> Result { + let raw_file = match fs::read_to_string(file_path) { + Ok(file) => file, + Err(e) => return Err(Error::FSError { msg: e.to_string() }), + }; let raw_file_bytes = raw_file.as_bytes().to_vec(); let (front_matter_str, content) = split_md_front_matter(raw_file); - let front_matter = parse_front_matter(&front_matter_str)?; + let front_matter = match parse_front_matter(&front_matter_str) { + Ok(front_matter) => front_matter, + Err(e) => return Err(Error::InvalidFrontMatter { msg: e.to_string() }), + }; let hash = calculate_hash_base64(&raw_file_bytes); - let file_id = Uuid::from_str(front_matter["id"].to_string().as_str())?; + let file_id = match front_matter["id"].as_str() { + Some(id) => match Uuid::from_str(id) { + Ok(id) => id, + Err(e) => return Err(Error::InvalidID { msg: e.to_string() }), + }, + None => return Err(Error::MissingID), + }; let file_name = Path::new(file_path) .file_name() .unwrap() @@ -39,7 +52,7 @@ impl MdFile { Ok(MdFile { file_id, - file_name, + filename: file_name, meta: front_matter, content, hash, diff --git a/backend/markdown/src/utils/front_matter.rs b/backend/markdown/src/utils/front_matter.rs index f38f029..5bf77d2 100644 --- a/backend/markdown/src/utils/front_matter.rs +++ b/backend/markdown/src/utils/front_matter.rs @@ -16,8 +16,12 @@ pub fn split_md_front_matter(raw_file: String) -> (String, String) { } pub fn parse_front_matter(front_matter: &str) -> Result> { - let parsed_f_matter = YamlLoader::load_from_str(front_matter)?[0].clone(); - Ok(Value::from(yaml_to_serde(&parsed_f_matter))) + let parsed_f_matter = YamlLoader::load_from_str(front_matter)?; + if parsed_f_matter.is_empty() { + Ok(Value::String(String::new())) + } else { + Ok(yaml_to_serde(&parsed_f_matter[0])) + } } #[cfg(test)]