fix(cron::update) fix several errors

backend/api_server/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 macros = { path = "../macros" }
 constants = { path = "../constants" }
+markdown = { path = "../markdown" }
 chrono = { version = "0.4.38", features = ["serde", "alloc"] }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] }
@@ -19,6 +20,6 @@ serde = { version = "1.0.199", features = ["derive"] }
 serde_json = "1.0.128"
 uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] }
 futures = { version = "0.3.30", features = ["executor"] }
-sha2 = "0.10.8"
+cron = "0.13.0"
-cron = "0.12.1"
 git2 = "0.19.0"
+thiserror = "2.0.3"

backend/api_server/src/blog/cron/check_update.rs

@@ -1,24 +0,0 @@
-use sqlx::{Pool, Postgres};
-
-pub fn task(db_connection: &Pool<Postgres>) {
-    // get dir
-    // let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| {
-    //     if let Ok(default_log_path) = env::var("WORK_DIR") {
-    //         if default_log_path.starts_with('~') {
-    //             let home_path = home_dir().unwrap_or_else(|| panic!("Cannot find home directory"));
-    //             home_path
-    //                 .join(default_log_path.strip_prefix("~/").unwrap())
-    //                 .into_os_string()
-    //                 .into_string()
-    //                 .expect(
-    //                     "[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead",
-    //                 )
-    //         } else {
-    //             default_log_path
-    //         }
-    //     } else {
-    //         panic!("[fatal] failed to parse env `GIT_WORK_DIR`: {}", e);
-    //     }
-    // });
-    println!("check update");
-}

backend/api_server/src/blog/cron/check_update/error.rs

@@ -0,0 +1,49 @@
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum PostIdxError {
+    #[error("missing essential field `{}` for file `{}`", field, filename)]
+    MissingField { field: String, filename: String },
+
+    #[error(
+        "invalid field `{}` for file `{}`, <{}>: {}",
+        field_name,
+        filename,
+        err_field,
+        msg
+    )]
+    InvalidField {
+        field_name: String,
+        err_field: String,
+        filename: String,
+        msg: String,
+    },
+
+    #[error(
+        "database error on writing `{}` to table `{}`: <{}>: {}",
+        data_desc,
+        db_table,
+        id,
+        msg
+    )]
+    DBWriteFailure {
+        data_desc: String,
+        db_table: String,
+        id: String,
+        msg: String,
+    },
+
+    #[error(
+        "Database error on Reading `{}` from table `{}`: <{}>: {}",
+        data_desc,
+        db_table,
+        id,
+        msg
+    )]
+    DBReadFailure {
+        data_desc: String,
+        db_table: String,
+        id: String,
+        msg: String,
+    },
+}

backend/api_server/src/blog/cron/check_update/index.rs

@@ -0,0 +1,351 @@
+use std::error::Error;
+use std::path::Path;
+
+use crate::blog::cron::check_update::utils::get_md_file_basename;
+use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date};
+use markdown::MdFile;
+
+use sqlx::{query, PgPool, Pool, Postgres, Row};
+use tokio::runtime::Runtime;
+use tracing::{info, warn};
+
+async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
+    let post_id = post.file_id;
+    let title = match post.meta.get("title") {
+        None => {
+            return Err(PostIdxError::MissingField {
+                field: "title".to_string(),
+                filename: post.filename.clone(),
+            })
+        }
+        Some(title) => title.as_str().unwrap_or_else(|| {
+            warn!("invalid title for `{}`", &post.filename);
+            "invalid title"
+        }),
+    };
+    let sub_title = post
+        .meta
+        .get("sub_title")
+        .and_then(|v| v.as_str())
+        .unwrap_or("");
+    let summary = post
+        .meta
+        .get("summary")
+        .and_then(|v| v.as_str())
+        .unwrap_or("");
+    let content = post.content.as_str();
+    let category = match post.meta.get("category") {
+        None => {
+            return Err(PostIdxError::MissingField {
+                field: "category".to_string(),
+                filename: post.filename.clone(),
+            })
+        }
+        Some(title) => title.as_str().unwrap_or_else(|| {
+            warn!("invalid category for `{}`", &post.filename);
+            "invalid category"
+        }),
+    };
+    let last_update = {
+        if let Some(last_update) = post.meta.get("last_update") {
+            match parse_date(last_update.as_str().unwrap()) {
+                Ok(date) => Some(date),
+                Err(e) => {
+                    return Err(PostIdxError::InvalidField {
+                        field_name: "last_update".to_string(),
+                        err_field: last_update.to_string(),
+                        filename: post.filename.clone(),
+                        msg: e.to_string(),
+                    })
+                }
+            }
+        } else {
+            None
+        }
+    };
+    let first_update = match post.meta.get("date") {
+        None => {
+            return Err(PostIdxError::MissingField {
+                field: "date".to_string(),
+                filename: post.filename.clone(),
+            })
+        }
+        Some(first_update) => match parse_date(first_update.as_str().unwrap()) {
+            Ok(date) => date,
+            Err(e) => {
+                return Err(PostIdxError::InvalidField {
+                    field_name: "date".to_string(),
+                    err_field: first_update.to_string(),
+                    filename: post.filename.clone(),
+                    msg: e.to_string(),
+                })
+            }
+        },
+    };
+    let filename = get_md_file_basename(&post.filename);
+
+    if let Err(e) = query(
+        r##"
+INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ON CONFLICT (post_id) DO
+UPDATE SET
+    title = EXCLUDED.title,
+    summary = EXCLUDED.summary,
+    content = EXCLUDED.content,
+    last_update = EXCLUDED.last_update,
+    first_update = EXCLUDED.first_update,
+    sub_title = EXCLUDED.sub_title,
+    category = EXCLUDED.category,
+    filename = EXCLUDED.filename;
+        "##,
+    )
+        .bind(post_id)
+        .bind(title)
+        .bind(summary)
+        .bind(content)
+        .bind(last_update)
+        .bind(first_update)
+        .bind(sub_title)
+        .bind(category)
+        .bind(filename)
+        .execute(db_connection)
+        .await
+    {
+        return Err(
+            PostIdxError::DBWriteFailure {
+                data_desc: "post base information".to_string(),
+                db_table: "Post".to_string(),
+                id: post_id.to_string(),
+                msg: e.to_string(),
+            }
+        );
+    }
+    Ok(())
+}
+
+async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
+    if let Some(tags_string) = post.meta.get("tags").to_owned() {
+        let tags = match tags_string.as_array().map(|vec| {
+            vec.iter()
+                .map(|val| val.as_str().unwrap())
+                .collect::<Vec<_>>()
+        }) {
+            None => {
+                return Err(PostIdxError::MissingField {
+                    field: "tags".to_string(),
+                    filename: post.filename.clone(),
+                })
+            }
+            Some(val) => val,
+        };
+
+        let post_id = post.file_id;
+
+        // remove old record
+        if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1")
+            .bind(post_id)
+            .execute(db_connection)
+            .await
+        {
+            return Err(PostIdxError::DBWriteFailure {
+                data_desc: "post tag information".to_string(),
+                db_table: "Tag".to_string(),
+                id: post_id.to_string(),
+                msg: e.to_string(),
+            });
+        }
+
+        // add new record
+        for tag in tags {
+            match query("INSERT INTO Tag VALUES ($1, $2);")
+                .bind(post_id)
+                .bind(tag)
+                .execute(db_connection)
+                .await
+            {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(PostIdxError::DBWriteFailure {
+                        data_desc: "post tag information".to_string(),
+                        db_table: "Tag".to_string(),
+                        id: post_id.to_string(),
+                        msg: e.to_string(),
+                    })
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn hash(db_connection: &PgPool, post: &MdFile) -> Result<bool, PostIdxError> {
+    let base64_form = post.hash.clone();
+    match query("SELECT hash FROM Hash WHERE post_id = $1")
+        .bind(post.file_id)
+        .fetch_optional(db_connection)
+        .await
+    {
+        Ok(row) => {
+            let db_record: String = match row {
+                None => "".to_string(),
+                Some(row) => row.try_get("hash").unwrap_or("".to_string()),
+            };
+            if base64_form == db_record {
+                Ok(false)
+            } else {
+                if let Err(e) = query(
+                    r#"
+INSERT INTO Hash (post_id, hash) VALUES ($1, $2)
+    ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#,
+                )
+                .bind(post.file_id)
+                .bind(&base64_form)
+                .execute(db_connection)
+                .await
+                {
+                    return Err(PostIdxError::DBWriteFailure {
+                        data_desc: "post hash".to_string(),
+                        db_table: "Hash".to_string(),
+                        id: post.file_id.to_string(),
+                        msg: e.to_string(),
+                    });
+                }
+                Ok(true)
+            }
+        }
+        Err(e) => Err(PostIdxError::DBReadFailure {
+            data_desc: "post hash".to_string(),
+            db_table: "Hash".to_string(),
+            id: post.file_id.to_string(),
+            msg: e.to_string(),
+        }),
+    }
+}
+
+async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
+    if let Err(e) = query(
+        // todo: sanitization
+        format!(
+            r#"
+DO
+$$
+    DECLARE
+        _post_id        UUID  := '{}';
+        _meta_cols      JSONB := '{}';
+        _exists         BOOLEAN;
+        _col_name       TEXT;
+        _col_value      TEXT;
+        _sql            TEXT;
+        _insert_columns TEXT  := 'post_id';
+        _insert_values  TEXT  := quote_literal(_post_id);
+        _update_set     TEXT  := '';
+        _blacklist      TEXT[] := ARRAY['tags'];
+    BEGIN
+        SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists;
+
+        FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols)
+            LOOP
+                IF _col_name = ANY(_blacklist) THEN
+                    CONTINUE;
+                END IF;
+
+                IF NOT EXISTS (SELECT 1
+                               FROM information_schema.columns
+                               WHERE table_name = 'meta'
+                                 AND column_name = _col_name
+                                 -- may change in the future
+                                 AND table_schema = 'public') THEN
+                    _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT';
+                    EXECUTE _sql;
+                END IF;
+                _insert_columns := _insert_columns || ', ' || quote_ident(_col_name);
+                _insert_values := _insert_values || ', ' || quote_literal(_col_value);
+                _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value);
+            END LOOP;
+
+        _update_set := substring(_update_set FROM 3);
+
+        IF _exists THEN
+            _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id);
+            EXECUTE _sql;
+        ELSE
+            _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')';
+            EXECUTE _sql;
+        END IF;
+    END;
+$$ LANGUAGE plpgsql;
+        "#, post.file_id.to_string().as_str(), post.meta
+        ).as_str()
+    )
+        .execute(db_connection)
+        .await {
+        Err(PostIdxError::DBWriteFailure {
+            data_desc: "post meta data".to_string(),
+            db_table: "Meta".to_string(),
+            id: post.file_id.to_string(),
+            msg: e.to_string(),
+        })
+    } else {
+        Ok(())
+    }
+}
+
+pub async fn index_file(path: &Path, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
+    let md_file = MdFile::from_file(path)?;
+
+    async fn trail(db_conn: &Pool<Postgres>, md_file: &MdFile) -> Result<(), PostIdxError> {
+        base(db_conn, md_file).await?;
+        tag(db_conn, md_file).await?;
+        meta(db_conn, md_file).await?;
+        hash(db_conn, md_file).await.map(|_| ())
+    }
+    let mut cnt = 5;
+
+    while let Err(e) = trail(db_conn, &md_file).await {
+        match e {
+            PostIdxError::DBWriteFailure { .. } => {
+                cnt -= 1;
+            }
+            PostIdxError::DBReadFailure { .. } => {
+                cnt -= 1;
+            }
+            _ => return Err(e.into()),
+        }
+        if cnt <= 0 {
+            return Err(e.into());
+        }
+    }
+
+    info!("indexed markdown file: {}", path.display());
+    Ok(())
+}
+
+pub fn drop_index(file_basename: &str, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
+    let rt = Runtime::new().unwrap();
+    rt.block_on(async {
+        if let Err(e) = query(
+            r#"
+WITH deleted_post AS (
+    DELETE FROM post
+    WHERE id = (SELECT id FROM post WHERE filename = $1)
+    RETURNING id
+)
+DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post);
+DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post);
+DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post);
+    "#,
+        )
+        .bind(file_basename)
+        .execute(db_conn)
+        .await
+        {
+            Err(Box::from(
+                format!("failed to drop index: {}", e).to_string(),
+            ))
+        } else {
+            Ok(())
+        }
+    })
+}

backend/api_server/src/blog/cron/check_update/mod.rs

@@ -0,0 +1,146 @@
+use std::env;
+use std::fs::remove_dir_all;
+use std::path::{Path, PathBuf};
+
+use crate::{
+    blog::cron::check_update::{
+        index::drop_index,
+        utils::{expand_path, get_md_file_basename, index_files, search_md},
+    },
+    utils::git,
+};
+use constants::config::general::{default_value, var_name};
+use macros::panic_with_log;
+
+use git2::{Delta, Repository};
+use sqlx::{Pool, Postgres};
+use tracing::{error, info, warn, Level};
+
+mod error;
+mod index;
+mod utils;
+
+pub fn task(db_connection: &Pool<Postgres>) {
+    // init
+    let git_work_dir = {
+        let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
+            expand_path(default_log_path)
+        } else {
+            let fallback_path = expand_path(default_value::WORK_DIR.to_string());
+            warn!(
+                "unset env var `{}`, using default path: {}/blog_git",
+                var_name::WORK_DIR,
+                fallback_path
+            );
+            fallback_path
+        };
+        Path::new(&path).join("blog_git")
+    };
+
+    // clone & check update
+
+    let git_url = match env::var(var_name::BLOG_GIT_URL) {
+        Ok(url) => url,
+        Err(e) => {
+            panic_with_log!(
+                Level::ERROR,
+                "failed to read `{}`: {}",
+                var_name::BLOG_GIT_URL,
+                e
+            )
+        }
+    };
+    let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME)
+        .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string());
+    let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH)
+        .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string());
+
+    match Repository::open(&git_work_dir) {
+        Ok(repo) => {
+            // prevent git remote url mismatch (clean cache)
+            match repo.find_remote(&git_remote_name) {
+                Ok(remote) => {
+                    if remote.url().is_some_and(|url| url != git_url) {
+                        remove_dir_all(&git_work_dir).unwrap();
+                        warn!("blog content git({}) url mismatch, reinitialized", &git_url);
+                        return;
+                    }
+                }
+                Err(e) => {
+                    remove_dir_all(&git_work_dir).unwrap();
+                    error!(
+                        "unexpected error on indexing blog content git({}), reinitialized: {}",
+                        &git_url, e
+                    );
+                }
+            }
+
+            match git::sync(&git_remote_name, &git_remote_branch, &repo) {
+                Ok(updates) => {
+                    if !updates.is_empty() {
+                        // incremental update
+                        info!(
+                            "blog content git({}) update detected: {:?}",
+                            &git_url, updates
+                        );
+
+                        let mut update_md: Vec<PathBuf> = vec![];
+
+                        for update in updates {
+                            if [Delta::Deleted, Delta::Untracked].contains(&update.status) {
+                                if let Some(path) = &update.old_path {
+                                    if drop_index(
+                                        &get_md_file_basename(
+                                            path.file_name().unwrap().to_str().unwrap(),
+                                        ),
+                                        db_connection,
+                                    )
+                                    .is_ok()
+                                    {
+                                        info!(
+                                            "blog content git({}) file `{}` is detected as deleted/untracked, index dropped",
+                                            &git_url,
+                                            &path.display()
+                                        );
+                                    } else {
+                                        error!(
+                                            "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index",
+                                            &git_url,
+                                            &path.display()
+                                        )
+                                    }
+                                }
+                            } else if let Some(path) = &update.new_path {
+                                if path.components().any(|comp| comp.as_os_str() == "posts")
+                                    && path.extension().is_some_and(|ext| ext == "md")
+                                {
+                                    update_md.push(path.to_path_buf());
+                                }
+                            }
+                        }
+
+                        index_files(&update_md, db_connection);
+                    }
+                }
+                Err(e) => {
+                    error!(
+                        "failed to sync blog content git({}) updates: {}",
+                        &git_url, e
+                    );
+                }
+            }
+        }
+        Err(_) => {
+            match Repository::clone(&git_url, &git_work_dir) {
+                Ok(_repo) => {
+                    // full repo index
+                    let files = search_md(git_work_dir.join("posts"));
+                    index_files(&files, db_connection);
+                }
+                Err(e) => {
+                    error!("failed to clone blog content git({}): {}", &git_url, e);
+                }
+            }
+        }
+    };
+}

backend/api_server/src/blog/cron/check_update/utils.rs

@@ -0,0 +1,86 @@
+use dirs::home_dir;
+use std::path::PathBuf;
+
+use crate::blog::cron::check_update::index::index_file;
+use chrono::{DateTime, FixedOffset, ParseResult};
+use constants::config::general::{default_value, var_name};
+use constants::time::DEFAULT_DATE_FORMAT;
+
+use futures::executor::block_on;
+use futures::future::join_all;
+use sqlx::{Pool, Postgres};
+use tokio::runtime::Runtime;
+use tokio::spawn;
+use tracing::{error, warn};
+
+pub fn search_md(entry: PathBuf) -> Vec<PathBuf> {
+    if entry.ends_with(".git") {
+        return vec![];
+    }
+    if entry.is_dir() {
+        if let Ok(files) = entry.read_dir() {
+            files
+                .flat_map(|file| {
+                    let file_path = file.unwrap().path();
+                    search_md(file_path)
+                })
+                .collect()
+        } else {
+            vec![]
+        }
+    } else if entry.extension().is_some_and(|ext| ext == "md") {
+        vec![entry]
+    } else {
+        vec![]
+    }
+}
+
+pub fn index_files(files: &[PathBuf], db_conn: &Pool<Postgres>) {
+    let rt = Runtime::new().unwrap();
+    rt.block_on(async {
+        let files_clone = files.to_owned();
+        let handlers = files_clone
+            .into_iter()
+            .map(|file: PathBuf| {
+                let conn_clone = db_conn.clone();
+                spawn(async move {
+                    if let Err(e) = index_file(&file, &conn_clone).await {
+                        error!("failed to index markdown file `{}`: {}", file.display(), e);
+                    }
+                })
+            })
+            .collect::<Vec<_>>();
+
+        block_on(join_all(handlers));
+    });
+}
+
+pub fn expand_path(path: String) -> String {
+    if path.starts_with('~') {
+        home_dir()
+            .map(|dir| {
+                dir.join(path.strip_prefix("~/").unwrap())
+                    .into_os_string()
+                    .into_string()
+                    .unwrap()
+            })
+            .unwrap_or_else(|| {
+                warn!(
+                    "failed to parse env var `{}`, using default value `{}`",
+                    var_name::WORK_DIR,
+                    default_value::WORK_DIR
+                );
+                default_value::WORK_DIR.to_string()
+            })
+    } else {
+        path
+    }
+}
+
+pub fn parse_date(date_str: &str) -> ParseResult<DateTime<FixedOffset>> {
+    DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT)
+}
+
+pub fn get_md_file_basename(f_name: &str) -> String {
+    f_name.replace(".md", "")
+}

backend/api_server/src/blog/cron/mod.rs

@@ -7,7 +7,7 @@ pub mod check_update;
 pub fn get_tasks(db_connection: Pool<Postgres>) -> Vec<Box<dyn Task>> {
     let db_conn_clone = db_connection.clone();
     vec![
-        BasicTask::new(move || check_update::task(&db_conn_clone), "* * * * * *")
+        BasicTask::new(move || check_update::task(&db_conn_clone), "0 * * * * *")
             .unwrap()
             .to_task(),
     ]

backend/api_server/src/init/misc.rs

@@ -0,0 +1,14 @@
+use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds};
+use tracing::warn;
+
+pub fn init() {
+    // git2 option
+    unsafe {
+        if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) {
+            warn!("failed to set git2 server connect timeout: {}", e);
+        }
+        if let Err(e) = set_server_timeout_in_milliseconds(60_000) {
+            warn!("failed to set git2 server timeout: {}", e);
+        }
+    }
+}

backend/api_server/src/init/mod.rs

@@ -1,6 +1,7 @@
 mod cron;
 mod database;
 pub mod logger;
+mod misc;
 mod schema;
 pub mod server;
 
@@ -24,6 +25,8 @@ pub fn load_env() {
 }
 
 pub async fn start() {
+    misc::init();
+
     // connect database
     let db_pool = match database::init().await {
         Ok(pool) => {

backend/api_server/src/init/schema.rs

@@ -7,11 +7,11 @@ CREATE TABLE IF NOT EXISTS Post
     post_id      UUID PRIMARY KEY,
     title        VARCHAR(255) NOT NULL,
     category     VARCHAR(255) NOT NULL,
-    header_img   TEXT,
+    filename     TEXT UNIQUE  NOT NULL,
     summary      TEXT,
     sub_title    TEXT,
     content      TEXT         NOT NULL,
-    last_update  TIMESTAMPTZ  NOT NULL,
+    last_update  TIMESTAMPTZ,
     first_update TIMESTAMPTZ  NOT NULL
 );
 "##;

backend/api_server/src/main.rs

@@ -1,6 +1,7 @@
 mod blog;
 mod init;
 mod scheduler;
+mod utils;
 
 #[tokio::main]
 async fn main() {

backend/api_server/src/utils/git.rs

@@ -0,0 +1,195 @@
+/// git merge
+/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs
+/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837
+///
+use git2::{
+    build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository,
+};
+use std::path::PathBuf;
+use tracing::{error, warn};
+
+
+#[derive(Debug, Eq, PartialEq)]
+pub struct FileDelta {
+    pub old_path: Option<PathBuf>,
+    pub new_path: Option<PathBuf>,
+    pub status: Delta,
+}
+
+fn do_fetch<'a>(
+    repo: &'a Repository,
+    refs: &[&str],
+    remote: &'a mut Remote,
+) -> Result<AnnotatedCommit<'a>, git2::Error> {
+    let mut fetch_option = FetchOptions::new();
+
+    // Always fetch all tags.
+    // Perform a download and also update tips
+    fetch_option.download_tags(AutotagOption::All);
+    remote.fetch(refs, Some(&mut fetch_option), None)?;
+
+    let fetch_head = repo.find_reference("FETCH_HEAD")?;
+    repo.reference_to_annotated_commit(&fetch_head)
+}
+
+fn fast_forward(
+    repo: &Repository,
+    lb: &mut Reference,
+    rc: &AnnotatedCommit,
+) -> Result<(), git2::Error> {
+    let name = match lb.name() {
+        Some(s) => s.to_string(),
+        None => String::from_utf8_lossy(lb.name_bytes()).to_string(),
+    };
+    let msg = format!("fast-forward({}) -> {}", name, rc.id());
+    // println!("{}", msg);
+    lb.set_target(rc.id(), &msg)?;
+    repo.set_head(&name)?;
+    repo.checkout_head(Some(
+        build::CheckoutBuilder::default()
+            // For some reason, the force is required to make the working directory actually get updated
+            // I suspect we should be adding some logic to handle dirty working directory states,
+            // but this is just an example so maybe not.
+            .force(),
+    ))?;
+    Ok(())
+}
+
+fn normal_merge(
+    repo: &Repository,
+    local: &AnnotatedCommit,
+    remote: &AnnotatedCommit,
+) -> Result<(), git2::Error> {
+    let local_tree = repo.find_commit(local.id())?.tree()?;
+    let remote_tree = repo.find_commit(remote.id())?.tree()?;
+    let ancestor = repo
+        .find_commit(repo.merge_base(local.id(), remote.id())?)?
+        .tree()?;
+    let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?;
+
+    if idx.has_conflicts() {
+        warn!("merge conflicts detected");
+        repo.checkout_index(Some(&mut idx), None)?;
+        return Ok(());
+    }
+    let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?;
+    // now create the merge commit
+    let msg = format!("merge({}) -> {}", remote.id(), local.id());
+    let sig = repo.signature()?;
+    let local_commit = repo.find_commit(local.id())?;
+    let remote_commit = repo.find_commit(remote.id())?;
+
+    // Do our merge commit and set the current branch head to that commit.
+    let _merge_commit = repo.commit(
+        Some("HEAD"),
+        &sig,
+        &sig,
+        &msg,
+        &result_tree,
+        &[&local_commit, &remote_commit],
+    )?;
+    // Set working tree to match head.
+    repo.checkout_head(None)?;
+    Ok(())
+}
+
+fn do_merge<'a>(
+    repo: &'a Repository,
+    remote_branch: &str,
+    fetch_commit: AnnotatedCommit<'a>,
+) -> Result<Vec<FileDelta>, git2::Error> {
+    // 1. do a merge analysis
+    let analysis = repo.merge_analysis(&[&fetch_commit])?;
+
+    let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?;
+
+    // 2. Do the appropriate merge
+    if analysis.0.is_fast_forward() {
+        // println!("Doing a fast-forward");
+        // do a fast-forward
+        let ref_name = format!("refs/heads/{}", remote_branch);
+        match repo.find_reference(&ref_name) {
+            Ok(mut r) => {
+                fast_forward(repo, &mut r, &fetch_commit)?;
+            }
+            Err(_) => {
+                // The branch doesn't exist, so set the reference to the
+                // commit directly. Usually this is because you are pulling
+                // into an empty repository.
+                repo.reference(
+                    &ref_name,
+                    fetch_commit.id(),
+                    true,
+                    &format!("({}) -> {}", remote_branch, fetch_commit.id()),
+                )?;
+                repo.set_head(&ref_name)?;
+                repo.checkout_head(Some(
+                    build::CheckoutBuilder::default()
+                        .allow_conflicts(true)
+                        .conflict_style_merge(true)
+                        .force(),
+                ))?;
+            }
+        };
+    } else if analysis.0.is_normal() {
+        // do a normal merge
+        normal_merge(repo, &head_commit, &fetch_commit)?;
+    }
+    // else {
+    //     // already up-to-date
+    //     println!("Nothing to do...");
+    // }
+
+    calculate_diff(repo, &head_commit, &fetch_commit)
+}
+
+fn calculate_diff(
+    repo: &Repository,
+    local: &AnnotatedCommit,
+    remote: &AnnotatedCommit,
+) -> Result<Vec<FileDelta>, git2::Error> {
+    let local_tree = repo.find_commit(local.id())?.tree()?;
+    let remote_tree = repo.find_commit(remote.id())?.tree()?;
+    let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?;
+
+    let mut updated_files: Vec<FileDelta> = Vec::new();
+
+    if let Err(e) = diff.foreach(
+        &mut |d, _| {
+            // let delta = d.status();
+            //
+            // let n_path = d.new_file().path();
+            // let o_path = d.old_file().path();
+            //
+            // if let Some(path) = n_path {
+            //     updated_files.insert(path.to_owned());
+            // }
+            // if let Some(path) = o_path {
+            //     updated_files.insert(path.to_owned());
+            // }
+            updated_files.push(FileDelta {
+                old_path: d.old_file().path().map(PathBuf::from),
+                new_path: d.new_file().path().map(PathBuf::from),
+                status: d.status(),
+            });
+            true // Continue
+        },
+        None,
+        None,
+        None,
+    ) {
+        error!("error at calculating diff: {:?}", e);
+    }
+
+    Ok(updated_files.into_iter().collect())
+}
+
+pub fn sync(
+    remote_name: &str,
+    remote_branch: &str,
+    repo: &Repository,
+) -> Result<Vec<FileDelta>, git2::Error> {
+    let mut remote = repo.find_remote(remote_name)?;
+    let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?;
+    do_merge(repo, remote_branch, fetch_commit)
+}

backend/api_server/src/utils/mod.rs

@@ -0,0 +1 @@
+pub mod git;

backend/constants/src/config.rs

@@ -35,9 +35,14 @@ pub mod general {
     pub mod var_name {
         pub const LOG_DIR: &str = "LOG_DIR";
         pub const WORK_DIR: &str = "WORK_DIR";
+        pub const BLOG_GIT_URL: &str = "BLOG_GIT_URL";
+        pub const BLOG_GIT_REMOTE_BRANCH: &str = "BLOG_GIT_REMOTE_BRANCH";
+        pub const BLOG_GIT_REMOTE_NAME: &str = "BLOG_GIT_REMOTE_NAME";
     }
 
     pub mod default_value {
         pub const WORK_DIR: &str = "~/.nocturne";
+        pub const BLOG_GIT_REMOTE_BRANCH: &str = "master";
+        pub const BLOG_GIT_REMOTE_NAME: &str = "origin";
     }
 }

backend/constants/src/time.rs

@@ -1 +1,2 @@
 pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z";
+pub const DEFAULT_DATE_FORMAT: &str = "%Y-%m-%d%zT%H%M%S";

backend/markdown/Cargo.toml

@@ -5,10 +5,10 @@ edition = "2021"
 
 [dependencies]
 uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] }
-chrono = { version = "0.4.38", features = ["serde", "alloc"] }
 serde = { version = "1.0.209", features = ["derive"] }
 serde_json = "1.0.128"
 regex = "1.10.4"
 yaml-rust2 = "0.9.0"
 sha2 = "0.10.8"
 base64 = "0.22.1"
+thiserror = "2.0.3"

backend/markdown/src/error.rs

@@ -0,0 +1,16 @@
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("missing front-matter field: id")]
+    MissingID,
+
+    #[error("invalid file id: {}", msg)]
+    InvalidID { msg: String },
+
+    #[error("invalid front-matter: {}", msg)]
+    InvalidFrontMatter { msg: String },
+
+    #[error("fs error: {}", msg)]
+    FSError { msg: String },
+}

backend/markdown/src/lib.rs

@@ -1,35 +1,48 @@
+mod error;
 mod utils;
 
-use std::error::Error;
 use std::fs;
 use std::path::Path;
 use std::str::FromStr;
 
+use crate::error::Error;
+use crate::utils::front_matter::{parse_front_matter, split_md_front_matter};
+
 use base64::{engine::general_purpose, Engine};
 use serde::Serialize;
 use serde_json::Value;
 use sha2::{Digest, Sha256};
 use uuid::Uuid;
 
-use crate::utils::front_matter::{parse_front_matter, split_md_front_matter};
-
 #[derive(Default, Debug, Serialize)]
 pub struct MdFile {
     pub file_id: Uuid,
-    pub file_name: String,
+    pub filename: String,
     pub meta: Value,
     pub content: String,
     pub hash: String,
 }
 
 impl MdFile {
-    pub async fn from_file(file_path: &str) -> Result<MdFile, Box<dyn Error>> {
+    pub fn from_file(file_path: &Path) -> Result<MdFile, Error> {
-        let raw_file = fs::read_to_string(file_path)?;
+        let raw_file = match fs::read_to_string(file_path) {
+            Ok(file) => file,
+            Err(e) => return Err(Error::FSError { msg: e.to_string() }),
+        };
         let raw_file_bytes = raw_file.as_bytes().to_vec();
         let (front_matter_str, content) = split_md_front_matter(raw_file);
-        let front_matter = parse_front_matter(&front_matter_str)?;
+        let front_matter = match parse_front_matter(&front_matter_str) {
+            Ok(front_matter) => front_matter,
+            Err(e) => return Err(Error::InvalidFrontMatter { msg: e.to_string() }),
+        };
         let hash = calculate_hash_base64(&raw_file_bytes);
-        let file_id = Uuid::from_str(front_matter["id"].to_string().as_str())?;
+        let file_id = match front_matter["id"].as_str() {
+            Some(id) => match Uuid::from_str(id) {
+                Ok(id) => id,
+                Err(e) => return Err(Error::InvalidID { msg: e.to_string() }),
+            },
+            None => return Err(Error::MissingID),
+        };
         let file_name = Path::new(file_path)
             .file_name()
             .unwrap()
@@ -39,7 +52,7 @@ impl MdFile {
 
         Ok(MdFile {
             file_id,
-            file_name,
+            filename: file_name,
             meta: front_matter,
             content,
             hash,

backend/markdown/src/utils/front_matter.rs

@@ -16,8 +16,12 @@ pub fn split_md_front_matter(raw_file: String) -> (String, String) {
 }
 
 pub fn parse_front_matter(front_matter: &str) -> Result<Value, Box<dyn Error>> {
-    let parsed_f_matter = YamlLoader::load_from_str(front_matter)?[0].clone();
+    let parsed_f_matter = YamlLoader::load_from_str(front_matter)?;
-    Ok(Value::from(yaml_to_serde(&parsed_f_matter)))
+    if parsed_f_matter.is_empty() {
+        Ok(Value::String(String::new()))
+    } else {
+        Ok(yaml_to_serde(&parsed_f_matter[0]))
+    }
 }
 
 #[cfg(test)]