feat(blog::components::check+update) update

backend/markdown/src/lib.rs

@@ -1,4 +1,4 @@
-mod error;
+pub mod error;
 mod utils;
 
 use std::{fs, path::Path, str::FromStr};
@@ -8,12 +8,12 @@ use crate::{
     utils::front_matter::{parse_front_matter, split_md_front_matter},
 };
 
-use base64::{Engine, engine::general_purpose};
+use base64::{engine::general_purpose, Engine};
 use serde::Serialize;
 use serde_json::Value;
 use sha2::{Digest, Sha256};
 
-#[derive(Default, Debug, Serialize)]
+#[derive(Default, Debug, Serialize, Clone)]
 pub struct MdFile {
     pub meta: Value,
     pub content: String,

backend/server/src/modules/blog/components/check_update/changes.rs

@@ -1,30 +1,67 @@
-use git2::Delta;
 use std::path::PathBuf;
+
+use markdown::MdFile;
+
 use uuid::Uuid;
 
+#[allow(dead_code)]
 #[derive(Debug)]
 pub enum Change {
-    DELETE {
+    Delete(Delete),
-        uuid: Uuid,
+    Move(Move),
-    },
+    Update(Update),
-    RENAME {
+    Create(Create),
-        uuid: Uuid,
-        from: PathBuf,
-        to: PathBuf,
-    },
-    UPDATE {
-        uuid: Uuid,
-        content: String,
-    },
-    CREATE {
-        uuid: Uuid,
-        path: PathBuf,
-        content: String,
-    },
 }
 
+#[allow(dead_code)]
 #[derive(Debug, Clone)]
-pub struct DeltaIdx {
+pub struct Delete {
-    pub delta_type: Delta,
+    pub uuid: Uuid,
-    pub content: String,
+    pub path: PathBuf,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone)]
+pub struct Move {
+    pub uuid: Uuid,
+    pub from: PathBuf,
+    pub to: PathBuf,
+    pub payload: MdFile,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone)]
+pub struct Update {
+    pub uuid: Uuid,
+    pub payload: MdFile,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone)]
+pub struct Create {
+    pub uuid: Uuid,
+    pub path: PathBuf,
+    pub payload: MdFile,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct CreateDelete2UpdateSlot {
+    pub create: Option<Create>,
+    pub delete: Option<Delete>,
+}
+
+impl CreateDelete2UpdateSlot {
+    pub fn to_change(&self) -> Option<Change> {
+        match (&self.create, &self.delete) {
+            (Some(create), Some(delete)) => Some(Change::Move(Move {
+                uuid: create.uuid,
+                from: delete.path.clone(),
+                to: create.path.clone(),
+                payload: create.payload.clone(),
+            })),
+            (Some(create), None) => Some(Change::Create(create.clone())),
+            (None, Some(delete)) => Some(Change::Delete(delete.clone())),
+            (None, None) => None,
+        }
+    }
 }

backend/server/src/modules/blog/components/check_update/error.rs

@@ -1,7 +1,13 @@
+use markdown::error::Error as MarkdownError;
+
 use thiserror::Error;
 
+#[allow(dead_code)]
 #[derive(Error, Debug)]
 pub enum Error {
+    #[error("markdown file error")]
+    MdFile(#[from] MarkdownError),
+
     #[error("missing essential field `{}` for file `{}`", field, filename)]
     MissingField { field: String, filename: String },
 

backend/server/src/modules/blog/components/check_update/mod.rs

@@ -1,20 +1,18 @@
-use crate::constants::config::general::{default_value, var_name};
-use macros::panic_with_log;
-use std::sync::Arc;
 use std::{
     env,
     path::{Path, PathBuf},
+    sync::Arc,
 };
 
-use crate::modules::blog::components::check_update::pull::{analyze_deltas, fetch_deltas};
+use crate::{
-use crate::modules::blog::components::check_update::{
+    constants::config::general::{default_value, var_name},
-    // index::drop_index,
+    modules::blog::components::check_update::{pull::fetch_deltas, utils::expand_path},
-    // utils::{expand_path, index_files, search_md},
+    scheduler::task_func::AsyncTaskFunc,
-    utils::expand_path,
 };
-use crate::scheduler::task_func::AsyncTaskFunc;
+use macros::panic_with_log;
+
 use sqlx::PgPool;
-use tracing::{info, trace, warn, Level};
+use tracing::{trace, warn, Level};
 
 // mod error;
 // mod index;
@@ -95,5 +93,5 @@ async fn workflow(args: Arc<Args>) {
         return;
     }
 
-    let updates = analyze_deltas(deltas, &args.git_work_dir).await;
+    // todo: load delta to database
 }

backend/server/src/modules/blog/components/check_update/pull.rs

@@ -1,20 +1,26 @@
-use crate::modules::blog::components::check_update::changes::{Change, DeltaIdx};
-use crate::modules::blog::components::check_update::error::Error;
-use crate::utils::git;
-use crate::utils::git::FileDelta;
-use git2::{Delta, Repository};
-use markdown::MdFile;
 use std::collections::HashMap;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
+
+use crate::{
+    modules::blog::components::check_update::{
+        changes::{Change, Create, Delete, Update},
+        error::Error,
+    },
+    utils::{git, git::FileDelta},
+};
+use markdown::MdFile;
+
+use crate::modules::blog::components::check_update::changes::CreateDelete2UpdateSlot;
+use git2::{Delta, Repository};
 use tracing::error;
 use uuid::Uuid;
 
 pub async fn fetch_deltas(
     git_url: &str,
     git_remote_name: &str,
-    git_branch: &str,
+    git_remote_branch: &str,
     git_work_dir: &PathBuf,
-) -> Result<Vec<FileDelta>, Error> {
+) -> Result<Vec<Change>, Error> {
     let repo = match Repository::open(git_work_dir) {
         Ok(repo) => repo,
         Err(_) => match Repository::clone(git_url, git_work_dir) {
@@ -26,114 +32,87 @@ pub async fn fetch_deltas(
         },
     };
 
-    let updates = match git::sync(git_remote_name, git_branch, &repo) {
+    let mut remote = repo.find_remote(git_remote_name)?;
-        Ok(updates) => updates,
+    let fetched_commit = git::do_fetch(&repo, &[git_remote_branch], &mut remote)?;
-        Err(e) => {
+    let commit_deltas = git::calculate_diff(
-            error!(
+        &repo,
-                "failed to sync blog content git({}) updates: {}",
+        &repo.reference_to_annotated_commit(&repo.head()?)?,
-                &git_url, e
+        &fetched_commit,
-            );
+    )?;
-            return Err(e.into());
-        }
-    };
 
-    Ok(updates)
+    // dbg!(&commit_deltas);
-}
-
-pub async fn analyze_deltas(file_deltas: Vec<FileDelta>, git_work_dir: &Path) -> Vec<Change> {
-    dbg!(&file_deltas);
     let mut changes: Vec<Change> = vec![];
-    let mut merge_deltas: Vec<FileDelta> = vec![];
+    let mut non_delete_deltas: Vec<FileDelta> = vec![];
 
-    for delta in file_deltas.into_iter() {
+    // process deleted deltas before merging commit to prevent "file not found" os error
-        if delta.status == Delta::Modified {
+    for delta in commit_deltas.into_iter() {
+        if delta.status == Delta::Deleted {
             let file_path = &git_work_dir.join(
                 delta
                     .new_path
                     .expect("cannot parse modified post `new_path`"),
             );
-            let post = match MdFile::from_file(file_path) {
+            if let Some(id) = extract_post_id(&MdFile::from_file(file_path)?) {
-                Ok(post) => post,
+                changes.push(Change::Delete(Delete {
-                Err(e) => {
-                    error!("md file parsing error: {}", e);
-                    continue;
-                }
-            };
-
-            let id = match extract_post_id(&post) {
-                Some(id) => id,
-                None => {
-                    // todo: add to unresolved error list
-                    error!(
-                        "failed to parse uuid on post {}",
-                        file_path.to_string_lossy()
-                    );
-                    continue;
-                }
-            };
-
-            changes.push(Change::UPDATE {
                     uuid: id,
-                content: post.content,
+                    path: file_path.clone(),
-            })
+                }))
             } else {
-            merge_deltas.push(delta)
+                error!("failed to parse uuid for file {}", file_path.display());
-        };
+                // todo: add to unresolved error list
+            }
+        } else {
+            non_delete_deltas.push(delta);
+        }
     }
 
-    // merge some addition and remove deltas to rename/move changes
+    git::do_merge(&repo, git_remote_branch, fetched_commit)?;
-    let delta_idx = {
+
-        let mut delta_idx: HashMap<Uuid, Vec<DeltaIdx>> = HashMap::new();
+    // process adds and modifies deltas
-        for file_delta in merge_deltas.into_iter() {
+    for delta in non_delete_deltas.into_iter() {
         let file_path = &git_work_dir.join(
-                file_delta
+            delta
                 .new_path
                 .expect("cannot parse modified post `new_path`"),
         );
-            match file_delta.status {
+        match delta.status {
-                // todo: split impl
+            Delta::Added => {
-                Delta::Added | Delta::Deleted => {
+                let md_file = MdFile::from_file(file_path)?;
-                    let file_path_str = file_path.to_string_lossy();
+                if let Some(id) = extract_post_id(&md_file) {
-
+                    changes.push(Change::Create(Create {
-                    let post = match MdFile::from_file(file_path) {
+                        uuid: id,
-                        Ok(post) => post,
+                        path: file_path.clone(),
-                        Err(e) => {
+                        payload: md_file,
-                            error!("md file parsing error ({}): {}", file_path_str, e);
+                    }))
-                            continue;
+                } else {
-                        }
+                    error!("failed to parse uuid for file {}", file_path.display());
-                    };
-
-                    let id = match extract_post_id(&post) {
-                        Some(id) => id,
-                        None => {
                     // todo: add to unresolved error list
-                            error!("failed to parse uuid on post {}", file_path_str);
-                            continue;
                 }
-                    };
-
-                    let file_delta_idx = DeltaIdx {
-                        delta_type: file_delta.status,
-                        content: post.content,
-                    };
-                    delta_idx
-                        .entry(id)
-                        // todo: change to shallow copy
-                        .and_modify(|v| v.push(file_delta_idx.clone()))
-                        .or_insert(vec![file_delta_idx]);
             }
+            Delta::Modified => {
+                let md_file = MdFile::from_file(file_path)?;
+                if let Some(id) = extract_post_id(&md_file) {
+                    changes.push(Change::Update(Update {
+                        uuid: id,
+                        payload: md_file,
+                    }))
+                } else {
+                    error!("failed to parse uuid for file {}", file_path.display());
+                    // todo: add to unresolved error list
+                }
+            }
+            Delta::Deleted => continue,
             _ => {
-                    error!("unsupported delta on file {}", file_path.to_string_lossy());
+                error!("unsupported delta type on file {}", file_path.display());
                 continue;
             }
-            };
         }
-        delta_idx
+    }
-    };
 
-    dbg!(&delta_idx);
+    // merge adds and deletes deltas to moves
+    let changes = dbg!(merge_changes(changes));
 
-    changes
+    Ok(changes)
 }
 
 fn extract_post_id(post: &MdFile) -> Option<Uuid> {
@@ -145,3 +124,30 @@ fn extract_post_id(post: &MdFile) -> Option<Uuid> {
         None => None,
     }
 }
+
+fn merge_changes(changes: Vec<Change>) -> Vec<Change> {
+    let mut slots: HashMap<Uuid, CreateDelete2UpdateSlot> = HashMap::new();
+    let mut merged: Vec<Change> = Vec::new();
+
+    for change in changes {
+        match change {
+            Change::Create(create) => {
+                let id = create.uuid;
+                slots.entry(id).or_default().create = Some(create);
+            }
+            Change::Delete(delete) => {
+                let id = delete.uuid;
+                slots.entry(id).or_default().delete = Some(delete);
+            }
+            _ => merged.push(change),
+        }
+    }
+
+    for slot in slots.values() {
+        if let Some(change) = slot.to_change() {
+            merged.push(change);
+        }
+    }
+
+    merged
+}

backend/server/src/utils/git.rs

@@ -1,13 +1,14 @@
-/// git merge
+/// git operations
 /// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs
 /// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837
 ///
 use git2::{
-    AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, build,
+    build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository,
 };
 use std::path::PathBuf;
 use tracing::{error, warn};
 
+
 #[derive(Debug, Eq, PartialEq)]
 pub struct FileDelta {
     pub old_path: Option<PathBuf>,
@@ -15,7 +16,7 @@ pub struct FileDelta {
     pub status: Delta,
 }
 
-fn do_fetch<'a>(
+pub fn do_fetch<'a>(
     repo: &'a Repository,
     refs: &[&str],
     remote: &'a mut Remote,
@@ -81,11 +82,11 @@ fn normal_merge(
     Ok(())
 }
 
-fn do_merge<'a>(
+pub fn do_merge<'a>(
     repo: &'a Repository,
     remote_branch: &str,
     fetch_commit: AnnotatedCommit<'a>,
-) -> Result<Vec<FileDelta>, git2::Error> {
+) -> Result<(), git2::Error> {
     let analysis = repo.merge_analysis(&[&fetch_commit])?;
     let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?;
 
@@ -114,10 +115,11 @@ fn do_merge<'a>(
     } else if analysis.0.is_normal() {
         normal_merge(repo, &head_commit, &fetch_commit)?;
     }
-    calculate_diff(repo, &head_commit, &fetch_commit)
+
+    Ok(())
 }
 
-fn calculate_diff(
+pub fn calculate_diff(
     repo: &Repository,
     local: &AnnotatedCommit,
     remote: &AnnotatedCommit,
@@ -147,6 +149,7 @@ fn calculate_diff(
     Ok(updated_files.into_iter().collect())
 }
 
+#[allow(dead_code)]
 pub fn sync(
     remote_name: &str,
     remote_branch: &str,
@@ -154,5 +157,11 @@ pub fn sync(
 ) -> Result<Vec<FileDelta>, git2::Error> {
     let mut remote = repo.find_remote(remote_name)?;
     let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?;
-    do_merge(repo, remote_branch, fetch_commit)
+    let delta = calculate_diff(
+        repo,
+        &repo.reference_to_annotated_commit(&repo.head()?)?,
+        &fetch_commit,
+    );
+    do_merge(repo, remote_branch, fetch_commit)?;
+    delta
 }