diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index e616146..471ed53 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -1,4 +1,4 @@ -mod error; +pub mod error; mod utils; use std::{fs, path::Path, str::FromStr}; @@ -8,12 +8,12 @@ use crate::{ utils::front_matter::{parse_front_matter, split_md_front_matter}, }; -use base64::{Engine, engine::general_purpose}; +use base64::{engine::general_purpose, Engine}; use serde::Serialize; use serde_json::Value; use sha2::{Digest, Sha256}; -#[derive(Default, Debug, Serialize)] +#[derive(Default, Debug, Serialize, Clone)] pub struct MdFile { pub meta: Value, pub content: String, diff --git a/backend/server/src/modules/blog/components/check_update/changes.rs b/backend/server/src/modules/blog/components/check_update/changes.rs index 76dd91b..c732b12 100644 --- a/backend/server/src/modules/blog/components/check_update/changes.rs +++ b/backend/server/src/modules/blog/components/check_update/changes.rs @@ -1,30 +1,67 @@ -use git2::Delta; use std::path::PathBuf; + +use markdown::MdFile; + use uuid::Uuid; +#[allow(dead_code)] #[derive(Debug)] pub enum Change { - DELETE { - uuid: Uuid, - }, - RENAME { - uuid: Uuid, - from: PathBuf, - to: PathBuf, - }, - UPDATE { - uuid: Uuid, - content: String, - }, - CREATE { - uuid: Uuid, - path: PathBuf, - content: String, - }, + Delete(Delete), + Move(Move), + Update(Update), + Create(Create), } +#[allow(dead_code)] #[derive(Debug, Clone)] -pub struct DeltaIdx { - pub delta_type: Delta, - pub content: String, +pub struct Delete { + pub uuid: Uuid, + pub path: PathBuf, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct Move { + pub uuid: Uuid, + pub from: PathBuf, + pub to: PathBuf, + pub payload: MdFile, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct Update { + pub uuid: Uuid, + pub payload: MdFile, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct Create { + pub uuid: Uuid, + pub path: PathBuf, + pub payload: MdFile, +} + +#[derive(Debug, Clone, Default)] +pub struct CreateDelete2UpdateSlot { + pub create: Option, + pub delete: Option, +} + +impl CreateDelete2UpdateSlot { + pub fn to_change(&self) -> Option { + match (&self.create, &self.delete) { + (Some(create), Some(delete)) => Some(Change::Move(Move { + uuid: create.uuid, + from: delete.path.clone(), + to: create.path.clone(), + payload: create.payload.clone(), + })), + (Some(create), None) => Some(Change::Create(create.clone())), + (None, Some(delete)) => Some(Change::Delete(delete.clone())), + (None, None) => None, + } + } } diff --git a/backend/server/src/modules/blog/components/check_update/error.rs b/backend/server/src/modules/blog/components/check_update/error.rs index f7955b3..56fb9ab 100644 --- a/backend/server/src/modules/blog/components/check_update/error.rs +++ b/backend/server/src/modules/blog/components/check_update/error.rs @@ -1,7 +1,13 @@ +use markdown::error::Error as MarkdownError; + use thiserror::Error; +#[allow(dead_code)] #[derive(Error, Debug)] pub enum Error { + #[error("markdown file error")] + MdFile(#[from] MarkdownError), + #[error("missing essential field `{}` for file `{}`", field, filename)] MissingField { field: String, filename: String }, diff --git a/backend/server/src/modules/blog/components/check_update/mod.rs b/backend/server/src/modules/blog/components/check_update/mod.rs index 474434f..5d13c39 100644 --- a/backend/server/src/modules/blog/components/check_update/mod.rs +++ b/backend/server/src/modules/blog/components/check_update/mod.rs @@ -1,20 +1,18 @@ -use crate::constants::config::general::{default_value, var_name}; -use macros::panic_with_log; -use std::sync::Arc; use std::{ env, path::{Path, PathBuf}, + sync::Arc, }; -use crate::modules::blog::components::check_update::pull::{analyze_deltas, fetch_deltas}; -use crate::modules::blog::components::check_update::{ - // index::drop_index, - // utils::{expand_path, index_files, search_md}, - utils::expand_path, +use crate::{ + constants::config::general::{default_value, var_name}, + modules::blog::components::check_update::{pull::fetch_deltas, utils::expand_path}, + scheduler::task_func::AsyncTaskFunc, }; -use crate::scheduler::task_func::AsyncTaskFunc; +use macros::panic_with_log; + use sqlx::PgPool; -use tracing::{info, trace, warn, Level}; +use tracing::{trace, warn, Level}; // mod error; // mod index; @@ -95,5 +93,5 @@ async fn workflow(args: Arc) { return; } - let updates = analyze_deltas(deltas, &args.git_work_dir).await; + // todo: load delta to database } diff --git a/backend/server/src/modules/blog/components/check_update/pull.rs b/backend/server/src/modules/blog/components/check_update/pull.rs index 38a471a..144485f 100644 --- a/backend/server/src/modules/blog/components/check_update/pull.rs +++ b/backend/server/src/modules/blog/components/check_update/pull.rs @@ -1,20 +1,26 @@ -use crate::modules::blog::components::check_update::changes::{Change, DeltaIdx}; -use crate::modules::blog::components::check_update::error::Error; -use crate::utils::git; -use crate::utils::git::FileDelta; -use git2::{Delta, Repository}; -use markdown::MdFile; use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; + +use crate::{ + modules::blog::components::check_update::{ + changes::{Change, Create, Delete, Update}, + error::Error, + }, + utils::{git, git::FileDelta}, +}; +use markdown::MdFile; + +use crate::modules::blog::components::check_update::changes::CreateDelete2UpdateSlot; +use git2::{Delta, Repository}; use tracing::error; use uuid::Uuid; pub async fn fetch_deltas( git_url: &str, git_remote_name: &str, - git_branch: &str, + git_remote_branch: &str, git_work_dir: &PathBuf, -) -> Result, Error> { +) -> Result, Error> { let repo = match Repository::open(git_work_dir) { Ok(repo) => repo, Err(_) => match Repository::clone(git_url, git_work_dir) { @@ -26,114 +32,87 @@ pub async fn fetch_deltas( }, }; - let updates = match git::sync(git_remote_name, git_branch, &repo) { - Ok(updates) => updates, - Err(e) => { - error!( - "failed to sync blog content git({}) updates: {}", - &git_url, e - ); - return Err(e.into()); - } - }; + let mut remote = repo.find_remote(git_remote_name)?; + let fetched_commit = git::do_fetch(&repo, &[git_remote_branch], &mut remote)?; + let commit_deltas = git::calculate_diff( + &repo, + &repo.reference_to_annotated_commit(&repo.head()?)?, + &fetched_commit, + )?; - Ok(updates) -} - -pub async fn analyze_deltas(file_deltas: Vec, git_work_dir: &Path) -> Vec { - dbg!(&file_deltas); + // dbg!(&commit_deltas); let mut changes: Vec = vec![]; - let mut merge_deltas: Vec = vec![]; + let mut non_delete_deltas: Vec = vec![]; - for delta in file_deltas.into_iter() { - if delta.status == Delta::Modified { + // process deleted deltas before merging commit to prevent "file not found" os error + for delta in commit_deltas.into_iter() { + if delta.status == Delta::Deleted { let file_path = &git_work_dir.join( delta .new_path .expect("cannot parse modified post `new_path`"), ); - let post = match MdFile::from_file(file_path) { - Ok(post) => post, - Err(e) => { - error!("md file parsing error: {}", e); - continue; - } - }; - - let id = match extract_post_id(&post) { - Some(id) => id, - None => { - // todo: add to unresolved error list - error!( - "failed to parse uuid on post {}", - file_path.to_string_lossy() - ); - continue; - } - }; - - changes.push(Change::UPDATE { - uuid: id, - content: post.content, - }) + if let Some(id) = extract_post_id(&MdFile::from_file(file_path)?) { + changes.push(Change::Delete(Delete { + uuid: id, + path: file_path.clone(), + })) + } else { + error!("failed to parse uuid for file {}", file_path.display()); + // todo: add to unresolved error list + } } else { - merge_deltas.push(delta) - }; + non_delete_deltas.push(delta); + } } - // merge some addition and remove deltas to rename/move changes - let delta_idx = { - let mut delta_idx: HashMap> = HashMap::new(); - for file_delta in merge_deltas.into_iter() { - let file_path = &git_work_dir.join( - file_delta - .new_path - .expect("cannot parse modified post `new_path`"), - ); - match file_delta.status { - // todo: split impl - Delta::Added | Delta::Deleted => { - let file_path_str = file_path.to_string_lossy(); + git::do_merge(&repo, git_remote_branch, fetched_commit)?; - let post = match MdFile::from_file(file_path) { - Ok(post) => post, - Err(e) => { - error!("md file parsing error ({}): {}", file_path_str, e); - continue; - } - }; - - let id = match extract_post_id(&post) { - Some(id) => id, - None => { - // todo: add to unresolved error list - error!("failed to parse uuid on post {}", file_path_str); - continue; - } - }; - - let file_delta_idx = DeltaIdx { - delta_type: file_delta.status, - content: post.content, - }; - delta_idx - .entry(id) - // todo: change to shallow copy - .and_modify(|v| v.push(file_delta_idx.clone())) - .or_insert(vec![file_delta_idx]); + // process adds and modifies deltas + for delta in non_delete_deltas.into_iter() { + let file_path = &git_work_dir.join( + delta + .new_path + .expect("cannot parse modified post `new_path`"), + ); + match delta.status { + Delta::Added => { + let md_file = MdFile::from_file(file_path)?; + if let Some(id) = extract_post_id(&md_file) { + changes.push(Change::Create(Create { + uuid: id, + path: file_path.clone(), + payload: md_file, + })) + } else { + error!("failed to parse uuid for file {}", file_path.display()); + // todo: add to unresolved error list } - _ => { - error!("unsupported delta on file {}", file_path.to_string_lossy()); - continue; + } + Delta::Modified => { + let md_file = MdFile::from_file(file_path)?; + if let Some(id) = extract_post_id(&md_file) { + changes.push(Change::Update(Update { + uuid: id, + payload: md_file, + })) + } else { + error!("failed to parse uuid for file {}", file_path.display()); + // todo: add to unresolved error list } - }; + } + Delta::Deleted => continue, + _ => { + error!("unsupported delta type on file {}", file_path.display()); + continue; + } } - delta_idx - }; + } - dbg!(&delta_idx); + // merge adds and deletes deltas to moves + let changes = dbg!(merge_changes(changes)); - changes + Ok(changes) } fn extract_post_id(post: &MdFile) -> Option { @@ -145,3 +124,30 @@ fn extract_post_id(post: &MdFile) -> Option { None => None, } } + +fn merge_changes(changes: Vec) -> Vec { + let mut slots: HashMap = HashMap::new(); + let mut merged: Vec = Vec::new(); + + for change in changes { + match change { + Change::Create(create) => { + let id = create.uuid; + slots.entry(id).or_default().create = Some(create); + } + Change::Delete(delete) => { + let id = delete.uuid; + slots.entry(id).or_default().delete = Some(delete); + } + _ => merged.push(change), + } + } + + for slot in slots.values() { + if let Some(change) = slot.to_change() { + merged.push(change); + } + } + + merged +} diff --git a/backend/server/src/utils/git.rs b/backend/server/src/utils/git.rs index 93f2a13..67af5a6 100644 --- a/backend/server/src/utils/git.rs +++ b/backend/server/src/utils/git.rs @@ -1,13 +1,14 @@ -/// git merge +/// git operations /// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs /// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 /// use git2::{ - AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, build, + build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, }; use std::path::PathBuf; use tracing::{error, warn}; + #[derive(Debug, Eq, PartialEq)] pub struct FileDelta { pub old_path: Option, @@ -15,7 +16,7 @@ pub struct FileDelta { pub status: Delta, } -fn do_fetch<'a>( +pub fn do_fetch<'a>( repo: &'a Repository, refs: &[&str], remote: &'a mut Remote, @@ -81,11 +82,11 @@ fn normal_merge( Ok(()) } -fn do_merge<'a>( +pub fn do_merge<'a>( repo: &'a Repository, remote_branch: &str, fetch_commit: AnnotatedCommit<'a>, -) -> Result, git2::Error> { +) -> Result<(), git2::Error> { let analysis = repo.merge_analysis(&[&fetch_commit])?; let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; @@ -114,10 +115,11 @@ fn do_merge<'a>( } else if analysis.0.is_normal() { normal_merge(repo, &head_commit, &fetch_commit)?; } - calculate_diff(repo, &head_commit, &fetch_commit) + + Ok(()) } -fn calculate_diff( +pub fn calculate_diff( repo: &Repository, local: &AnnotatedCommit, remote: &AnnotatedCommit, @@ -147,6 +149,7 @@ fn calculate_diff( Ok(updated_files.into_iter().collect()) } +#[allow(dead_code)] pub fn sync( remote_name: &str, remote_branch: &str, @@ -154,5 +157,11 @@ pub fn sync( ) -> Result, git2::Error> { let mut remote = repo.find_remote(remote_name)?; let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; - do_merge(repo, remote_branch, fetch_commit) + let delta = calculate_diff( + repo, + &repo.reference_to_annotated_commit(&repo.head()?)?, + &fetch_commit, + ); + do_merge(repo, remote_branch, fetch_commit)?; + delta }