diff --git a/backend/server/src/constants/config.rs b/backend/server/src/constants/config.rs index bf6ba77..74ece8c 100644 --- a/backend/server/src/constants/config.rs +++ b/backend/server/src/constants/config.rs @@ -1,3 +1,5 @@ +// todo: rebuild with static OnceCell + // Database related pub mod database { pub mod var_name { diff --git a/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql b/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql index e69de29..7dc5107 100644 --- a/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql +++ b/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql @@ -0,0 +1 @@ +SELECT \ No newline at end of file diff --git a/backend/server/src/modules/blog/api/get_post/mod.rs b/backend/server/src/modules/blog/api/get_post/mod.rs index 3bdf2f8..4722392 100644 --- a/backend/server/src/modules/blog/api/get_post/mod.rs +++ b/backend/server/src/modules/blog/api/get_post/mod.rs @@ -1,9 +1,9 @@ use macros::dev_consume; use axum::{ + Json, extract::{Path, State}, response::IntoResponse, - Json, }; use serde_json::json; use sqlx::PgPool; diff --git a/backend/server/src/modules/blog/components/check_update/changes.rs b/backend/server/src/modules/blog/components/check_update/changes.rs new file mode 100644 index 0000000..76dd91b --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/changes.rs @@ -0,0 +1,30 @@ +use git2::Delta; +use std::path::PathBuf; +use uuid::Uuid; + +#[derive(Debug)] +pub enum Change { + DELETE { + uuid: Uuid, + }, + RENAME { + uuid: Uuid, + from: PathBuf, + to: PathBuf, + }, + UPDATE { + uuid: Uuid, + content: String, + }, + CREATE { + uuid: Uuid, + path: PathBuf, + content: String, + }, +} + +#[derive(Debug, Clone)] +pub struct DeltaIdx { + pub delta_type: Delta, + pub content: String, +} diff --git a/backend/server/src/modules/blog/components/check_update/error.rs b/backend/server/src/modules/blog/components/check_update/error.rs index 0d27c29..f7955b3 100644 --- a/backend/server/src/modules/blog/components/check_update/error.rs +++ b/backend/server/src/modules/blog/components/check_update/error.rs @@ -1,7 +1,7 @@ use thiserror::Error; #[derive(Error, Debug)] -pub enum PostIdxError { +pub enum Error { #[error("missing essential field `{}` for file `{}`", field, filename)] MissingField { field: String, filename: String }, @@ -46,4 +46,7 @@ pub enum PostIdxError { id: String, msg: String, }, + + #[error("git error")] + Git2(#[from] git2::Error), } diff --git a/backend/server/src/modules/blog/components/check_update/mod.rs b/backend/server/src/modules/blog/components/check_update/mod.rs index 6e9f5dd..474434f 100644 --- a/backend/server/src/modules/blog/components/check_update/mod.rs +++ b/backend/server/src/modules/blog/components/check_update/mod.rs @@ -6,6 +6,7 @@ use std::{ path::{Path, PathBuf}, }; +use crate::modules::blog::components::check_update::pull::{analyze_deltas, fetch_deltas}; use crate::modules::blog::components::check_update::{ // index::drop_index, // utils::{expand_path, index_files, search_md}, @@ -13,14 +14,18 @@ use crate::modules::blog::components::check_update::{ }; use crate::scheduler::task_func::AsyncTaskFunc; use sqlx::PgPool; -use tracing::{info, warn, Level}; +use tracing::{info, trace, warn, Level}; // mod error; // mod index; +mod changes; +mod error; +mod pull; mod utils; #[derive(Debug)] struct Args { + db_connection: PgPool, git_work_dir: PathBuf, git_url: String, git_remote_name: String, @@ -45,7 +50,6 @@ pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc { }; // clone & check update - let git_url = match env::var(var_name::BLOG_GIT_URL) { Ok(url) => url, Err(e) => { @@ -63,16 +67,33 @@ pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc { .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); let arc_args = Arc::new(Args { + db_connection, git_work_dir, git_url, git_remote_name, git_remote_branch, }); - move || Box::pin(test(arc_args.clone())) + move || Box::pin(workflow(arc_args.clone())) } -async fn test(args: Arc) { - info!("test"); - dbg!(&args); +async fn workflow(args: Arc) { + let deltas = match fetch_deltas( + &args.git_url, + &args.git_remote_name, + &args.git_remote_branch, + &args.git_work_dir, + ) + .await + { + Ok(updates) => updates, + Err(_) => return, + }; + + if deltas.is_empty() { + trace!("blog git already up-to-date"); + return; + } + + let updates = analyze_deltas(deltas, &args.git_work_dir).await; } diff --git a/backend/server/src/modules/blog/components/check_update/pull.rs b/backend/server/src/modules/blog/components/check_update/pull.rs new file mode 100644 index 0000000..38a471a --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/pull.rs @@ -0,0 +1,147 @@ +use crate::modules::blog::components::check_update::changes::{Change, DeltaIdx}; +use crate::modules::blog::components::check_update::error::Error; +use crate::utils::git; +use crate::utils::git::FileDelta; +use git2::{Delta, Repository}; +use markdown::MdFile; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use tracing::error; +use uuid::Uuid; + +pub async fn fetch_deltas( + git_url: &str, + git_remote_name: &str, + git_branch: &str, + git_work_dir: &PathBuf, +) -> Result, Error> { + let repo = match Repository::open(git_work_dir) { + Ok(repo) => repo, + Err(_) => match Repository::clone(git_url, git_work_dir) { + Ok(repo) => repo, + Err(e) => { + error!("failed to clone blog content git({}): {}", &git_url, e); + return Err(e.into()); + } + }, + }; + + let updates = match git::sync(git_remote_name, git_branch, &repo) { + Ok(updates) => updates, + Err(e) => { + error!( + "failed to sync blog content git({}) updates: {}", + &git_url, e + ); + return Err(e.into()); + } + }; + + Ok(updates) +} + +pub async fn analyze_deltas(file_deltas: Vec, git_work_dir: &Path) -> Vec { + dbg!(&file_deltas); + let mut changes: Vec = vec![]; + let mut merge_deltas: Vec = vec![]; + + for delta in file_deltas.into_iter() { + if delta.status == Delta::Modified { + let file_path = &git_work_dir.join( + delta + .new_path + .expect("cannot parse modified post `new_path`"), + ); + let post = match MdFile::from_file(file_path) { + Ok(post) => post, + Err(e) => { + error!("md file parsing error: {}", e); + continue; + } + }; + + let id = match extract_post_id(&post) { + Some(id) => id, + None => { + // todo: add to unresolved error list + error!( + "failed to parse uuid on post {}", + file_path.to_string_lossy() + ); + continue; + } + }; + + changes.push(Change::UPDATE { + uuid: id, + content: post.content, + }) + } else { + merge_deltas.push(delta) + }; + } + + // merge some addition and remove deltas to rename/move changes + let delta_idx = { + let mut delta_idx: HashMap> = HashMap::new(); + for file_delta in merge_deltas.into_iter() { + let file_path = &git_work_dir.join( + file_delta + .new_path + .expect("cannot parse modified post `new_path`"), + ); + match file_delta.status { + // todo: split impl + Delta::Added | Delta::Deleted => { + let file_path_str = file_path.to_string_lossy(); + + let post = match MdFile::from_file(file_path) { + Ok(post) => post, + Err(e) => { + error!("md file parsing error ({}): {}", file_path_str, e); + continue; + } + }; + + let id = match extract_post_id(&post) { + Some(id) => id, + None => { + // todo: add to unresolved error list + error!("failed to parse uuid on post {}", file_path_str); + continue; + } + }; + + let file_delta_idx = DeltaIdx { + delta_type: file_delta.status, + content: post.content, + }; + delta_idx + .entry(id) + // todo: change to shallow copy + .and_modify(|v| v.push(file_delta_idx.clone())) + .or_insert(vec![file_delta_idx]); + } + _ => { + error!("unsupported delta on file {}", file_path.to_string_lossy()); + continue; + } + }; + } + delta_idx + }; + + dbg!(&delta_idx); + + changes +} + +fn extract_post_id(post: &MdFile) -> Option { + match post.meta["uuid"].as_str() { + Some(id) => match Uuid::parse_str(id) { + Ok(id) => Some(id), + Err(_) => None, + }, + None => None, + } +} diff --git a/backend/server/src/modules/blog/components/check_update/utils.rs b/backend/server/src/modules/blog/components/check_update/utils.rs index 9f00fe5..5b67a19 100644 --- a/backend/server/src/modules/blog/components/check_update/utils.rs +++ b/backend/server/src/modules/blog/components/check_update/utils.rs @@ -8,48 +8,6 @@ use chrono::{DateTime, FixedOffset, ParseResult}; use dirs::home_dir; use tracing::warn; -// pub fn search_md(entry: PathBuf) -> Vec { -// if entry.ends_with(".git") { -// return vec![]; -// } -// if entry.is_dir() { -// if let Ok(files) = entry.read_dir() { -// files -// .flat_map(|file| { -// let file_path = file.unwrap().path(); -// search_md(file_path) -// }) -// .collect() -// } else { -// vec![] -// } -// } else if entry.extension().is_some_and(|ext| ext == "md") { -// vec![entry] -// } else { -// vec![] -// } -// } - -// pub fn index_files(files: &[PathBuf], db_conn: &Pool) { -// let rt = Runtime::new().unwrap(); -// rt.block_on(async { -// let files_clone = files.to_owned(); -// let handlers = files_clone -// .into_iter() -// .map(|file: PathBuf| { -// let conn_clone = db_conn.clone(); -// spawn(async move { -// if let Err(e) = index_file(&file, &conn_clone).await { -// error!("failed to index markdown file `{}`: {}", file.display(), e); -// } -// }) -// }) -// .collect::>(); -// -// block_on(join_all(handlers)); -// }); -// } - pub fn expand_path(path: String) -> String { if path.starts_with('~') { home_dir() @@ -75,7 +33,3 @@ pub fn expand_path(path: String) -> String { pub fn parse_date(date_str: &str) -> ParseResult> { DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) } - -// pub fn get_md_file_basename(f_name: &str) -> String { -// f_name.replace(".md", "") -// } diff --git a/backend/server/src/modules/blog/components/consistency_guard/mod.rs b/backend/server/src/modules/blog/components/consistency_guard/mod.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/server/src/modules/blog/components/consistency_guard/mod.rs @@ -0,0 +1 @@ + diff --git a/backend/server/src/modules/blog/components/mod.rs b/backend/server/src/modules/blog/components/mod.rs index 2683687..0f3bc78 100644 --- a/backend/server/src/modules/blog/components/mod.rs +++ b/backend/server/src/modules/blog/components/mod.rs @@ -1,2 +1,3 @@ pub mod check_update; +mod consistency_guard; mod rag; diff --git a/backend/server/src/modules/blog/components/rag/mod.rs b/backend/server/src/modules/blog/components/rag/mod.rs index e69de29..8b13789 100644 --- a/backend/server/src/modules/blog/components/rag/mod.rs +++ b/backend/server/src/modules/blog/components/rag/mod.rs @@ -0,0 +1 @@ + diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index b870da8..28cb2b9 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -1,6 +1,8 @@ mod api; mod components; +use crate::modules::blog::components::check_update; +use crate::scheduler::tasks::async_basic::AsyncBasic; use crate::{ modules::{ blog::{ @@ -16,9 +18,6 @@ use crate::{ scheduler::tasks::CronTask, }; -use crate::modules::blog::components::check_update; -use crate::scheduler::tasks::async_basic::AsyncBasic; -use crate::scheduler::tasks::basic::BasicTask; use axum::{routing::get, Router}; use sqlx::PgPool; @@ -46,19 +45,11 @@ impl Module for Blog { fn get_cron_tasks(&self) -> Vec<(&str, Box)> { let db_handler = self.db_handler.clone(); - vec![ - ( - "Blog Check Update", - AsyncBasic::new(check_update::task(db_handler.clone()), "* * * * * *") - .unwrap() - .to_task(), - ), - ( - "Sync test task", - BasicTask::new(|| println!("test lol"), "* * * * * *") - .unwrap() - .to_task(), - ), - ] + vec![( + "Blog Check Update", + AsyncBasic::new(check_update::task(db_handler.clone()), "*/5 * * * * *") + .unwrap() + .to_task(), + )] } } diff --git a/backend/server/src/modules/blog/schema.sql b/backend/server/src/modules/blog/schema.sql index d6cd75d..7ed752f 100644 --- a/backend/server/src/modules/blog/schema.sql +++ b/backend/server/src/modules/blog/schema.sql @@ -2,20 +2,20 @@ CREATE TABLE tags ( id SERIAL PRIMARY KEY, name VARCHAR(20) UNIQUE NOT NULL, - description TEXT + description TEXT DEFAULT '' ); CREATE TABLE categories ( id SERIAL PRIMARY KEY, name VARCHAR(20) UNIQUE NOT NULL, - description TEXT + description TEXT DEFAULT '' ); CREATE TABLE posts ( - id SERIAL PRIMARY KEY, - identifier VARCHAR(50) UNIQUE NOT NULL, + id UUID UNIQUE NOT NULL, + identifier VARCHAR(100) UNIQUE NOT NULL, title VARCHAR(255) NOT NULL, subtitle VARCHAR(255) NOT NULL, published_date DATE NOT NULL, @@ -27,15 +27,15 @@ CREATE TABLE posts CREATE TABLE post_tag ( - post SERIAL REFERENCES posts (id), + post UUID REFERENCES posts (id), tag SERIAL REFERENCES tags (id) ); CREATE TABLE metadata ( - pid SERIAL REFERENCES posts (id) NOT NULL, - meta_key VARCHAR(50) NOT NULL, - meta_value TEXT NOT NULL, + pid UUID REFERENCES posts (id) NOT NULL, + meta_key VARCHAR(50) NOT NULL, + meta_value TEXT NOT NULL, CONSTRAINT meta_pk PRIMARY KEY (pid, meta_key) ); diff --git a/backend/server/src/modules/mod.rs b/backend/server/src/modules/mod.rs index 0838aa5..72e3149 100644 --- a/backend/server/src/modules/mod.rs +++ b/backend/server/src/modules/mod.rs @@ -2,7 +2,7 @@ use std::{env, num::ParseIntError}; use crate::{ constants::config::server::{default_value, var_name}, - scheduler::{tasks::CronTask, Scheduler}, + scheduler::{Scheduler, tasks::CronTask}, }; use macros::error_panic; diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index 0f8e8fd..f79c277 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -20,8 +20,6 @@ pub struct Scheduler { impl Scheduler { /// Start all tasks in the scheduler pub async fn start(&self) { - dbg!("entering"); - let fut = self .task_pool .clone() diff --git a/backend/server/src/scheduler/utils.rs b/backend/server/src/scheduler/utils.rs index e3ce91b..996e62a 100644 --- a/backend/server/src/scheduler/utils.rs +++ b/backend/server/src/scheduler/utils.rs @@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; use crate::scheduler::{error::SchedulerError, tasks::CronTask}; use chrono::{DateTime, Utc}; -use tokio::time::{sleep, Sleep}; +use tokio::time::{Sleep, sleep}; pub fn duration_from_now(future: DateTime) -> Duration { future diff --git a/backend/server/src/utils/git.rs b/backend/server/src/utils/git.rs index 94b00cc..93f2a13 100644 --- a/backend/server/src/utils/git.rs +++ b/backend/server/src/utils/git.rs @@ -3,12 +3,11 @@ /// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 /// use git2::{ - build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, + AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, build, }; use std::path::PathBuf; use tracing::{error, warn}; - #[derive(Debug, Eq, PartialEq)] pub struct FileDelta { pub old_path: Option, @@ -40,7 +39,6 @@ fn fast_forward( None => String::from_utf8_lossy(lb.name_bytes()).to_string(), }; let msg = format!("fast-forward({}) -> {}", name, rc.id()); - // println!("{}", msg); lb.set_target(rc.id(), &msg)?; repo.set_head(&name)?; repo.checkout_head(Some(build::CheckoutBuilder::default().force()))?;