diff --git a/.gitignore b/.gitignore index 3694827..67b7a43 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ # exclude post repo /blog_posts_git + +# rustc ICE dump +rustc-ice-*.txt diff --git a/Makefile b/Makefile index 0c4e0d9..581ed84 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,7 @@ ifeq ($(ARCH), arm64) ARCH := aarch64 endif -all: release - -release: - docker compose -f prod.compose.yaml build --build-arg ARCH=$(ARCH) - -dev: - docker-compose -f dev.compose.yaml up --build - +.PHONY: up clean: - docker-compose -f prod.compose.yaml -f dev.compose.yaml rm -fsv + echo "todo" diff --git a/README.md b/README.md index c8ea3a4..dd27986 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,10 @@ -# `Nocturne: personal website & CMS` +# `Nocturne: Personal Website` -This is the code repo of my personal website, and [here](https://github.com/Lumither/blog-posts/) is the link of the -content repo. +This is the code repo of my [personal website](https://lumither.com), +and [here](https://github.com/Lumither/blog-posts/) is the link to the +content (blog posts etc.) repo. Powered by Next.js and Rust, built with passion and love -

- GitHub Avatar -

- -*will add more here in the future - -## Nav: - - - -## Todos: - -### Backend - -- [ ] Add: log system -- [ ] Add: search api -- [ ] Add: tag filter (api) -- [ ] Abandon: UUID mechanism -- [ ] Fix: db connection issue - -### Frontend - -- [ ] Page: Home -- [ ] Page: About -- [ ] Page: Tags -- [ ] Page: Archive -- [ ] Page: Search -- [ ] Add: Right sidebar - +> [!NOTE] +> This is an unstable branch, checkout `release` for the (guaranteed) working version \ No newline at end of file diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6c75b0a..3dfd506 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,8 +1,7 @@ [workspace] -resolver = "2" +resolver = "3" members = [ - "api_server", - "constants", + "server", "macros", "markdown", ] diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml deleted file mode 100644 index af17326..0000000 --- a/backend/api_server/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "api_server" -version = "0.1.0" -edition = "2021" - -[dependencies] -macros = { path = "../macros" } -constants = { path = "../constants" } -markdown = { path = "../markdown" } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } -tracing-appender = "0.2.3" -dotenv = "0.15.0" -dirs = "5.0.1" -tokio = { version = "1.40.0", features = ["full"] } -sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -axum = { version = "0.7.5", features = ["macros"] } -serde = { version = "1.0.199", features = ["derive"] } -serde_json = "1.0.128" -uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } -futures = { version = "0.3.30", features = ["executor"] } -cron = "0.13.0" -git2 = "0.19.0" -thiserror = "2.0.3" diff --git a/backend/api_server/src/blog/api/get_page_count.rs b/backend/api_server/src/blog/api/get_page_count.rs deleted file mode 100644 index db540af..0000000 --- a/backend/api_server/src/blog/api/get_page_count.rs +++ /dev/null @@ -1,26 +0,0 @@ -use constants::blog::PAGE_SIZE; - -use axum::extract::State; -use axum::Json; -use serde_json::{json, Value}; -use sqlx::{query, PgPool, Row}; -use tracing::error; - -pub async fn get_page_count( - State(db_connection): State, -) -> Result, Json> { - match query("select cast(ceil(count(*) / $1) as integer) as res from post") - .bind(PAGE_SIZE as f32) - .fetch_one(&db_connection) - .await - { - Ok(res) => { - let res: i32 = res.get("res"); - Ok(Json::from(json!({"res": res}))) - } - Err(e) => { - error!("Failed to get blog page count: {}", e.to_string()); - Err(Json::from(json!({"error": e.to_string()}))) - } - } -} diff --git a/backend/api_server/src/blog/api/get_post.rs b/backend/api_server/src/blog/api/get_post.rs deleted file mode 100644 index 2374dec..0000000 --- a/backend/api_server/src/blog/api/get_post.rs +++ /dev/null @@ -1,153 +0,0 @@ -use axum::{ - extract::{Path, State}, - http::StatusCode, - Json, -}; -use serde_json::{json, Value}; -use sqlx::{query, Error, PgPool, Row}; -use std::str::FromStr; -use tracing::error; -use uuid::Uuid; - -pub async fn get_post( - State(db_connection): State, - Path(post_id): Path, -) -> Result, (StatusCode, Json)> { - let post_id = match Uuid::from_str(post_id.as_str()) { - Ok(id) => id, - Err(e) => { - error!( - "Failed to parse UUID <{}>: {}", - post_id.as_str(), - e.to_string() - ); - return Err(( - StatusCode::BAD_REQUEST, - Json::from(json!(("error", e.to_string()))), - )); - } - }; - - let post = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post -FROM post t -WHERE post_id = $1; -"#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("post"), - Err(e) => { - error!( - "Database error on Reading `base post information` from table `post`: <{}>: {}", - post_id.to_string(), - e.to_string() - ); - return match e { - Error::RowNotFound => Err(( - StatusCode::BAD_REQUEST, - Json::from(json!({"error": e.to_string()})), - )), - _ => Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )), - }; - } - }; - - let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;") - .bind(post_id) - .fetch_all(&db_connection) - .await - { - Ok(tags) => tags.iter().map(|tag| tag.get::("tag")).collect(), - Err(e) => { - error!("Database query error :{}", e.to_string()); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let meta = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta -FROM meta t -WHERE post_id = $1; - "#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("meta"), - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "post meta data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let adjacent = match query(r" -WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1), - prev AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date < (SELECT date FROM target_post) - ORDER BY t.date DESC - LIMIT 1), - next AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date > (SELECT date FROM target_post) - ORDER BY t.date - LIMIT 1) -SELECT json_strip_nulls(json_build_object( - 'prev', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM prev), - 'next', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM next) - )) AS result; -" - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(res) => {res.get::("result")} - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "adjoint posts data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - - }; - - let mut res = post.as_object().unwrap().to_owned(); - res.insert("tags".to_string(), tags.into()); - res.insert("meta".to_string(), meta); - res.insert("adj".to_string(), adjacent); - - Ok(Json::from(json!(res))) -} diff --git a/backend/api_server/src/blog/api/mod.rs b/backend/api_server/src/blog/api/mod.rs deleted file mode 100644 index 5e23c3d..0000000 --- a/backend/api_server/src/blog/api/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod get_page_count; -pub mod get_post; -pub mod get_post_list; diff --git a/backend/api_server/src/blog/cron/check_update/error.rs b/backend/api_server/src/blog/cron/check_update/error.rs deleted file mode 100644 index 0d27c29..0000000 --- a/backend/api_server/src/blog/cron/check_update/error.rs +++ /dev/null @@ -1,49 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum PostIdxError { - #[error("missing essential field `{}` for file `{}`", field, filename)] - MissingField { field: String, filename: String }, - - #[error( - "invalid field `{}` for file `{}`, <{}>: {}", - field_name, - filename, - err_field, - msg - )] - InvalidField { - field_name: String, - err_field: String, - filename: String, - msg: String, - }, - - #[error( - "database error on writing `{}` to table `{}`: <{}>: {}", - data_desc, - db_table, - id, - msg - )] - DBWriteFailure { - data_desc: String, - db_table: String, - id: String, - msg: String, - }, - - #[error( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - data_desc, - db_table, - id, - msg - )] - DBReadFailure { - data_desc: String, - db_table: String, - id: String, - msg: String, - }, -} diff --git a/backend/api_server/src/blog/cron/check_update/index.rs b/backend/api_server/src/blog/cron/check_update/index.rs deleted file mode 100644 index f391639..0000000 --- a/backend/api_server/src/blog/cron/check_update/index.rs +++ /dev/null @@ -1,351 +0,0 @@ -use std::error::Error; -use std::path::Path; - -use crate::blog::cron::check_update::utils::get_md_file_basename; -use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date}; -use markdown::MdFile; - -use sqlx::{query, PgPool, Pool, Postgres, Row}; -use tokio::runtime::Runtime; -use tracing::{info, warn}; - -async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - let post_id = post.file_id; - let title = match post.meta.get("title") { - None => { - return Err(PostIdxError::MissingField { - field: "title".to_string(), - filename: post.filename.clone(), - }) - } - Some(title) => title.as_str().unwrap_or_else(|| { - warn!("invalid title for `{}`", &post.filename); - "invalid title" - }), - }; - let sub_title = post - .meta - .get("sub_title") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let summary = post - .meta - .get("summary") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let content = post.content.as_str(); - let category = match post.meta.get("category") { - None => { - return Err(PostIdxError::MissingField { - field: "category".to_string(), - filename: post.filename.clone(), - }) - } - Some(title) => title.as_str().unwrap_or_else(|| { - warn!("invalid category for `{}`", &post.filename); - "invalid category" - }), - }; - let last_update = { - if let Some(last_update) = post.meta.get("last_update") { - match parse_date(last_update.as_str().unwrap()) { - Ok(date) => Some(date), - Err(e) => { - return Err(PostIdxError::InvalidField { - field_name: "last_update".to_string(), - err_field: last_update.to_string(), - filename: post.filename.clone(), - msg: e.to_string(), - }) - } - } - } else { - None - } - }; - let first_update = match post.meta.get("date") { - None => { - return Err(PostIdxError::MissingField { - field: "date".to_string(), - filename: post.filename.clone(), - }) - } - Some(first_update) => match parse_date(first_update.as_str().unwrap()) { - Ok(date) => date, - Err(e) => { - return Err(PostIdxError::InvalidField { - field_name: "date".to_string(), - err_field: first_update.to_string(), - filename: post.filename.clone(), - msg: e.to_string(), - }) - } - }, - }; - let filename = get_md_file_basename(&post.filename); - - if let Err(e) = query( - r##" -INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) -ON CONFLICT (post_id) DO -UPDATE SET - title = EXCLUDED.title, - summary = EXCLUDED.summary, - content = EXCLUDED.content, - last_update = EXCLUDED.last_update, - first_update = EXCLUDED.first_update, - sub_title = EXCLUDED.sub_title, - category = EXCLUDED.category, - filename = EXCLUDED.filename; - "##, - ) - .bind(post_id) - .bind(title) - .bind(summary) - .bind(content) - .bind(last_update) - .bind(first_update) - .bind(sub_title) - .bind(category) - .bind(filename) - .execute(db_connection) - .await - { - return Err( - PostIdxError::DBWriteFailure { - data_desc: "post base information".to_string(), - db_table: "Post".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - } - ); - } - Ok(()) -} - -async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - if let Some(tags_string) = post.meta.get("tags").to_owned() { - let tags = match tags_string.as_array().map(|vec| { - vec.iter() - .map(|val| val.as_str().unwrap()) - .collect::>() - }) { - None => { - return Err(PostIdxError::MissingField { - field: "tags".to_string(), - filename: post.filename.clone(), - }) - } - Some(val) => val, - }; - - let post_id = post.file_id; - - // remove old record - if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") - .bind(post_id) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - }); - } - - // add new record - for tag in tags { - match query("INSERT INTO Tag VALUES ($1, $2);") - .bind(post_id) - .bind(tag) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - }) - } - } - } - } - - Ok(()) -} - -async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { - let base64_form = post.hash.clone(); - match query("SELECT hash FROM Hash WHERE post_id = $1") - .bind(post.file_id) - .fetch_optional(db_connection) - .await - { - Ok(row) => { - let db_record: String = match row { - None => "".to_string(), - Some(row) => row.try_get("hash").unwrap_or("".to_string()), - }; - if base64_form == db_record { - Ok(false) - } else { - if let Err(e) = query( - r#" -INSERT INTO Hash (post_id, hash) VALUES ($1, $2) - ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, - ) - .bind(post.file_id) - .bind(&base64_form) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }); - } - Ok(true) - } - } - Err(e) => Err(PostIdxError::DBReadFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }), - } -} - -async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - if let Err(e) = query( - // todo: sanitization - format!( - r#" -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY['tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY(_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - "#, post.file_id.to_string().as_str(), post.meta - ).as_str() - ) - .execute(db_connection) - .await { - Err(PostIdxError::DBWriteFailure { - data_desc: "post meta data".to_string(), - db_table: "Meta".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }) - } else { - Ok(()) - } -} - -pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { - let md_file = MdFile::from_file(path)?; - - async fn trail(db_conn: &Pool, md_file: &MdFile) -> Result<(), PostIdxError> { - base(db_conn, md_file).await?; - tag(db_conn, md_file).await?; - meta(db_conn, md_file).await?; - hash(db_conn, md_file).await.map(|_| ()) - } - let mut cnt = 5; - - while let Err(e) = trail(db_conn, &md_file).await { - match e { - PostIdxError::DBWriteFailure { .. } => { - cnt -= 1; - } - PostIdxError::DBReadFailure { .. } => { - cnt -= 1; - } - _ => return Err(e.into()), - } - if cnt <= 0 { - return Err(e.into()); - } - } - - info!("indexed markdown file: {}", path.display()); - Ok(()) -} - -pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - if let Err(e) = query( - r#" -WITH deleted_post AS ( - DELETE FROM post - WHERE id = (SELECT id FROM post WHERE filename = $1) - RETURNING id -) -DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); -DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); -DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); - "#, - ) - .bind(file_basename) - .execute(db_conn) - .await - { - Err(Box::from( - format!("failed to drop index: {}", e).to_string(), - )) - } else { - Ok(()) - } - }) -} diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs deleted file mode 100644 index 9568ae2..0000000 --- a/backend/api_server/src/blog/cron/check_update/mod.rs +++ /dev/null @@ -1,146 +0,0 @@ -use std::env; -use std::fs::remove_dir_all; -use std::path::{Path, PathBuf}; - -use crate::{ - blog::cron::check_update::{ - index::drop_index, - utils::{expand_path, get_md_file_basename, index_files, search_md}, - }, - utils::git, -}; -use constants::config::general::{default_value, var_name}; -use macros::panic_with_log; - -use git2::{Delta, Repository}; -use sqlx::{Pool, Postgres}; -use tracing::{error, info, warn, Level}; - -mod error; -mod index; -mod utils; - -pub fn task(db_connection: &Pool) { - // init - let git_work_dir = { - let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { - expand_path(default_log_path) - } else { - let fallback_path = expand_path(default_value::WORK_DIR.to_string()); - warn!( - "unset env var `{}`, using default path: {}/blog_git", - var_name::WORK_DIR, - fallback_path - ); - fallback_path - }; - Path::new(&path).join("blog_git") - }; - - // clone & check update - - let git_url = match env::var(var_name::BLOG_GIT_URL) { - Ok(url) => url, - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to read `{}`: {}", - var_name::BLOG_GIT_URL, - e - ) - } - }; - let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) - .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); - let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) - .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); - - match Repository::open(&git_work_dir) { - Ok(repo) => { - // prevent git remote url mismatch (clean cache) - match repo.find_remote(&git_remote_name) { - Ok(remote) => { - if remote.url().is_some_and(|url| url != git_url) { - remove_dir_all(&git_work_dir).unwrap(); - warn!("blog content git({}) url mismatch, reinitialized", &git_url); - return; - } - } - Err(e) => { - remove_dir_all(&git_work_dir).unwrap(); - error!( - "unexpected error on indexing blog content git({}), reinitialized: {}", - &git_url, e - ); - } - } - - match git::sync(&git_remote_name, &git_remote_branch, &repo) { - Ok(updates) => { - if !updates.is_empty() { - // incremental update - info!( - "blog content git({}) update detected: {:?}", - &git_url, updates - ); - - let mut update_md: Vec = vec![]; - - for update in updates { - if [Delta::Deleted, Delta::Untracked].contains(&update.status) { - if let Some(path) = &update.old_path { - if drop_index( - &get_md_file_basename( - path.file_name().unwrap().to_str().unwrap(), - ), - db_connection, - ) - .is_ok() - { - info!( - "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", - &git_url, - &path.display() - ); - } else { - error!( - "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", - &git_url, - &path.display() - ) - } - } - } else if let Some(path) = &update.new_path { - if path.components().any(|comp| comp.as_os_str() == "posts") - && path.extension().is_some_and(|ext| ext == "md") - { - update_md.push(path.to_path_buf()); - } - } - } - - index_files(&update_md, db_connection); - } - } - Err(e) => { - error!( - "failed to sync blog content git({}) updates: {}", - &git_url, e - ); - } - } - } - Err(_) => { - match Repository::clone(&git_url, &git_work_dir) { - Ok(_repo) => { - // full repo index - let files = search_md(git_work_dir.join("posts")); - index_files(&files, db_connection); - } - Err(e) => { - error!("failed to clone blog content git({}): {}", &git_url, e); - } - } - } - }; -} diff --git a/backend/api_server/src/blog/cron/check_update/utils.rs b/backend/api_server/src/blog/cron/check_update/utils.rs deleted file mode 100644 index d0da864..0000000 --- a/backend/api_server/src/blog/cron/check_update/utils.rs +++ /dev/null @@ -1,86 +0,0 @@ -use dirs::home_dir; -use std::path::PathBuf; - -use crate::blog::cron::check_update::index::index_file; -use chrono::{DateTime, FixedOffset, ParseResult}; -use constants::config::general::{default_value, var_name}; -use constants::time::DEFAULT_DATE_FORMAT; - -use futures::executor::block_on; -use futures::future::join_all; -use sqlx::{Pool, Postgres}; -use tokio::runtime::Runtime; -use tokio::spawn; -use tracing::{error, warn}; - -pub fn search_md(entry: PathBuf) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry] - } else { - vec![] - } -} - -pub fn index_files(files: &[PathBuf], db_conn: &Pool) { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let files_clone = files.to_owned(); - let handlers = files_clone - .into_iter() - .map(|file: PathBuf| { - let conn_clone = db_conn.clone(); - spawn(async move { - if let Err(e) = index_file(&file, &conn_clone).await { - error!("failed to index markdown file `{}`: {}", file.display(), e); - } - }) - }) - .collect::>(); - - block_on(join_all(handlers)); - }); -} - -pub fn expand_path(path: String) -> String { - if path.starts_with('~') { - home_dir() - .map(|dir| { - dir.join(path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .unwrap() - }) - .unwrap_or_else(|| { - warn!( - "failed to parse env var `{}`, using default value `{}`", - var_name::WORK_DIR, - default_value::WORK_DIR - ); - default_value::WORK_DIR.to_string() - }) - } else { - path - } -} - -pub fn parse_date(date_str: &str) -> ParseResult> { - DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) -} - -pub fn get_md_file_basename(f_name: &str) -> String { - f_name.replace(".md", "") -} diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs deleted file mode 100644 index 686fb26..0000000 --- a/backend/api_server/src/blog/cron/mod.rs +++ /dev/null @@ -1,14 +0,0 @@ -use crate::scheduler::tasks::basic::BasicTask; -use crate::scheduler::tasks::Task; -use sqlx::{Pool, Postgres}; - -pub mod check_update; - -pub fn get_tasks(db_connection: Pool) -> Vec> { - let db_conn_clone = db_connection.clone(); - vec![ - BasicTask::new(move || check_update::task(&db_conn_clone), "0/15 * * * * *") - .unwrap() - .to_task(), - ] -} diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs deleted file mode 100644 index 6d14b15..0000000 --- a/backend/api_server/src/blog/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod api; -pub mod cron; - -use crate::blog::api::{ - get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, -}; - -use axum::routing::get; -use axum::Router; -use sqlx::{Pool, Postgres}; - -pub fn get_router() -> Router> { - Router::new() - .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count)) - .route("/get_post/:post_id", get(get_post)) -} diff --git a/backend/api_server/src/init/cron.rs b/backend/api_server/src/init/cron.rs deleted file mode 100644 index 429537a..0000000 --- a/backend/api_server/src/init/cron.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::blog; -use crate::scheduler::Scheduler; -use sqlx::{Pool, Postgres}; - -pub fn start(db_pool: Pool) { - let scheduler = Scheduler::new(); - let _ = vec![blog::cron::get_tasks(db_pool.clone())] - .into_iter() - .map(|list| scheduler.insert_list(list)) - .collect::>(); - scheduler.start().unwrap() -} diff --git a/backend/api_server/src/init/schema.rs b/backend/api_server/src/init/schema.rs deleted file mode 100644 index ef9def0..0000000 --- a/backend/api_server/src/init/schema.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sqlx::{query, Pool, Postgres}; -use tracing::info; - -const POST_BASE_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - filename TEXT UNIQUE NOT NULL, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ, - first_update TIMESTAMPTZ NOT NULL -); -"##; - -// base model, will add more cols at runtime -const POST_META_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); -"##; - -const POST_HASH_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); -"##; - -const TAG_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); -"##; - -pub async fn init(db_connection: &Pool) { - query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); - query(POST_META_MODEL).execute(db_connection).await.unwrap(); - query(TAG_MODEL).execute(db_connection).await.unwrap(); - query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); - info!("database schema initialized"); -} diff --git a/backend/api_server/src/init/server.rs b/backend/api_server/src/init/server.rs deleted file mode 100644 index 703a043..0000000 --- a/backend/api_server/src/init/server.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::env; -use std::error::Error; -use std::num::ParseIntError; -use std::process::exit; - -use crate::blog; -use axum::Router; -use constants::config::server::{default_value, var_name}; -use macros::panic_with_log; -use sqlx::{Pool, Postgres}; -use tracing::{error, info, warn, Level}; - -pub async fn start(db_pool: Pool) -> Result<(), Box> { - let app = Router::new() - .nest("/blog", blog::get_router()) - .with_state(db_pool); - - let port: u32 = match env::var(var_name::PORT) { - Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { - warn!( - "failed to parse `{}`, using default port: {}", - var_name::PORT, - e.to_string() - ); - default_value::PORT - }), - Err(_) => default_value::PORT, - }; - - let listener = - match tokio::net::TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await { - Ok(listener) => { - info!("server started on {}:{}", default_value::HOST, port); - listener - } - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to start server on {}:{}: {}", - default_value::HOST, - port, - e.to_string() - ); - } - }; - - axum::serve(listener, app).await.unwrap_or_else(|e| { - error!("failed to start axum: {}", e.to_string()); - exit(1); - }); - Ok(()) -} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs deleted file mode 100644 index 8aab2c7..0000000 --- a/backend/api_server/src/main.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod blog; -mod init; -mod scheduler; -mod utils; - -#[tokio::main] -async fn main() { - init::load_env(); - - let _guards = init::logger::init(); - - init::start().await; -} diff --git a/backend/api_server/src/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs deleted file mode 100644 index ce86797..0000000 --- a/backend/api_server/src/scheduler/mod.rs +++ /dev/null @@ -1,194 +0,0 @@ -use crate::{ - scheduler::scheduler_error::SchedulerError, - scheduler::scheduler_error::SchedulerError::IdAlreadyRunning, - scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper, - scheduler::tasks::Task, -}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::RwLock; -use std::{collections::HashMap, sync::Arc, thread}; -use tracing::log::error; -use uuid::Uuid; - -pub mod scheduler_error; -mod sleeper; -mod task_func; -pub mod tasks; - -struct TaskStatus { - sleeper: Option>, - is_alive: AtomicBool, -} - -#[derive(Default)] -pub struct Scheduler { - tasks: Arc>>>>, - running: Arc>>>, -} - -impl Scheduler { - /// Start all tasks in the scheduler - pub fn start(&self) -> Result<(), SchedulerError> { - for (k, _v) in self.tasks.read()?.iter() { - self.run_non_blocking(k)? - } - Ok(()) - } - - /// Terminate all tasks in the scheduler - pub fn halt() {} - - fn run_task( - running: Arc>>>, - id: Uuid, - task: Arc>, - ) -> Result<(), SchedulerError> { - if running.read()?.get(&id).is_some() { - return Err(IdAlreadyRunning); - } else { - running.write()?.insert( - id, - TaskStatus { - sleeper: None, - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - loop { - if let Some(next_execution) = task.get_next_execution() { - // guarantee is alive - let task_status = { - let running_reading_guard = running.read()?; - running_reading_guard.get(&id).cloned().clone() - }; - if let Some(task_status) = task_status { - if !&task_status.is_alive.load(Ordering::SeqCst) { - { - let mut running_lock_guard = running.write()?; - running_lock_guard.remove(&id); - } - return Ok(()); - } else { - let sleeper = Arc::from(SkippableSleeper::from(next_execution)); - { - let mut running_lock_guard = running.write()?; - running_lock_guard.insert( - id, - TaskStatus { - sleeper: Some(sleeper.clone()), - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - sleeper.start(); - task.call(); - } - } - } else { - error!("unexpected error "); - break; - } - } - Ok(()) - } - - /// Run a task with `id` - pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - Self::run_task(self.running.clone(), id, task.clone()) - } else { - Err(IdNotFound) - } - } - - pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - let running_clone = self.running.clone(); - let task_clone = task.clone(); - thread::spawn(move || Self::run_task(running_clone, id, task_clone)); - Ok(()) - } else { - Err(IdNotFound) - } - } - - fn get_running(&self, id: &Uuid) -> Option> { - let ret = self.running.read().unwrap().get(id).cloned(); - ret - } - - /// Stop a task with `id` - pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - status.is_alive.store(false, Ordering::SeqCst); - if let Some(s) = &status.sleeper { - s.skip(); - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } - } - } - - pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - if let Some(s) = &status.sleeper { - s.skip() - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } - } - } - - pub fn insert(&self, task: Box) -> Result { - let id = Uuid::new_v4(); - self.tasks.write()?.insert(id, Arc::new(task)); - Ok(id) - } - - pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { - tasks.into_iter().map(|t| self.insert(t)).collect() - } - - pub fn new() -> Self { - Self::default() - } -} - -#[cfg(test)] -mod tests { - use crate::scheduler::tasks::basic::BasicTask; - use crate::scheduler::Scheduler; - use std::error::Error; - use std::thread; - use std::time::Duration; - - #[test] - fn test_run() -> Result<(), Box> { - let scheduler = Scheduler::default(); - let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; - let id = scheduler.insert(Box::new(task))?; - for i in 1..=2 { - println!("Loop {} start", i); - println!("Run {}", i); - scheduler.run_non_blocking(&id)?; - thread::sleep(Duration::from_secs(4)); - scheduler.stop(&id)?; - println!("Stop {}", i); - thread::sleep(Duration::from_secs(2)); - println!("Loop {} finished", i); - } - Ok(()) - } -} diff --git a/backend/api_server/src/scheduler/sleeper.rs b/backend/api_server/src/scheduler/sleeper.rs deleted file mode 100644 index 4155e57..0000000 --- a/backend/api_server/src/scheduler/sleeper.rs +++ /dev/null @@ -1,77 +0,0 @@ -use chrono::{DateTime, Utc}; -use std::sync::{Arc, Condvar, Mutex}; -use std::thread; -use std::time::Duration; - -#[derive(Clone, Debug, Default)] -pub struct SkippableSleeper { - pub duration: Duration, - state: Arc<(Mutex, Condvar)>, -} - -impl SkippableSleeper { - pub fn new(duration: Duration) -> Self { - SkippableSleeper { - duration, - state: Arc::new((Mutex::new(false), Condvar::new())), - } - } - - // Blocks until the duration has passed or skip() is called - pub fn start(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - let state_clone = Arc::clone(&self.state); - let duration_clone = self.duration; - thread::spawn(move || { - thread::sleep(duration_clone); - let (lock, cvar) = &*state_clone; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); - }); - while !*received { - received = cvar.wait(received).unwrap(); - } - } - - // Skip the blocking delay - pub fn skip(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); - } -} - -impl From> for SkippableSleeper { - fn from(value: DateTime) -> Self { - let from_utc_timestamp = value.timestamp() as u64; - let curr_utc_timestamp = Utc::now().timestamp() as u64; - SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) - } -} - -#[cfg(test)] -mod test { - use crate::scheduler::sleeper::SkippableSleeper; - use std::thread; - use std::time::Duration; - - #[test] - fn usage() { - let sleeper = SkippableSleeper::new(Duration::new(10, 0)); - thread::spawn({ - let sleeper = sleeper.clone(); - move || { - println!("Sleeping..."); - sleeper.start(); - println!("Woke up!"); - } - }); - println!("Start Sleeping..."); - thread::sleep(Duration::from_secs(4)); - println!("Skipping sleep..."); - sleeper.skip(); - } -} diff --git a/backend/api_server/src/scheduler/task_func.rs b/backend/api_server/src/scheduler/task_func.rs deleted file mode 100644 index af27ded..0000000 --- a/backend/api_server/src/scheduler/task_func.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub trait TaskFunc: Fn() + Send + Sync + 'static {} - -impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} diff --git a/backend/api_server/src/scheduler/tasks/mod.rs b/backend/api_server/src/scheduler/tasks/mod.rs deleted file mode 100644 index 7e3b87a..0000000 --- a/backend/api_server/src/scheduler/tasks/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -use chrono::{DateTime, Utc}; - -pub mod basic; - -pub trait Task: Send + Sync { - fn get_next_execution(&self) -> Option>; - fn call(&self); -} diff --git a/backend/macros/Cargo.toml b/backend/macros/Cargo.toml index 6291521..2e8a8d5 100644 --- a/backend/macros/Cargo.toml +++ b/backend/macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "macros" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] tracing = "0.1.40" diff --git a/backend/macros/src/dev.rs b/backend/macros/src/dev.rs new file mode 100644 index 0000000..1173d67 --- /dev/null +++ b/backend/macros/src/dev.rs @@ -0,0 +1,6 @@ +#[macro_export] +macro_rules! dev_consume { + ($($var:ident),+) => { + $(let _ = $var;)* + }; +} diff --git a/backend/macros/src/include_str.rs b/backend/macros/src/include_str.rs new file mode 100644 index 0000000..4e4a624 --- /dev/null +++ b/backend/macros/src/include_str.rs @@ -0,0 +1,24 @@ +/// Includes a UTF-8 encoded file as a string +/// +/// The file is located relative to `CARGO_MANIFEST_DIR`. +/// +/// # Example +/// ```rust +/// use macros::include_str_from_crate_root; +/// +/// let str: &str = include_str_from_crate_root!("/query/scheme.sql"); +/// +// todo: impl +// const PATH_PREFIX: &str = "/src"; +// let str: &str = include_str_from_crate_root!("{}/query/scheme.sql", PATH_PREFIX); +/// ``` +#[macro_export] +macro_rules! include_str_from_crate_root { + // ($path:literal $(, $($arg:tt)*)?) => { + // ($path:expr, $($args:tt)*) => { + // include_str!(concat!(env!("CARGO_MANIFEST_DIR"), format_args!($path, $($args)*))) + // }; + ($path:expr) => { + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), $path)) + }; +} diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs index aad31d7..e1ac3e9 100644 --- a/backend/macros/src/lib.rs +++ b/backend/macros/src/lib.rs @@ -1 +1,3 @@ +mod dev; +mod include_str; mod log; diff --git a/backend/macros/src/log.rs b/backend/macros/src/log.rs index f7dd357..17bd26c 100644 --- a/backend/macros/src/log.rs +++ b/backend/macros/src/log.rs @@ -6,7 +6,7 @@ /// use macros::panic_with_log; /// use tracing::Level; /// -/// panic_with_log!(Level::ERROR, "Test panicked: {}", 42); +/// panic_with_log!(Level::ERROR, "test panicked: {}", 42); /// ``` #[macro_export] macro_rules! panic_with_log { @@ -18,6 +18,25 @@ macro_rules! panic_with_log { }}; } +/// log the message with `Level::ERROR` and exit(1) +/// +/// # Example +/// ```rust +/// +/// use macros::error_panic; +/// +/// error_panic!("test panicked: {}", 42); +/// ``` +#[macro_export] +macro_rules! error_panic { + ($msg:literal $(, $($arg:tt)*)?) => {{ + use std::process; + use tracing::{event, Level}; + event!(Level::ERROR, $msg $(, $($arg)*)?); + process::exit(1); + }}; +} + #[cfg(test)] mod tests { #[test] @@ -25,7 +44,7 @@ mod tests { use std::io; use tracing::Level; use tracing_subscriber::prelude::*; - use tracing_subscriber::{fmt, Registry}; + use tracing_subscriber::{Registry, fmt}; let format = tracing_subscriber::fmt::format() .with_level(true) diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index e0ca45e..5ca020d 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "markdown" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } serde = { version = "1.0.209", features = ["derive"] } serde_json = "1.0.128" regex = "1.10.4" -yaml-rust2 = "0.9.0" +yaml-rust2 = "0.10.0" sha2 = "0.10.8" base64 = "0.22.1" thiserror = "2.0.3" diff --git a/backend/markdown/src/error.rs b/backend/markdown/src/error.rs index a93585f..c9ec0e9 100644 --- a/backend/markdown/src/error.rs +++ b/backend/markdown/src/error.rs @@ -2,12 +2,6 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum Error { - #[error("missing front-matter field: id")] - MissingID, - - #[error("invalid file id: {}", msg)] - InvalidID { msg: String }, - #[error("invalid front-matter: {}", msg)] InvalidFrontMatter { msg: String }, diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index 518d162..353e736 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -1,23 +1,20 @@ -mod error; +pub mod error; mod utils; -use std::fs; -use std::path::Path; -use std::str::FromStr; +use std::{fs, path::Path, str::FromStr}; -use crate::error::Error; -use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; +use crate::{ + error::Error, + utils::front_matter::{parse_front_matter, split_md_front_matter}, +}; -use base64::{engine::general_purpose, Engine}; +use base64::{Engine, engine::general_purpose}; use serde::Serialize; use serde_json::Value; use sha2::{Digest, Sha256}; -use uuid::Uuid; -#[derive(Default, Debug, Serialize)] +#[derive(Default, Debug, Serialize, Clone)] pub struct MdFile { - pub file_id: Uuid, - pub filename: String, pub meta: Value, pub content: String, pub hash: String, @@ -29,30 +26,22 @@ impl MdFile { Ok(file) => file, Err(e) => return Err(Error::FSError { msg: e.to_string() }), }; - let raw_file_bytes = raw_file.as_bytes().to_vec(); - let (front_matter_str, content) = split_md_front_matter(raw_file); + Self::from_str(&raw_file) + } +} + +impl FromStr for MdFile { + type Err = Error; + + fn from_str(s: &str) -> Result { + let bytes = s.as_bytes().to_vec(); + let (front_matter_str, content) = split_md_front_matter(s); let front_matter = match parse_front_matter(&front_matter_str) { Ok(front_matter) => front_matter, Err(e) => return Err(Error::InvalidFrontMatter { msg: e.to_string() }), }; - let hash = calculate_hash_base64(&raw_file_bytes); - let file_id = match front_matter["id"].as_str() { - Some(id) => match Uuid::from_str(id) { - Ok(id) => id, - Err(e) => return Err(Error::InvalidID { msg: e.to_string() }), - }, - None => return Err(Error::MissingID), - }; - let file_name = Path::new(file_path) - .file_name() - .unwrap() - .to_str() - .unwrap() - .to_string(); - + let hash = calculate_hash_base64(&bytes); Ok(MdFile { - file_id, - filename: file_name, meta: front_matter, content, hash, diff --git a/backend/markdown/src/utils/front_matter.rs b/backend/markdown/src/utils/front_matter.rs index 5bf77d2..966b390 100644 --- a/backend/markdown/src/utils/front_matter.rs +++ b/backend/markdown/src/utils/front_matter.rs @@ -1,17 +1,19 @@ +use std::error::Error; + use crate::utils::yaml::yaml_to_serde; + use regex::Regex; use serde_json::Value; -use std::error::Error; use yaml_rust2::YamlLoader; -pub fn split_md_front_matter(raw_file: String) -> (String, String) { +pub fn split_md_front_matter(raw_file: &str) -> (String, String) { let re = Regex::new(r"(?s)^(---\n.*?\n---\n)(.*)").unwrap(); - if let Some(captures) = re.captures(raw_file.as_str()) { + if let Some(captures) = re.captures(raw_file) { let front_matter = captures.get(1).map_or("", |m| m.as_str()).to_string(); let main_content = captures.get(2).map_or("", |m| m.as_str()).to_string(); (front_matter, main_content) } else { - (String::new(), raw_file) + (String::new(), raw_file.to_string()) } } @@ -43,8 +45,7 @@ tags: [ "test1", "test2" ] test body -"## - .to_string(), +"##, ); assert_eq!( front_matter, @@ -64,7 +65,7 @@ tags: [ "test1", "test2" ] #[test] fn test_front_matter_parser() { - dbg!(parse_front_matter( + parse_front_matter( r##"--- title: test_title date: 0000-00-00+0000 @@ -75,7 +76,7 @@ summary: summary tags: [ "test1", "test2" ] --- "##, - )) + ) .expect("Failed to parse front matter"); } } diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml new file mode 100644 index 0000000..f8e7c9a --- /dev/null +++ b/backend/server/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "server" +version = "0.1.0" +edition = "2024" + +[dependencies] +macros = { path = "../macros" } +markdown = { path = "../markdown" } + +axum = { version = "0.8.1", features = ["macros"] } +tracing = { version = "0.1.41", features = ["log"] } +git2 = "0.20.0" +chrono = { version = "0.4.39", features = ["serde"] } +cron = "0.15.0" +sqlx = { version = "0.8.3", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } +serde_json = "1.0.133" +serde = { version = "1.0.215", features = ["derive"] } +tracing-subscriber = "0.3.18" +tracing-appender = "0.2.3" +dirs = "6.0.0" +thiserror = "2.0.11" +tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros", "sync"] } +dotenv = "0.15.0" +uuid = { version = "1.13.2", features = ["v4"] } +futures = "0.3.31" + +[build-dependencies] +git2 = "0.20.0" +chrono = "0.4.39" diff --git a/backend/server/build.rs b/backend/server/build.rs new file mode 100644 index 0000000..aae7d8b --- /dev/null +++ b/backend/server/build.rs @@ -0,0 +1,16 @@ +use chrono::Utc; +use git2::Repository; + +fn main() { + let repo = Repository::open("../..").expect("failed to open repository"); + let commit = repo + .head() + .expect("failed to get HEAD") + .peel_to_commit() + .expect("failed to get HEAD commit"); + let hash = commit.id().to_string(); + let utc_timestamp = Utc::now().date_naive().format("%Y%m%d").to_string(); + let pkg_version = env!("CARGO_PKG_VERSION"); + let build_id = format!("{}:{}@{}", pkg_version, &hash[..8], utc_timestamp); + println!("cargo:rustc-env=BUILD_ID={}", build_id); +} diff --git a/backend/constants/src/blog.rs b/backend/server/src/constants/blog.rs similarity index 100% rename from backend/constants/src/blog.rs rename to backend/server/src/constants/blog.rs diff --git a/backend/constants/src/config.rs b/backend/server/src/constants/config.rs similarity index 97% rename from backend/constants/src/config.rs rename to backend/server/src/constants/config.rs index bf6ba77..74ece8c 100644 --- a/backend/constants/src/config.rs +++ b/backend/server/src/constants/config.rs @@ -1,3 +1,5 @@ +// todo: rebuild with static OnceCell + // Database related pub mod database { pub mod var_name { diff --git a/backend/constants/src/lib.rs b/backend/server/src/constants/mod.rs similarity index 100% rename from backend/constants/src/lib.rs rename to backend/server/src/constants/mod.rs diff --git a/backend/constants/src/time.rs b/backend/server/src/constants/time.rs similarity index 50% rename from backend/constants/src/time.rs rename to backend/server/src/constants/time.rs index d7b29ac..c5c6e21 100644 --- a/backend/constants/src/time.rs +++ b/backend/server/src/constants/time.rs @@ -1,2 +1,3 @@ pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; -pub const DEFAULT_DATE_FORMAT: &str = "%Y-%m-%d%zT%H%M%S"; + +pub const DEFAULT_NAIVE_DATE_FORMAT: &str = "%Y-%m-%d"; diff --git a/backend/api_server/src/init/database.rs b/backend/server/src/init/database.rs similarity index 91% rename from backend/api_server/src/init/database.rs rename to backend/server/src/init/database.rs index c2a8aad..f237e1e 100644 --- a/backend/api_server/src/init/database.rs +++ b/backend/server/src/init/database.rs @@ -2,7 +2,8 @@ use std::env; use std::error::Error; use std::str::FromStr; -use constants::config::database::{default_value, var_name}; +use crate::constants::config::database::{default_value, var_name}; + use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; use sqlx::{ConnectOptions, Pool, Postgres}; use tracing::info; @@ -12,7 +13,7 @@ pub async fn init() -> Result, Box> { .unwrap() .disable_statement_logging(); Ok(PgPoolOptions::new() - .max_connections(1) // todo: temporal solution, to be fixed + .max_connections(1) // todo: temporary solution, to be fixed .connect_with(db_connect_option) .await?) } diff --git a/backend/api_server/src/init/logger.rs b/backend/server/src/init/logger.rs similarity index 92% rename from backend/api_server/src/init/logger.rs rename to backend/server/src/init/logger.rs index 5fd56b4..887c633 100644 --- a/backend/api_server/src/init/logger.rs +++ b/backend/server/src/init/logger.rs @@ -1,14 +1,15 @@ use std::path::Path; use std::{env, io}; +use crate::constants::config::general::{default_value, var_name}; + use chrono::Local; -use constants::config::general::{default_value, var_name}; use dirs::home_dir; use tracing::info; use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; use tracing_subscriber::{ - filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, - layer::SubscriberExt, Layer, Registry, + Layer, Registry, filter::LevelFilter, fmt, fmt::MakeWriter, fmt::format::Writer, + fmt::time::FormatTime, layer::SubscriberExt, }; struct LocalTimer; @@ -18,7 +19,7 @@ impl FormatTime for LocalTimer { write!( w, "{}", - Local::now().format(constants::time::DEFAULT_TIME_FORMAT) + Local::now().format(crate::constants::time::DEFAULT_TIME_FORMAT) ) } } diff --git a/backend/api_server/src/init/misc.rs b/backend/server/src/init/misc.rs similarity index 100% rename from backend/api_server/src/init/misc.rs rename to backend/server/src/init/misc.rs diff --git a/backend/api_server/src/init/mod.rs b/backend/server/src/init/mod.rs similarity index 58% rename from backend/api_server/src/init/mod.rs rename to backend/server/src/init/mod.rs index 507d52a..8710bb0 100644 --- a/backend/api_server/src/init/mod.rs +++ b/backend/server/src/init/mod.rs @@ -1,13 +1,13 @@ -mod cron; mod database; pub mod logger; mod misc; -mod schema; -pub mod server; -use macros::panic_with_log; use std::{env, panic}; -use tracing::{info, Level}; + +use crate::modules::{ModuleTree, blog::Blog}; +use macros::error_panic; + +use tracing::info; pub fn load_env() { if let Some(env_file) = env::args().nth(1) { @@ -34,25 +34,17 @@ pub async fn start() { pool } Err(e) => { - panic_with_log!( - Level::ERROR, - "database connection pool failed to initialize: {}", - e - ); + error_panic!("database connection pool failed to initialize: {}", e); } }; - // schema init - schema::init(&db_pool).await; + // // schema init + // schema::init(&db_pool).await; - // load cron task - cron::start(db_pool.clone()); - - // api serve - match server::start(db_pool).await { - Ok(_) => {} - Err(e) => { - panic_with_log!(Level::ERROR, "server panicked: {}", e) - } - } + ModuleTree::new() + .add_module(Blog { + db_handler: db_pool, + }) + .blocking_serve() + .await } diff --git a/backend/server/src/main.rs b/backend/server/src/main.rs new file mode 100644 index 0000000..364b2b3 --- /dev/null +++ b/backend/server/src/main.rs @@ -0,0 +1,22 @@ +#[allow(dead_code)] +mod constants; + +mod scheduler; + +mod init; +mod modules; +mod utils; + +use crate::init::{logger, start}; + +const BUILD_ID: &str = env!("BUILD_ID"); + +#[tokio::main] +async fn main() { + println!("{}", BUILD_ID); + + init::load_env(); + let _guards = logger::init(); + + start().await; +} diff --git a/backend/server/src/modules/blog/api/get_page_count/mod.rs b/backend/server/src/modules/blog/api/get_page_count/mod.rs new file mode 100644 index 0000000..ce20576 --- /dev/null +++ b/backend/server/src/modules/blog/api/get_page_count/mod.rs @@ -0,0 +1,25 @@ +use crate::constants::blog::PAGE_SIZE; + +use axum::{Json, extract::State}; +use serde_json::{Value, json}; +use sqlx::{PgPool, Row, query}; +use tracing::error; + +const GET_PAGE_COUNT_QUERY: &str = include_str!("post_page_count.sql"); + +pub async fn handler(State(db_connection): State) -> Result, Json> { + match query(GET_PAGE_COUNT_QUERY) + .bind(PAGE_SIZE as i64) + .fetch_one(&db_connection) + .await + { + Ok(res) => { + let res: i32 = res.get("count"); + Ok(Json::from(json!({"res": res}))) + } + Err(e) => { + error!("failed to get blog page count: {}", e.to_string()); + Err(Json::from(json!({"error": e.to_string()}))) + } + } +} diff --git a/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql b/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql new file mode 100644 index 0000000..5ccce6b --- /dev/null +++ b/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql @@ -0,0 +1,2 @@ +SELECT cast(ceil(count(*) / cast($1 AS FLOAT)) AS INTEGER) AS count +FROM posts; diff --git a/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql b/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql new file mode 100644 index 0000000..7dc5107 --- /dev/null +++ b/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql @@ -0,0 +1 @@ +SELECT \ No newline at end of file diff --git a/frontend/app/public/css/article.css b/backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql similarity index 100% rename from frontend/app/public/css/article.css rename to backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql diff --git a/backend/server/src/modules/blog/api/get_post/mod.rs b/backend/server/src/modules/blog/api/get_post/mod.rs new file mode 100644 index 0000000..a2b8df7 --- /dev/null +++ b/backend/server/src/modules/blog/api/get_post/mod.rs @@ -0,0 +1,35 @@ +use macros::dev_consume; + +use axum::{ + Json, + extract::{Path, State}, + response::IntoResponse, +}; +use serde_json::json; +use sqlx::PgPool; +use uuid::Uuid; + +const GET_POST_BY_UUID: &str = include_str!("get_post_by_uuid.sql"); +const GET_POST_BY_STR_ID: &str = include_str!("get_post_by_str_id.sql"); + +pub async fn handler( + State(db_connection): State, + Path(identifier): Path, +) -> impl IntoResponse { + if let Ok(id) = identifier.parse::() { + select_post_by_uuid(&db_connection, id).await + } else { + select_post_by_str_id(&db_connection, &identifier).await + }; + Json::from(json!({"res": "ok"})).into_response() +} + +async fn select_post_by_uuid(db: &PgPool, id: Uuid) { + dev_consume!(db, id); + todo!() +} + +async fn select_post_by_str_id(db: &PgPool, id: &str) { + dev_consume!(db, id); + todo!() +} diff --git a/backend/api_server/src/blog/api/get_post_list.rs b/backend/server/src/modules/blog/api/get_post_list.rs similarity index 98% rename from backend/api_server/src/blog/api/get_post_list.rs rename to backend/server/src/modules/blog/api/get_post_list.rs index 9db7c61..81800aa 100644 --- a/backend/api_server/src/blog/api/get_post_list.rs +++ b/backend/server/src/modules/blog/api/get_post_list.rs @@ -1,6 +1,6 @@ use std::error::Error; -use constants::blog::PAGE_SIZE; +use crate::constants::blog::PAGE_SIZE; use axum::{extract::Query, extract::State, http::StatusCode, Json}; use chrono::{DateTime, Utc}; diff --git a/backend/server/src/modules/blog/api/mod.rs b/backend/server/src/modules/blog/api/mod.rs new file mode 100644 index 0000000..5eb1f92 --- /dev/null +++ b/backend/server/src/modules/blog/api/mod.rs @@ -0,0 +1,6 @@ +pub mod get_page_count; + +// todo: remove exception tag +#[allow(dead_code)] +pub mod get_post; +// pub mod get_post_list; diff --git a/backend/server/src/modules/blog/components/check_update/apply.rs b/backend/server/src/modules/blog/components/check_update/apply.rs new file mode 100644 index 0000000..1577947 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/apply.rs @@ -0,0 +1,208 @@ +use crate::modules::blog::components::check_update::{ + changes::{Change, Create, Delete, Move, Update}, + error::Error, + utils::{parse_naive_date_str, parse_post_identifier}, +}; + +use futures::{ + future::{join_all, BoxFuture}, + FutureExt, +}; +use sqlx::{query, PgPool, Postgres, Row, Transaction}; +use tracing::error; + +const DELETE_POST_BY_ID: &str = include_str!("sql/delete_post_by_id.sql"); +const GET_TAG_ID: &str = include_str!("sql/get_tag_id.sql"); +const GET_CATEGORY_ID: &str = include_str!("sql/get_category_id.sql"); +const CREATE_RETURN_TAG_ID: &str = include_str!("sql/create_return_tag_id.sql"); +const CREATE_RETURN_CATEGORY_ID: &str = include_str!("sql/create_return_category_id.sql"); +const CREATE_POST: &str = include_str!("sql/create_post.sql"); + +pub async fn apply_deltas(db_conn: &PgPool, changes: Vec) { + let handlers = changes + .into_iter() + .map(|change| match change { + Change::Delete(del) => handle_delete(db_conn, del).boxed(), + Change::Move(mv) => handle_move(db_conn, mv).boxed(), + Change::Update(update) => handle_update(db_conn, update).boxed(), + Change::Create(create) => handle_create(db_conn, create).boxed(), + }) + .collect::>>(); + + join_all(handlers).await; +} + +async fn handle_delete(db: &PgPool, del: Delete) { + if let Err(e) = query(DELETE_POST_BY_ID).bind(del.uuid).execute(db).await { + error!( + "failed to delete post: {}, {}", + del.uuid.to_string(), + e.to_string() + ) + } +} + +async fn handle_move(db: &PgPool, mv: Move) { + handle_delete( + db, + Delete { + uuid: mv.uuid, + path: mv.from, + }, + ) + .await; + handle_create( + db, + Create { + uuid: mv.uuid, + path: mv.to, + payload: mv.payload, + }, + ) + .await; +} + +async fn handle_update(db: &PgPool, update: Update) { + handle_delete( + db, + Delete { + uuid: update.uuid, + path: update.path.clone(), + }, + ) + .await; + handle_create( + db, + Create { + uuid: update.uuid, + path: update.path, + payload: update.payload, + }, + ) + .await; +} + +async fn handle_create(db: &PgPool, create: Create) { + let mut tx = db.begin().await.unwrap(); + + let identifier = parse_post_identifier(&create.path).unwrap(); + + let tags_json = create.payload.meta["tags"].clone(); + let tags: Vec<_> = if let Some(tags) = tags_json.as_array() { + tags.iter().filter_map(|tag| tag.as_str()).collect() + } else { + vec![] + }; + let mut tag_ids = vec![]; + for tag in tags { + match get_or_create_tag_id(&mut tx, tag).await { + Ok(tid) => tag_ids.push(tid), + Err(e) => { + error!("failed to fetch or create tag ({}): {}", tag, e); + return; + } + } + } + + let category_json = create.payload.meta["category"].clone(); + let category = category_json.as_str().unwrap_or("Uncategorized"); + let category_id = match get_or_create_category_id(&mut tx, category).await { + Ok(id) => id, + Err(e) => { + error!( + "failed to fetch category id for post {}: {}", + &identifier, e + ); + return; + } + }; + + let meta = create.payload.meta; + let post_id = create.uuid; + let post_identifier = &identifier; + let post_title = meta + .get("title") + .map(|t| t.as_str().unwrap()) + .unwrap_or("Untitled"); + let post_subtitle = meta + .get("subtitle") + .map(|t| t.as_str().unwrap()) + .unwrap_or(""); + let post_date_created = if let Some(Some(date)) = meta + .get("date") + .map(|d| parse_naive_date_str(d.as_str().unwrap_or(""))) + { + date + } else { + error!("failed to parse `date` for post: {}", post_identifier); + return; + }; + let post_date_updated = meta + .get("update") + .map(|d| parse_naive_date_str(d.as_str().unwrap_or(""))) + .unwrap_or(None); + let post_status = meta.get("status").map(|v| v.as_str()).unwrap_or(None); + let post_content = create.payload.content; + + if let Err(e) = query(CREATE_POST) + .bind(post_id) + .bind(post_identifier) + .bind(post_title) + .bind(post_subtitle) + .bind(post_date_created) + .bind(post_date_updated) + .bind(post_status) + .bind(post_content) + .bind(category_id) + .bind(tag_ids) + .execute(&mut *tx) + .await + { + error!("failed to insert post {}: {}", &identifier, e); + return; + } + + tx.commit().await.unwrap(); +} + +async fn get_or_create_tag_id( + db: &mut Transaction<'_, Postgres>, + tag_name: &str, +) -> Result { + get_create_id(db, tag_name, GET_TAG_ID, CREATE_RETURN_TAG_ID).await +} + +async fn get_or_create_category_id( + db: &mut Transaction<'_, Postgres>, + category_name: &str, +) -> Result { + get_create_id( + db, + category_name, + GET_CATEGORY_ID, + CREATE_RETURN_CATEGORY_ID, + ) + .await +} + +async fn get_create_id( + db: &mut Transaction<'_, Postgres>, + field_name: &str, + fetch_query: &str, + create_fetch_query: &str, +) -> Result { + if let Some(id) = query(fetch_query) + .bind(field_name) + .fetch_optional(&mut **db) + .await? + { + Ok(id.get("id")) + } else { + let id = query(create_fetch_query) + .bind(field_name) + .fetch_one(&mut **db) + .await? + .get("id"); + Ok(id) + } +} diff --git a/backend/server/src/modules/blog/components/check_update/changes.rs b/backend/server/src/modules/blog/components/check_update/changes.rs new file mode 100644 index 0000000..c0d782c --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/changes.rs @@ -0,0 +1,68 @@ +use std::path::PathBuf; + +use markdown::MdFile; + +use uuid::Uuid; + +#[allow(dead_code)] +#[derive(Debug)] +pub enum Change { + Delete(Delete), + Move(Move), + Update(Update), + Create(Create), +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct Delete { + pub uuid: Uuid, + pub path: PathBuf, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct Move { + pub uuid: Uuid, + pub from: PathBuf, + pub to: PathBuf, + pub payload: MdFile, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct Update { + pub uuid: Uuid, + pub path: PathBuf, + pub payload: MdFile, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct Create { + pub uuid: Uuid, + pub path: PathBuf, + pub payload: MdFile, +} + +#[derive(Debug, Clone, Default)] +pub struct CreateDelete2UpdateSlot { + pub create: Option, + pub delete: Option, +} + +impl CreateDelete2UpdateSlot { + pub fn to_change(&self) -> Option { + match (&self.create, &self.delete) { + (Some(create), Some(delete)) => Some(Change::Move(Move { + uuid: create.uuid, + from: delete.path.clone(), + to: create.path.clone(), + payload: create.payload.clone(), + })), + (Some(create), None) => Some(Change::Create(create.clone())), + (None, Some(delete)) => Some(Change::Delete(delete.clone())), + (None, None) => None, + } + } +} diff --git a/backend/server/src/modules/blog/components/check_update/error.rs b/backend/server/src/modules/blog/components/check_update/error.rs new file mode 100644 index 0000000..83ee773 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/error.rs @@ -0,0 +1,18 @@ +use markdown::error::Error as MarkdownError; + +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("markdown file error")] + MdFile(#[from] MarkdownError), + + #[error("db error")] + Db(#[from] sqlx::Error), + + #[error("git error")] + Git2(#[from] git2::Error), + + #[error("git remote url mismatch")] + GitUrlMismatch, +} diff --git a/backend/server/src/modules/blog/components/check_update/mod.rs b/backend/server/src/modules/blog/components/check_update/mod.rs new file mode 100644 index 0000000..9470fa3 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/mod.rs @@ -0,0 +1,103 @@ +use std::{ + env, + path::{Path, PathBuf}, + sync::Arc, +}; + +use crate::{ + constants::config::general::{default_value, var_name}, + modules::blog::components::check_update::{ + apply::apply_deltas, pull::fetch_deltas, utils::expand_path, + }, + scheduler::task_func::AsyncTaskFunc, +}; +use macros::panic_with_log; + +use sqlx::PgPool; +use tracing::{log::error, trace, warn, Level}; + +// mod error; +// mod index; +mod apply; +mod changes; +mod error; +mod pull; +mod utils; + +#[derive(Debug)] +struct Args { + db_connection: PgPool, + git_work_dir: PathBuf, + git_url: String, + git_remote_name: String, + git_remote_branch: String, +} + +pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc { + // init + let git_work_dir = { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_path(default_log_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") + }; + + // clone & check update + let git_url = match env::var(var_name::BLOG_GIT_URL) { + Ok(url) => url, + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to read `{}`: {}", + var_name::BLOG_GIT_URL, + e + ) + } + }; + let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) + .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); + let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) + .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); + + let arc_args = Arc::new(Args { + db_connection, + git_work_dir, + git_url, + git_remote_name, + git_remote_branch, + }); + + move || Box::pin(workflow(arc_args.clone())) +} + +async fn workflow(args: Arc) { + let deltas = match fetch_deltas( + &args.git_url, + &args.git_remote_name, + &args.git_remote_branch, + &args.git_work_dir, + ) + .await + { + Ok(updates) => updates, + Err(e) => { + error!("delta parsing error: {}", e); + return; + } + }; + + if deltas.is_empty() { + trace!("blog git already up-to-date"); + return; + } + + apply_deltas(&args.db_connection, deltas).await; +} diff --git a/backend/server/src/modules/blog/components/check_update/pull.rs b/backend/server/src/modules/blog/components/check_update/pull.rs new file mode 100644 index 0000000..e52cdc1 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/pull.rs @@ -0,0 +1,171 @@ +use std::{collections::HashMap, fs::remove_dir_all, path::PathBuf}; + +use crate::{ + modules::blog::components::check_update::{ + changes::{Change, Create, CreateDelete2UpdateSlot, Delete, Update}, + error::Error, + }, + utils::{git, git::FileDelta}, +}; +use markdown::MdFile; + +use git2::{Delta, Repository}; +use tracing::{error, warn}; +use uuid::Uuid; + +pub async fn fetch_deltas( + git_url: &str, + git_remote_name: &str, + git_remote_branch: &str, + git_work_dir: &PathBuf, +) -> Result, Error> { + let repo = match Repository::open(git_work_dir) { + Ok(repo) => repo, + Err(_) => match Repository::clone(git_url, git_work_dir) { + Ok(repo) => repo, + Err(e) => { + error!("failed to clone blog content git({}): {}", &git_url, e); + return Err(e.into()); + } + }, + }; + + let mut remote = match repo.find_remote(git_remote_name) { + Ok(remote) => remote, + Err(e) => { + remove_dir_all(git_work_dir).unwrap(); + warn!( + "cannot find git remote ({}), reinitializing: {}", + git_remote_name, e + ); + return Err(e.into()); + } + }; + if remote.url().is_some_and(|url| url != git_url) { + remove_dir_all(git_work_dir).unwrap(); + warn!( + "git remote url mismatch (expected: {}, actual: {}), reinitializing", + git_url, + remote.url().unwrap() + ); + return Err(Error::GitUrlMismatch); + } + + let fetched_commit = git::do_fetch(&repo, &[git_remote_branch], &mut remote)?; + let commit_deltas = git::calculate_diff( + &repo, + &repo.reference_to_annotated_commit(&repo.head()?)?, + &fetched_commit, + )?; + + // dbg!(&commit_deltas); + let mut changes: Vec = vec![]; + let mut non_delete_deltas: Vec = vec![]; + + // process deleted deltas before merging commit to prevent "file not found" os error + for delta in commit_deltas.into_iter() { + // todo: limit scanning scope to post only OR extract merging logic to external component + + if delta.status == Delta::Deleted { + let file_path = &git_work_dir.join( + delta + .new_path + .expect("cannot parse modified post `new_path`"), + ); + if let Some(id) = extract_post_id(&MdFile::from_file(file_path)?) { + changes.push(Change::Delete(Delete { + uuid: id, + path: file_path.clone(), + })) + } else { + error!("failed to parse uuid for file {}", file_path.display()); + // todo: add to unresolved error list + } + } else { + non_delete_deltas.push(delta); + } + } + + git::do_merge(&repo, git_remote_branch, fetched_commit)?; + + // process adds and modifies deltas + for delta in non_delete_deltas.into_iter() { + let file_path = &git_work_dir.join( + delta + .new_path + .expect("cannot parse modified post `new_path`"), + ); + match delta.status { + Delta::Added => { + let md_file = MdFile::from_file(file_path)?; + if let Some(id) = extract_post_id(&md_file) { + changes.push(Change::Create(Create { + uuid: id, + path: file_path.clone(), + payload: md_file, + })) + } else { + error!("failed to parse uuid for file {}", file_path.display()); + // todo: add to unresolved error list + } + } + Delta::Modified => { + let md_file = MdFile::from_file(file_path)?; + if let Some(id) = extract_post_id(&md_file) { + changes.push(Change::Update(Update { + uuid: id, + path: file_path.clone(), + payload: md_file, + })) + } else { + error!("failed to parse uuid for file {}", file_path.display()); + // todo: add to unresolved error list + } + } + Delta::Deleted => continue, + _ => { + error!("unsupported delta type on file {}", file_path.display()); + continue; + } + } + } + + // merge adds and deletes deltas to moves + let changes = dbg!(merge_changes(changes)); + + Ok(changes) +} + +fn extract_post_id(post: &MdFile) -> Option { + match post.meta["uuid"].as_str() { + Some(id) => Uuid::parse_str(id).ok(), + None => None, + } +} + +fn merge_changes(changes: Vec) -> Vec { + let mut slots: HashMap = HashMap::new(); + let mut merged: Vec = Vec::new(); + + for change in changes { + match change { + Change::Create(create) => { + let id = create.uuid; + slots.entry(id).or_default().create = Some(create); + } + Change::Delete(delete) => { + let id = delete.uuid; + slots.entry(id).or_default().delete = Some(delete); + } + _ => merged.push(change), + } + } + + for slot in slots.values() { + if let Some(change) = slot.to_change() { + merged.push(change); + } + } + + merged +} diff --git a/backend/server/src/modules/blog/components/check_update/sql/create_post.sql b/backend/server/src/modules/blog/components/check_update/sql/create_post.sql new file mode 100644 index 0000000..2cf2700 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/create_post.sql @@ -0,0 +1,7 @@ +INSERT INTO posts +(id, identifier, title, subtitle, date_created, date_updated, status, content, category) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9); + +INSERT INTO post_tag (post, tag) +SELECT $1, tag +FROM UNNEST($10) AS t(tag); diff --git a/backend/server/src/modules/blog/components/check_update/sql/create_return_category_id.sql b/backend/server/src/modules/blog/components/check_update/sql/create_return_category_id.sql new file mode 100644 index 0000000..7ff139a --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/create_return_category_id.sql @@ -0,0 +1,3 @@ +INSERT INTO categories(name) +VALUES ($1) +RETURNING id; diff --git a/backend/server/src/modules/blog/components/check_update/sql/create_return_tag_id.sql b/backend/server/src/modules/blog/components/check_update/sql/create_return_tag_id.sql new file mode 100644 index 0000000..96e49a1 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/create_return_tag_id.sql @@ -0,0 +1,3 @@ +INSERT INTO tags(name) +VALUES ($1) +RETURNING id; diff --git a/backend/server/src/modules/blog/components/check_update/sql/delete_post_by_id.sql b/backend/server/src/modules/blog/components/check_update/sql/delete_post_by_id.sql new file mode 100644 index 0000000..1c3f157 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/delete_post_by_id.sql @@ -0,0 +1,3 @@ +DELETE +FROM posts +WHERE id = $1; \ No newline at end of file diff --git a/backend/server/src/modules/blog/components/check_update/sql/get_category_id.sql b/backend/server/src/modules/blog/components/check_update/sql/get_category_id.sql new file mode 100644 index 0000000..a404aea --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/get_category_id.sql @@ -0,0 +1,3 @@ +SELECT id +FROM categories +WHERE name = $1; \ No newline at end of file diff --git a/backend/server/src/modules/blog/components/check_update/sql/get_tag_id.sql b/backend/server/src/modules/blog/components/check_update/sql/get_tag_id.sql new file mode 100644 index 0000000..965effa --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/get_tag_id.sql @@ -0,0 +1,3 @@ +SELECT id +FROM tags +WHERE name = $1; \ No newline at end of file diff --git a/backend/server/src/modules/blog/components/check_update/utils.rs b/backend/server/src/modules/blog/components/check_update/utils.rs new file mode 100644 index 0000000..f3c84ec --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/utils.rs @@ -0,0 +1,58 @@ +use std::path::Path; + +use crate::constants::{ + config::general::{default_value, var_name}, + time::DEFAULT_NAIVE_DATE_FORMAT, +}; + +use chrono::NaiveDate; +use dirs::home_dir; +use tracing::warn; + +pub fn expand_path(path: String) -> String { + if path.starts_with('~') { + home_dir() + .map(|dir| { + dir.join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + }) + .unwrap_or_else(|| { + warn!( + "failed to parse env var `{}`, using default value `{}`", + var_name::WORK_DIR, + default_value::WORK_DIR + ); + default_value::WORK_DIR.to_string() + }) + } else { + path + } +} + +pub fn parse_post_identifier(path: &Path) -> Option { + if let Some(comp) = path.components().nth_back(1) { + comp.as_os_str().to_str().map(|s| s.to_string()) + } else { + None + } +} + +pub fn parse_naive_date_str(date_str: &str) -> Option { + NaiveDate::parse_from_str(date_str, DEFAULT_NAIVE_DATE_FORMAT).ok() +} + +#[cfg(test)] +mod tests { + use crate::modules::blog::components::check_update::utils::parse_naive_date_str; + use chrono::NaiveDate; + + #[test] + fn test_parse_naive_date() { + assert_eq!( + NaiveDate::from_ymd_opt(2015, 1, 1), + parse_naive_date_str("2015-1-01") + ); + } +} diff --git a/backend/server/src/modules/blog/components/consistency_guard/mod.rs b/backend/server/src/modules/blog/components/consistency_guard/mod.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/server/src/modules/blog/components/consistency_guard/mod.rs @@ -0,0 +1 @@ + diff --git a/backend/server/src/modules/blog/components/mod.rs b/backend/server/src/modules/blog/components/mod.rs new file mode 100644 index 0000000..0f3bc78 --- /dev/null +++ b/backend/server/src/modules/blog/components/mod.rs @@ -0,0 +1,3 @@ +pub mod check_update; +mod consistency_guard; +mod rag; diff --git a/backend/server/src/modules/blog/components/rag/mod.rs b/backend/server/src/modules/blog/components/rag/mod.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/server/src/modules/blog/components/rag/mod.rs @@ -0,0 +1 @@ + diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs new file mode 100644 index 0000000..f536972 --- /dev/null +++ b/backend/server/src/modules/blog/mod.rs @@ -0,0 +1,55 @@ +mod api; +mod components; + +use crate::modules::blog::components::check_update; +use crate::scheduler::tasks::async_basic::AsyncBasic; +use crate::{ + modules::{ + Module, + blog::{ + api::{ + get_page_count, + get_post, + // get_post_list::get_post_list, + }, + // cron::check_update, + }, + }, + scheduler::tasks::CronTask, +}; + +use axum::{Router, routing::get}; +use sqlx::PgPool; + +const MOUNT_POINT: &str = "/blog"; + +pub struct Blog { + pub db_handler: PgPool, +} + +impl Module for Blog { + fn get_server_router(&self) -> Router { + let db_handler = self.db_handler.clone(); + // todo: move resource init to new func + // todo: remove debug comment + Router::new() + // .route("/get_post_list", get(get_post_list)) + .route("/get_page_count", get(get_page_count::handler)) + .route("/get_post/{identifier}", get(get_post::handler)) + .with_state(db_handler) + } + + fn get_mount_point(&self) -> &'static str { + MOUNT_POINT + } + + fn get_cron_tasks(&self) -> Vec<(&str, Box)> { + let db_handler = self.db_handler.clone(); + vec![( + "Blog Check Update", + AsyncBasic::new(check_update::task(db_handler.clone()), "*/5 * * * * *") + .unwrap() + .to_task(), + )] + } +} diff --git a/backend/server/src/modules/blog/schema.sql b/backend/server/src/modules/blog/schema.sql new file mode 100644 index 0000000..a8d7b35 --- /dev/null +++ b/backend/server/src/modules/blog/schema.sql @@ -0,0 +1,65 @@ +CREATE TABLE tags +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT DEFAULT '' +); + +CREATE TABLE categories +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT DEFAULT '' +); + +CREATE TABLE post_status +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL +); + +INSERT INTO post_status (name) +VALUES ('draft'), + ('published'), + ('planned'), + ('hidden'); + +CREATE TABLE posts +( + id UUID PRIMARY KEY, + identifier VARCHAR(100) UNIQUE NOT NULL, + title VARCHAR(255) NOT NULL, + subtitle VARCHAR(255) NOT NULL, + date_created DATE NOT NULL, + date_updated DATE, + status INTEGER REFERENCES post_status (id) DEFAULT 1 NOT NULL, + content TEXT NOT NULL, + category INTEGER REFERENCES categories (id) NOT NULL +); + +CREATE TABLE post_tag +( + post UUID REFERENCES posts (id) ON DELETE CASCADE, + tag INTEGER REFERENCES tags (id) ON DELETE CASCADE, + + CONSTRAINT post_tag_pk PRIMARY KEY (post, tag) +); + +CREATE TABLE metadata +( + pid UUID REFERENCES posts (id) ON DELETE CASCADE NOT NULL, + meta_key VARCHAR(50) NOT NULL, + meta_value TEXT NOT NULL, + + CONSTRAINT metadata_pk PRIMARY KEY (pid, meta_key) +); + +-- index +CREATE INDEX idx_tags_name ON tags (name); +CREATE INDEX idx_categories_name ON categories (name); +CREATE INDEX idx_posts_identifier ON posts (identifier); +CREATE INDEX idx_posts_status ON posts (status); +CREATE INDEX idx_posts_category ON posts (category); +CREATE INDEX idx_post_tag_post ON post_tag (post); +CREATE INDEX idx_post_tag_tag ON post_tag (tag); +CREATE INDEX idx_metadata_pid ON metadata (pid); \ No newline at end of file diff --git a/backend/server/src/modules/mod.rs b/backend/server/src/modules/mod.rs new file mode 100644 index 0000000..72e3149 --- /dev/null +++ b/backend/server/src/modules/mod.rs @@ -0,0 +1,82 @@ +use std::{env, num::ParseIntError}; + +use crate::{ + constants::config::server::{default_value, var_name}, + scheduler::{Scheduler, tasks::CronTask}, +}; +use macros::error_panic; + +use axum::Router; +use tokio::net::TcpListener; +use tracing::{info, warn}; + +pub mod blog; + +// todo: merge `get_server_router` & `get_mount_point`, make Option +pub trait Module { + fn get_server_router(&self) -> Router; + + fn get_mount_point(&self) -> &'static str; + + fn get_cron_tasks(&self) -> Vec<(&str, Box)>; +} + +pub struct ModuleTree { + pub modules: Vec>, +} + +impl ModuleTree { + pub fn new() -> Self { + ModuleTree { modules: vec![] } + } + + pub fn add_module(&mut self, module: impl Module + 'static) -> &mut Self { + self.modules.push(Box::new(module)); + self + } + + pub async fn blocking_serve(&self) { + let mut scheduler = Scheduler::new(); + let mut app = Router::new(); + + let port: u32 = match env::var(var_name::PORT) { + Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { + warn!( + "failed to parse `{}`, using default port: {}", + var_name::PORT, + e.to_string() + ); + default_value::PORT + }), + Err(_) => default_value::PORT, + }; + + let listener = match TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await { + Ok(listener) => { + info!("server starting on {}:{}", default_value::HOST, port); + listener + } + Err(e) => { + error_panic!( + "failed to start server on {}:{}: {}", + default_value::HOST, + port, + e.to_string() + ); + } + }; + + for module in &self.modules { + let _ = scheduler.insert_list(module.get_cron_tasks()); + app = app.nest(module.get_mount_point(), module.get_server_router()); + } + + tokio::spawn(async move { + scheduler.start().await; + }); + + axum::serve(listener, app).await.unwrap_or_else(|e| { + error_panic!("failed to start axum: {}", e.to_string()); + }); + } +} diff --git a/backend/api_server/src/scheduler/scheduler_error.rs b/backend/server/src/scheduler/error.rs similarity index 82% rename from backend/api_server/src/scheduler/scheduler_error.rs rename to backend/server/src/scheduler/error.rs index 43f30dd..a7e5cd5 100644 --- a/backend/api_server/src/scheduler/scheduler_error.rs +++ b/backend/server/src/scheduler/error.rs @@ -1,8 +1,13 @@ -use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; -use std::error::Error; -use std::fmt::{Debug, Display, Formatter}; -use std::sync::PoisonError; +use std::{ + error::Error, + fmt::{Debug, Display, Formatter}, + sync::PoisonError, +}; +use crate::scheduler::error::SchedulerError::MutexPoison; + +// todo: refactor with thiserror +#[allow(dead_code)] pub enum SchedulerError { IdNotFound, IdAlreadyRunning, diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs new file mode 100644 index 0000000..f79c277 --- /dev/null +++ b/backend/server/src/scheduler/mod.rs @@ -0,0 +1,54 @@ +use std::{collections::HashMap, sync::Arc}; + +use crate::scheduler::{error::SchedulerError, tasks::CronTask, utils::execute}; + +use futures::future::join_all; +use tracing::warn; + +pub mod error; +pub mod task_func; +pub mod tasks; +mod utils; + +pub type TaskPool = HashMap>>; + +#[derive(Default)] +pub struct Scheduler { + task_pool: TaskPool, +} + +impl Scheduler { + /// Start all tasks in the scheduler + pub async fn start(&self) { + let fut = self + .task_pool + .clone() + .into_iter() + .map(|(name, task)| execute(name, task)) + .collect::>(); + // self.runner.run_all().await + join_all(fut).await; + } + + pub fn insert(&mut self, task: (&str, Box)) -> Result<&Self, SchedulerError> { + let (k, v) = task; + if self.task_pool.insert(k.into(), v.into()).is_some() { + warn!("task list duplicated key detected: {}", k); + } + Ok(self) + } + + pub fn insert_list( + &mut self, + tasks: Vec<(&str, Box)>, + ) -> Result<&Self, SchedulerError> { + for task in tasks { + self.insert(task)?; + } + Ok(self) + } + + pub fn new() -> Self { + Self::default() + } +} diff --git a/backend/server/src/scheduler/task_func.rs b/backend/server/src/scheduler/task_func.rs new file mode 100644 index 0000000..ceafe54 --- /dev/null +++ b/backend/server/src/scheduler/task_func.rs @@ -0,0 +1,15 @@ +use std::pin::Pin; + +pub trait TaskFunc: Fn() + Send + Sync + 'static {} + +impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} + +pub trait AsyncTaskFunc: + Fn() -> Pin + Send>> + Send + Sync + 'static +{ +} + +impl AsyncTaskFunc for T where + T: Fn() -> Pin + Send>> + Send + Sync + Clone + 'static +{ +} diff --git a/backend/server/src/scheduler/tasks/async_basic.rs b/backend/server/src/scheduler/tasks/async_basic.rs new file mode 100644 index 0000000..3de6f37 --- /dev/null +++ b/backend/server/src/scheduler/tasks/async_basic.rs @@ -0,0 +1,37 @@ +use std::{error::Error, pin::Pin, str::FromStr}; + +use crate::{scheduler::task_func::AsyncTaskFunc, scheduler::tasks::CronTask}; + +use chrono::{DateTime, Utc}; +use cron::Schedule; + +pub struct AsyncBasic { + schedule: Schedule, + func: Box, +} + +impl AsyncBasic { + pub fn new(func: F, cron_expr: &str) -> Result> + where + F: AsyncTaskFunc, + { + Ok(AsyncBasic { + schedule: Schedule::from_str(cron_expr)?, + func: Box::new(func), + }) + } + + pub fn to_task(self) -> Box { + Box::new(self) + } +} + +impl CronTask for AsyncBasic { + fn get_next_execution(&self) -> Option> { + self.schedule.upcoming(Utc).next() + } + + fn call(&self) -> Pin + Send)>> { + Box::pin((self.func)()) + } +} diff --git a/backend/api_server/src/scheduler/tasks/basic.rs b/backend/server/src/scheduler/tasks/basic.rs similarity index 63% rename from backend/api_server/src/scheduler/tasks/basic.rs rename to backend/server/src/scheduler/tasks/basic.rs index 71fc3a6..cc02ed7 100644 --- a/backend/api_server/src/scheduler/tasks/basic.rs +++ b/backend/server/src/scheduler/tasks/basic.rs @@ -1,9 +1,9 @@ -use crate::scheduler::task_func::TaskFunc; -use crate::scheduler::tasks::Task; +use std::{error::Error, pin::Pin, str::FromStr}; + +use crate::scheduler::{task_func::TaskFunc, tasks::CronTask}; + use chrono::{DateTime, Utc}; use cron::Schedule; -use std::error::Error; -use std::str::FromStr; pub struct BasicTask { schedule: Schedule, @@ -21,17 +21,17 @@ impl BasicTask { }) } - pub fn to_task(self) -> Box { + pub fn to_task(self) -> Box { Box::new(self) } } -impl Task for BasicTask { +impl CronTask for BasicTask { fn get_next_execution(&self) -> Option> { self.schedule.upcoming(Utc).next() } - fn call(&self) { - (self.func)() + fn call(&self) -> Pin + Send + '_>> { + Box::pin(async { (self.func)() }) } } diff --git a/backend/server/src/scheduler/tasks/mod.rs b/backend/server/src/scheduler/tasks/mod.rs new file mode 100644 index 0000000..6f0371b --- /dev/null +++ b/backend/server/src/scheduler/tasks/mod.rs @@ -0,0 +1,14 @@ +use std::pin::Pin; + +use chrono::{DateTime, Utc}; + +#[allow(dead_code)] +pub mod async_basic; + +#[allow(dead_code)] +pub mod basic; + +pub trait CronTask: Send + Sync { + fn get_next_execution(&self) -> Option>; + fn call(&self) -> Pin + Send + '_>>; +} diff --git a/backend/server/src/scheduler/utils.rs b/backend/server/src/scheduler/utils.rs new file mode 100644 index 0000000..996e62a --- /dev/null +++ b/backend/server/src/scheduler/utils.rs @@ -0,0 +1,32 @@ +use std::{sync::Arc, time::Duration}; + +use crate::scheduler::{error::SchedulerError, tasks::CronTask}; + +use chrono::{DateTime, Utc}; +use tokio::time::{Sleep, sleep}; + +pub fn duration_from_now(future: DateTime) -> Duration { + future + .signed_duration_since(Utc::now()) + .to_std() + .unwrap_or(Duration::ZERO) +} + +pub fn sleeper_from_now(future: DateTime) -> Sleep { + sleep(duration_from_now(future)) +} + +#[allow(unused_variables)] +pub async fn execute(name: String, func: Arc>) -> Result<(), SchedulerError> { + // todo: stats + while let Some(next_execution) = func.get_next_execution() { + let sleeper = sleeper_from_now(next_execution); + tokio::pin!(sleeper); + tokio::select! { + _ = sleeper => { + func.call().await + } + } + } + Ok(()) +} diff --git a/backend/api_server/src/utils/git.rs b/backend/server/src/utils/git.rs similarity index 71% rename from backend/api_server/src/utils/git.rs rename to backend/server/src/utils/git.rs index 6743058..471a7cc 100644 --- a/backend/api_server/src/utils/git.rs +++ b/backend/server/src/utils/git.rs @@ -1,14 +1,13 @@ -/// git merge +/// git operations /// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs /// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 /// use git2::{ - build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, + AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, build, }; use std::path::PathBuf; use tracing::{error, warn}; - #[derive(Debug, Eq, PartialEq)] pub struct FileDelta { pub old_path: Option, @@ -16,15 +15,13 @@ pub struct FileDelta { pub status: Delta, } -fn do_fetch<'a>( +pub fn do_fetch<'a>( repo: &'a Repository, refs: &[&str], remote: &'a mut Remote, ) -> Result, git2::Error> { let mut fetch_option = FetchOptions::new(); - // Always fetch all tags. - // Perform a download and also update tips fetch_option.download_tags(AutotagOption::All); remote.fetch(refs, Some(&mut fetch_option), None)?; @@ -42,16 +39,9 @@ fn fast_forward( None => String::from_utf8_lossy(lb.name_bytes()).to_string(), }; let msg = format!("fast-forward({}) -> {}", name, rc.id()); - // println!("{}", msg); lb.set_target(rc.id(), &msg)?; repo.set_head(&name)?; - repo.checkout_head(Some( - build::CheckoutBuilder::default() - // For some reason, the force is required to make the working directory actually get updated - // I suspect we should be adding some logic to handle dirty working directory states, - // but this is just an example so maybe not. - .force(), - ))?; + repo.checkout_head(Some(build::CheckoutBuilder::default().force()))?; Ok(()) } @@ -73,13 +63,12 @@ fn normal_merge( return Ok(()); } let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; - // now create the merge commit + let msg = format!("merge({}) -> {}", remote.id(), local.id()); let sig = repo.signature()?; let local_commit = repo.find_commit(local.id())?; let remote_commit = repo.find_commit(remote.id())?; - // Do our merge commit and set the current branch head to that commit. let _merge_commit = repo.commit( Some("HEAD"), &sig, @@ -88,34 +77,25 @@ fn normal_merge( &result_tree, &[&local_commit, &remote_commit], )?; - // Set working tree to match head. repo.checkout_head(None)?; Ok(()) } -fn do_merge<'a>( +pub fn do_merge<'a>( repo: &'a Repository, remote_branch: &str, fetch_commit: AnnotatedCommit<'a>, -) -> Result, git2::Error> { - // 1. do a merge analysis +) -> Result<(), git2::Error> { let analysis = repo.merge_analysis(&[&fetch_commit])?; - let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; - // 2. Do the appropriate merge if analysis.0.is_fast_forward() { - // println!("Doing a fast-forward"); - // do a fast-forward let ref_name = format!("refs/heads/{}", remote_branch); match repo.find_reference(&ref_name) { Ok(mut r) => { fast_forward(repo, &mut r, &fetch_commit)?; } Err(_) => { - // The branch doesn't exist, so set the reference to the - // commit directly. Usually this is because you are pulling - // into an empty repository. repo.reference( &ref_name, fetch_commit.id(), @@ -132,18 +112,13 @@ fn do_merge<'a>( } }; } else if analysis.0.is_normal() { - // do a normal merge normal_merge(repo, &head_commit, &fetch_commit)?; } - // else { - // // already up-to-date - // println!("Nothing to do..."); - // } - calculate_diff(repo, &head_commit, &fetch_commit) + Ok(()) } -fn calculate_diff( +pub fn calculate_diff( repo: &Repository, local: &AnnotatedCommit, remote: &AnnotatedCommit, @@ -156,23 +131,12 @@ fn calculate_diff( if let Err(e) = diff.foreach( &mut |d, _| { - // let delta = d.status(); - // - // let n_path = d.new_file().path(); - // let o_path = d.old_file().path(); - // - // if let Some(path) = n_path { - // updated_files.insert(path.to_owned()); - // } - // if let Some(path) = o_path { - // updated_files.insert(path.to_owned()); - // } updated_files.push(FileDelta { old_path: d.old_file().path().map(PathBuf::from), new_path: d.new_file().path().map(PathBuf::from), status: d.status(), }); - true // Continue + true }, None, None, @@ -184,6 +148,7 @@ fn calculate_diff( Ok(updated_files.into_iter().collect()) } +#[allow(dead_code)] pub fn sync( remote_name: &str, remote_branch: &str, @@ -191,5 +156,11 @@ pub fn sync( ) -> Result, git2::Error> { let mut remote = repo.find_remote(remote_name)?; let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; - do_merge(repo, remote_branch, fetch_commit) + let delta = calculate_diff( + repo, + &repo.reference_to_annotated_commit(&repo.head()?)?, + &fetch_commit, + ); + do_merge(repo, remote_branch, fetch_commit)?; + delta } diff --git a/backend/api_server/src/utils/mod.rs b/backend/server/src/utils/mod.rs similarity index 100% rename from backend/api_server/src/utils/mod.rs rename to backend/server/src/utils/mod.rs diff --git a/client/.gitignore b/client/.gitignore new file mode 100644 index 0000000..1c461db --- /dev/null +++ b/client/.gitignore @@ -0,0 +1,11 @@ +.venv/ + +.idea/ + +.DS_store + +__pycache__ + +.cache + +content/* diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..76fc27b --- /dev/null +++ b/client/README.md @@ -0,0 +1,30 @@ +## Module Structure + +```shell +. +├── README.md +├── cli +├── content +│ ├── pages +│ │ └── about +│ │ └── .md +│ └── posts +│ └── +│ └── +│ ├── assets +│ │ └── +│ ├── translate_.md +│ └── index.md +├── requirements.txt +├── templates +│ ├──