From 5bf6d24733726dfcb7afa08a1df393d4824e86b6 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 14 Apr 2025 15:41:02 -0700 Subject: [PATCH 01/13] feat(blog::components::check_update, README.md) add tracking filter --- backend/server/Cargo.toml | 1 + ...ost_by_i64_id.sql => get_post_by_uuid.sql} | 0 .../blog/components/check_update/pull.rs | 24 ++++++++++++++----- .../server/src/modules/blog/components/mod.rs | 4 ++-- 4 files changed, 21 insertions(+), 8 deletions(-) rename backend/server/src/modules/blog/api/get_post/{get_post_by_i64_id.sql => get_post_by_uuid.sql} (100%) diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml index f8e7c9a..0d60478 100644 --- a/backend/server/Cargo.toml +++ b/backend/server/Cargo.toml @@ -20,6 +20,7 @@ tracing-appender = "0.2.3" dirs = "6.0.0" thiserror = "2.0.11" tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros", "sync"] } +regex = "1.11.1" dotenv = "0.15.0" uuid = { version = "1.13.2", features = ["v4"] } futures = "0.3.31" diff --git a/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql b/backend/server/src/modules/blog/api/get_post/get_post_by_uuid.sql similarity index 100% rename from backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql rename to backend/server/src/modules/blog/api/get_post/get_post_by_uuid.sql diff --git a/backend/server/src/modules/blog/components/check_update/pull.rs b/backend/server/src/modules/blog/components/check_update/pull.rs index e52cdc1..53e37e9 100644 --- a/backend/server/src/modules/blog/components/check_update/pull.rs +++ b/backend/server/src/modules/blog/components/check_update/pull.rs @@ -1,3 +1,4 @@ +use std::sync::LazyLock; use std::{collections::HashMap, fs::remove_dir_all, path::PathBuf}; use crate::{ @@ -10,6 +11,7 @@ use crate::{ use markdown::MdFile; use git2::{Delta, Repository}; +use regex::Regex; use tracing::{error, warn}; use uuid::Uuid; @@ -64,14 +66,17 @@ pub async fn fetch_deltas( // process deleted deltas before merging commit to prevent "file not found" os error for delta in commit_deltas.into_iter() { - // todo: limit scanning scope to post only OR extract merging logic to external component + let relative_file_path = delta + .new_path + .clone() + .expect("cannot parse modified post `new_path`"); + + if !should_delta_tracked(&relative_file_path) { + continue; + } if delta.status == Delta::Deleted { - let file_path = &git_work_dir.join( - delta - .new_path - .expect("cannot parse modified post `new_path`"), - ); + let file_path = &git_work_dir.join(relative_file_path); if let Some(id) = extract_post_id(&MdFile::from_file(file_path)?) { changes.push(Change::Delete(Delete { uuid: id, @@ -169,3 +174,10 @@ fn merge_changes(changes: Vec) -> Vec { merged } + +static TRACKED_DELTA_RULE: LazyLock = + LazyLock::new(|| Regex::new(r"posts/\d{4}/[a-zA-Z0-9-]+/index\.md").unwrap()); + +fn should_delta_tracked(path: &PathBuf) -> bool { + TRACKED_DELTA_RULE.is_match(path.to_str().unwrap()) +} diff --git a/backend/server/src/modules/blog/components/mod.rs b/backend/server/src/modules/blog/components/mod.rs index 0f3bc78..fb2e2dd 100644 --- a/backend/server/src/modules/blog/components/mod.rs +++ b/backend/server/src/modules/blog/components/mod.rs @@ -1,3 +1,3 @@ pub mod check_update; -mod consistency_guard; -mod rag; +pub mod consistency_guard; +pub mod rag; From 15a88074e98731052c61705778e2675332207e8c Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 14 Apr 2025 15:41:31 -0700 Subject: [PATCH 02/13] feat(cli::template) update post template --- client/templates/post.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/client/templates/post.py b/client/templates/post.py index 6e116f0..d500c83 100644 --- a/client/templates/post.py +++ b/client/templates/post.py @@ -10,13 +10,14 @@ from utils.lang import eprint, wprint POST_TEMPLATE = ''' --- +uuid: {{ uuid }} title: {{ title }} subtitle: {{ subtitle }} -published_date: {{ published_date }} -last_updated: {{ last_update }} +status: {{ status }} +date: {{ date }} +update: {{ update }} category: {{ category }} tags: {{ tags }} -uuid: {{ uuid }} --- WIP @@ -38,9 +39,9 @@ def get_subtitle(): return input("subtitle > ") -def get_published_date(): +def get_date(): while True: - d = input("published_date (YYYY-MM-DD) > ") + d = input("date (YYYY-MM-DD) > ") if d == "": wprint("using today's date") return dt.today().strftime("%Y-%m-%d") @@ -78,8 +79,8 @@ class BlogPost(Template): index_md = POST_TEMPLATE.format( title=get_title(), subtitle=get_subtitle(), - published_date=get_published_date(), - last_update=dt.today().strftime("%Y-%m-%d"), + date=get_date(), + update=dt.today().strftime("%Y-%m-%d"), tags=get_tags(), category=get_category(), uuid=uuid.uuid4() From 71af26a09cee53b9d7cd9c71a33e733e9aa9ea39 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Thu, 17 Apr 2025 23:35:13 -0700 Subject: [PATCH 03/13] feat(cli) template update --- client/templates/post.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/templates/post.py b/client/templates/post.py index d500c83..0c6a0ec 100644 --- a/client/templates/post.py +++ b/client/templates/post.py @@ -10,7 +10,7 @@ from utils.lang import eprint, wprint POST_TEMPLATE = ''' --- -uuid: {{ uuid }} +id: {{ id }} title: {{ title }} subtitle: {{ subtitle }} status: {{ status }} @@ -83,6 +83,6 @@ class BlogPost(Template): update=dt.today().strftime("%Y-%m-%d"), tags=get_tags(), category=get_category(), - uuid=uuid.uuid4() + id=uuid.uuid4() ) create_all_files(self.WORK_PATH.format(year=year, identifier="test"), self.ITEMS) From e83cea1d423d2beb91f9c67668f477baf18e7e95 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Thu, 17 Apr 2025 23:40:30 -0700 Subject: [PATCH 04/13] feat(blog::components::check_update) impl consistency_guard, fix handle_create issues --- backend/server/Cargo.toml | 3 + .../blog/components/check_update/apply.rs | 24 +++- .../blog/components/check_update/changes.rs | 4 +- .../blog/components/check_update/mod.rs | 45 ++---- .../blog/components/check_update/pull.rs | 32 ++--- .../check_update/sql/create_post.sql | 7 - .../check_update/sql/create_post_general.sql | 3 + .../check_update/sql/create_post_tags.sql | 3 + .../blog/components/check_update/utils.rs | 9 ++ .../components/consistency_guard/checker.rs | 130 ++++++++++++++++++ .../components/consistency_guard/error.rs | 19 +++ .../blog/components/consistency_guard/mod.rs | 43 ++++++ .../consistency_guard/sql/fetch_post_hash.sql | 2 + .../components/consistency_guard/utils.rs | 77 +++++++++++ .../server/src/modules/blog/components/mod.rs | 1 + .../src/modules/blog/components/static_rsc.rs | 31 +++++ backend/server/src/modules/blog/mod.rs | 26 ++-- backend/server/src/modules/blog/schema.sql | 18 +-- 18 files changed, 392 insertions(+), 85 deletions(-) delete mode 100644 backend/server/src/modules/blog/components/check_update/sql/create_post.sql create mode 100644 backend/server/src/modules/blog/components/check_update/sql/create_post_general.sql create mode 100644 backend/server/src/modules/blog/components/check_update/sql/create_post_tags.sql create mode 100644 backend/server/src/modules/blog/components/consistency_guard/checker.rs create mode 100644 backend/server/src/modules/blog/components/consistency_guard/error.rs create mode 100644 backend/server/src/modules/blog/components/consistency_guard/sql/fetch_post_hash.sql create mode 100644 backend/server/src/modules/blog/components/consistency_guard/utils.rs create mode 100644 backend/server/src/modules/blog/components/static_rsc.rs diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml index 0d60478..80fa351 100644 --- a/backend/server/Cargo.toml +++ b/backend/server/Cargo.toml @@ -25,6 +25,9 @@ dotenv = "0.15.0" uuid = { version = "1.13.2", features = ["v4"] } futures = "0.3.31" +[dev-dependencies] +tempfile = "3.19.1" + [build-dependencies] git2 = "0.20.0" chrono = "0.4.39" diff --git a/backend/server/src/modules/blog/components/check_update/apply.rs b/backend/server/src/modules/blog/components/check_update/apply.rs index 1577947..808b3c0 100644 --- a/backend/server/src/modules/blog/components/check_update/apply.rs +++ b/backend/server/src/modules/blog/components/check_update/apply.rs @@ -5,10 +5,10 @@ use crate::modules::blog::components::check_update::{ }; use futures::{ - future::{join_all, BoxFuture}, FutureExt, + future::{BoxFuture, join_all}, }; -use sqlx::{query, PgPool, Postgres, Row, Transaction}; +use sqlx::{PgPool, Postgres, Row, Transaction, query}; use tracing::error; const DELETE_POST_BY_ID: &str = include_str!("sql/delete_post_by_id.sql"); @@ -16,7 +16,8 @@ const GET_TAG_ID: &str = include_str!("sql/get_tag_id.sql"); const GET_CATEGORY_ID: &str = include_str!("sql/get_category_id.sql"); const CREATE_RETURN_TAG_ID: &str = include_str!("sql/create_return_tag_id.sql"); const CREATE_RETURN_CATEGORY_ID: &str = include_str!("sql/create_return_category_id.sql"); -const CREATE_POST: &str = include_str!("sql/create_post.sql"); +const CREATE_POST_GENERAL: &str = include_str!("sql/create_post_general.sql"); +const CREATE_POST_TAGS: &str = include_str!("sql/create_post_tags.sql"); pub async fn apply_deltas(db_conn: &PgPool, changes: Vec) { let handlers = changes @@ -47,7 +48,7 @@ async fn handle_move(db: &PgPool, mv: Move) { db, Delete { uuid: mv.uuid, - path: mv.from, + path: Some(mv.from), }, ) .await; @@ -67,7 +68,7 @@ async fn handle_update(db: &PgPool, update: Update) { db, Delete { uuid: update.uuid, - path: update.path.clone(), + path: Some(update.path.clone()), }, ) .await; @@ -143,8 +144,9 @@ async fn handle_create(db: &PgPool, create: Create) { .unwrap_or(None); let post_status = meta.get("status").map(|v| v.as_str()).unwrap_or(None); let post_content = create.payload.content; + let post_hash = create.payload.hash; - if let Err(e) = query(CREATE_POST) + if let Err(e) = query(CREATE_POST_GENERAL) .bind(post_id) .bind(post_identifier) .bind(post_title) @@ -153,7 +155,17 @@ async fn handle_create(db: &PgPool, create: Create) { .bind(post_date_updated) .bind(post_status) .bind(post_content) + .bind(post_hash) .bind(category_id) + .execute(&mut *tx) + .await + { + error!("failed to insert post {}: {}", &identifier, e); + return; + } + + if let Err(e) = query(CREATE_POST_TAGS) + .bind(post_id) .bind(tag_ids) .execute(&mut *tx) .await diff --git a/backend/server/src/modules/blog/components/check_update/changes.rs b/backend/server/src/modules/blog/components/check_update/changes.rs index c0d782c..14e98d4 100644 --- a/backend/server/src/modules/blog/components/check_update/changes.rs +++ b/backend/server/src/modules/blog/components/check_update/changes.rs @@ -17,7 +17,7 @@ pub enum Change { #[derive(Debug, Clone)] pub struct Delete { pub uuid: Uuid, - pub path: PathBuf, + pub path: Option, } #[allow(dead_code)] @@ -56,7 +56,7 @@ impl CreateDelete2UpdateSlot { match (&self.create, &self.delete) { (Some(create), Some(delete)) => Some(Change::Move(Move { uuid: create.uuid, - from: delete.path.clone(), + from: delete.path.clone()?, to: create.path.clone(), payload: create.payload.clone(), })), diff --git a/backend/server/src/modules/blog/components/check_update/mod.rs b/backend/server/src/modules/blog/components/check_update/mod.rs index 9470fa3..313d76a 100644 --- a/backend/server/src/modules/blog/components/check_update/mod.rs +++ b/backend/server/src/modules/blog/components/check_update/mod.rs @@ -1,28 +1,23 @@ -use std::{ - env, - path::{Path, PathBuf}, - sync::Arc, -}; +pub mod apply; +pub mod changes; +mod error; +mod pull; +pub mod utils; + +use std::{env, path::PathBuf, sync::Arc}; use crate::{ constants::config::general::{default_value, var_name}, - modules::blog::components::check_update::{ - apply::apply_deltas, pull::fetch_deltas, utils::expand_path, + modules::blog::components::{ + check_update::{apply::apply_deltas, pull::fetch_deltas}, + static_rsc::GIT_WORK_DIR, }, scheduler::task_func::AsyncTaskFunc, }; use macros::panic_with_log; use sqlx::PgPool; -use tracing::{log::error, trace, warn, Level}; - -// mod error; -// mod index; -mod apply; -mod changes; -mod error; -mod pull; -mod utils; +use tracing::{Level, log::error, trace}; #[derive(Debug)] struct Args { @@ -34,22 +29,6 @@ struct Args { } pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc { - // init - let git_work_dir = { - let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { - expand_path(default_log_path) - } else { - let fallback_path = expand_path(default_value::WORK_DIR.to_string()); - warn!( - "unset env var `{}`, using default path: {}/blog_git", - var_name::WORK_DIR, - fallback_path - ); - fallback_path - }; - Path::new(&path).join("blog_git") - }; - // clone & check update let git_url = match env::var(var_name::BLOG_GIT_URL) { Ok(url) => url, @@ -69,7 +48,7 @@ pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc { let arc_args = Arc::new(Args { db_connection, - git_work_dir, + git_work_dir: GIT_WORK_DIR.clone(), git_url, git_remote_name, git_remote_branch, diff --git a/backend/server/src/modules/blog/components/check_update/pull.rs b/backend/server/src/modules/blog/components/check_update/pull.rs index 53e37e9..41cb26c 100644 --- a/backend/server/src/modules/blog/components/check_update/pull.rs +++ b/backend/server/src/modules/blog/components/check_update/pull.rs @@ -1,17 +1,19 @@ -use std::sync::LazyLock; -use std::{collections::HashMap, fs::remove_dir_all, path::PathBuf}; +use std::{collections::HashMap, fs::remove_dir_all, path::Path, path::PathBuf}; use crate::{ - modules::blog::components::check_update::{ - changes::{Change, Create, CreateDelete2UpdateSlot, Delete, Update}, - error::Error, + modules::blog::components::{ + check_update::{ + changes::{Change, Create, CreateDelete2UpdateSlot, Delete, Update}, + error::Error, + utils::extract_post_id, + }, + static_rsc::BLOG_POST_PATH_PATTERN, }, utils::{git, git::FileDelta}, }; use markdown::MdFile; use git2::{Delta, Repository}; -use regex::Regex; use tracing::{error, warn}; use uuid::Uuid; @@ -80,7 +82,7 @@ pub async fn fetch_deltas( if let Some(id) = extract_post_id(&MdFile::from_file(file_path)?) { changes.push(Change::Delete(Delete { uuid: id, - path: file_path.clone(), + path: Some(file_path.clone()), })) } else { error!("failed to parse uuid for file {}", file_path.display()); @@ -136,18 +138,11 @@ pub async fn fetch_deltas( } // merge adds and deletes deltas to moves - let changes = dbg!(merge_changes(changes)); + let changes = merge_changes(changes); Ok(changes) } -fn extract_post_id(post: &MdFile) -> Option { - match post.meta["uuid"].as_str() { - Some(id) => Uuid::parse_str(id).ok(), - None => None, - } -} - fn merge_changes(changes: Vec) -> Vec { let mut slots: HashMap = HashMap::new(); let mut merged: Vec = Vec::new(); @@ -175,9 +170,6 @@ fn merge_changes(changes: Vec) -> Vec { merged } -static TRACKED_DELTA_RULE: LazyLock = - LazyLock::new(|| Regex::new(r"posts/\d{4}/[a-zA-Z0-9-]+/index\.md").unwrap()); - -fn should_delta_tracked(path: &PathBuf) -> bool { - TRACKED_DELTA_RULE.is_match(path.to_str().unwrap()) +fn should_delta_tracked(path: &Path) -> bool { + BLOG_POST_PATH_PATTERN.is_match(path.to_str().unwrap()) } diff --git a/backend/server/src/modules/blog/components/check_update/sql/create_post.sql b/backend/server/src/modules/blog/components/check_update/sql/create_post.sql deleted file mode 100644 index 2cf2700..0000000 --- a/backend/server/src/modules/blog/components/check_update/sql/create_post.sql +++ /dev/null @@ -1,7 +0,0 @@ -INSERT INTO posts -(id, identifier, title, subtitle, date_created, date_updated, status, content, category) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9); - -INSERT INTO post_tag (post, tag) -SELECT $1, tag -FROM UNNEST($10) AS t(tag); diff --git a/backend/server/src/modules/blog/components/check_update/sql/create_post_general.sql b/backend/server/src/modules/blog/components/check_update/sql/create_post_general.sql new file mode 100644 index 0000000..087a568 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/create_post_general.sql @@ -0,0 +1,3 @@ +INSERT INTO posts +(id, identifier, title, subtitle, date_created, date_updated, status, content, hash, category) +VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, 'draft'), $8, $9, $10); diff --git a/backend/server/src/modules/blog/components/check_update/sql/create_post_tags.sql b/backend/server/src/modules/blog/components/check_update/sql/create_post_tags.sql new file mode 100644 index 0000000..fd266a4 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/create_post_tags.sql @@ -0,0 +1,3 @@ +INSERT INTO post_tag (post, tag) +SELECT $1, tag +FROM UNNEST($2) AS t(tag); diff --git a/backend/server/src/modules/blog/components/check_update/utils.rs b/backend/server/src/modules/blog/components/check_update/utils.rs index f3c84ec..2ecaf68 100644 --- a/backend/server/src/modules/blog/components/check_update/utils.rs +++ b/backend/server/src/modules/blog/components/check_update/utils.rs @@ -7,7 +7,9 @@ use crate::constants::{ use chrono::NaiveDate; use dirs::home_dir; +use markdown::MdFile; use tracing::warn; +use uuid::Uuid; pub fn expand_path(path: String) -> String { if path.starts_with('~') { @@ -31,6 +33,13 @@ pub fn expand_path(path: String) -> String { } } +pub fn extract_post_id(post: &MdFile) -> Option { + match post.meta["id"].as_str() { + Some(id) => Uuid::parse_str(id).ok(), + None => None, + } +} + pub fn parse_post_identifier(path: &Path) -> Option { if let Some(comp) = path.components().nth_back(1) { comp.as_os_str().to_str().map(|s| s.to_string()) diff --git a/backend/server/src/modules/blog/components/consistency_guard/checker.rs b/backend/server/src/modules/blog/components/consistency_guard/checker.rs new file mode 100644 index 0000000..ce9fabd --- /dev/null +++ b/backend/server/src/modules/blog/components/consistency_guard/checker.rs @@ -0,0 +1,130 @@ +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, + str::FromStr, +}; + +use crate::modules::blog::components::{ + check_update::{ + changes::{Change, Create, Delete, Update}, + utils::extract_post_id, + }, + consistency_guard::{error::Error, utils::rec_find_dir_entries}, + static_rsc::BLOG_POST_PATH_PATTERN, +}; +use markdown::MdFile; + +use sqlx::{FromRow, PgPool, query_as}; +use tracing::error; +use uuid::Uuid; + +const FETCH_POST_HASH: &str = include_str!("sql/fetch_post_hash.sql"); + +#[derive(Debug, FromRow)] +struct PostIdHash { + id: Uuid, + hash: String, +} + +struct PostPath { + id: Uuid, + path: PathBuf, + payload: MdFile, +} + +#[derive(Default)] +struct DeltaSlot { + local: Option, + db: Option, +} + +impl DeltaSlot { + pub fn to_change(&self) -> Option { + match (&self.local, &self.db) { + (Some(local), Some(db)) => { + if local.payload.hash == db.hash { + None + } else { + Some(Change::Update(Update { + uuid: local.id, + path: local.path.clone(), + payload: local.payload.clone(), + })) + } + } + (Some(local), None) => Some(Change::Create(Create { + uuid: local.id, + path: local.path.clone(), + payload: local.payload.clone(), + })), + (None, Some(db)) => Some(Change::Delete(Delete { + uuid: db.id, + path: None, + })), + (None, None) => None, + } + } +} + +pub async fn compare_db_local_post_hash( + db: &PgPool, + git_work_dir: &Path, +) -> Result, Error> { + let db_post_hashes = query_as::<_, PostIdHash>(FETCH_POST_HASH) + .fetch_all(db) + .await?; + let local_posts = fetch_local_posts(git_work_dir)?; + calculate_post_changes(local_posts, db_post_hashes) +} + +fn fetch_local_posts(git_work_dir: &Path) -> Result, Error> { + let mut post_hashes = vec![]; + let entries = rec_find_dir_entries(&git_work_dir.join("posts"))?; + + for entry in entries { + let relative_pth = entry.strip_prefix(git_work_dir).unwrap(); + if !BLOG_POST_PATH_PATTERN.is_match(relative_pth.to_str().unwrap()) { + continue; + } + let post_str = fs::read_to_string(&entry)?; + let post = MdFile::from_str(&post_str)?; + let post_id = match extract_post_id(&post) { + None => { + error!( + "failed to parse uuid for file {}, skipping", + &entry.display() + ); + continue; + } + Some(v) => v, + }; + post_hashes.push(PostPath { + id: post_id, + path: entry, + payload: post, + }); + } + + Ok(post_hashes) +} + +fn calculate_post_changes(local: Vec, db: Vec) -> Result, Error> { + let mut slots: HashMap = HashMap::new(); + for local_post in local { + let id = local_post.id; + slots.entry(id).or_default().local = Some(local_post); + } + for db_post in db { + let id = db_post.id; + slots.entry(id).or_default().db = Some(db_post); + } + + let mut changes = vec![]; + for slot in slots.values() { + if let Some(change) = slot.to_change() { + changes.push(change); + } + } + Ok(changes) +} diff --git a/backend/server/src/modules/blog/components/consistency_guard/error.rs b/backend/server/src/modules/blog/components/consistency_guard/error.rs new file mode 100644 index 0000000..050244f --- /dev/null +++ b/backend/server/src/modules/blog/components/consistency_guard/error.rs @@ -0,0 +1,19 @@ +use std::io; +use std::path::PathBuf; + +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("db error")] + Db(#[from] sqlx::Error), + + #[error("io error")] + Io(#[from] io::Error), + + #[error("invalid path: {0}")] + InvalidPath(PathBuf), + + #[error("invalid md file")] + MdFile(#[from] markdown::error::Error), +} diff --git a/backend/server/src/modules/blog/components/consistency_guard/mod.rs b/backend/server/src/modules/blog/components/consistency_guard/mod.rs index 8b13789..50b2db0 100644 --- a/backend/server/src/modules/blog/components/consistency_guard/mod.rs +++ b/backend/server/src/modules/blog/components/consistency_guard/mod.rs @@ -1 +1,44 @@ +mod checker; +mod error; +mod utils; +use std::{path::PathBuf, sync::Arc}; + +use crate::{ + modules::blog::components::{ + check_update::apply::apply_deltas, consistency_guard::checker::compare_db_local_post_hash, + static_rsc::GIT_WORK_DIR, + }, + scheduler::task_func::AsyncTaskFunc, +}; + +use sqlx::PgPool; +use tracing::{error, trace}; + +struct Args { + db_connection: PgPool, + git_work_dir: PathBuf, +} + +pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc { + let args = Arc::new(Args { + db_connection, + git_work_dir: GIT_WORK_DIR.clone(), + }); + move || Box::pin(workflow(args.clone())) +} + +async fn workflow(args: Arc) { + let db = &args.db_connection; + let git_work_dir = &args.git_work_dir; + + let changes = match compare_db_local_post_hash(db, git_work_dir).await { + Ok(c) => c, + Err(e) => { + error!("failed to compare db and local post list: {}", e); + return; + } + }; + apply_deltas(db, changes).await; + trace!("local-database consistency checked"); +} diff --git a/backend/server/src/modules/blog/components/consistency_guard/sql/fetch_post_hash.sql b/backend/server/src/modules/blog/components/consistency_guard/sql/fetch_post_hash.sql new file mode 100644 index 0000000..6bc1063 --- /dev/null +++ b/backend/server/src/modules/blog/components/consistency_guard/sql/fetch_post_hash.sql @@ -0,0 +1,2 @@ +SELECT id, hash +FROM posts \ No newline at end of file diff --git a/backend/server/src/modules/blog/components/consistency_guard/utils.rs b/backend/server/src/modules/blog/components/consistency_guard/utils.rs new file mode 100644 index 0000000..7db0dc7 --- /dev/null +++ b/backend/server/src/modules/blog/components/consistency_guard/utils.rs @@ -0,0 +1,77 @@ +use crate::modules::blog::components::consistency_guard::error::Error; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +pub fn rec_find_dir_entries(entry: &Path) -> Result, Error> { + if !entry.exists() { + return Err(Error::InvalidPath(entry.to_path_buf())); + } + + let mut result = vec![]; + + if !entry.is_dir() { + return Ok(vec![entry.to_path_buf()]); + } + + for entry in fs::read_dir(entry)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + result.extend(rec_find_dir_entries(&path)?); + } else { + result.push(path); + } + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::{self, File}; + use tempfile::TempDir; + + #[test] + fn test_valid_directory() { + let temp_dir = TempDir::new().unwrap(); + let dir_path = temp_dir.path(); + + fs::create_dir(dir_path.join("subdir")).unwrap(); + File::create(dir_path.join("file1.txt")).unwrap(); + File::create(dir_path.join("subdir/file2.txt")).unwrap(); + + let entries = rec_find_dir_entries(dir_path).unwrap(); + assert_eq!(entries.len(), 2); + } + + #[test] + fn test_non_existent_path() { + let non_existent = PathBuf::from("/non/existent/path"); + assert!(matches!( + rec_find_dir_entries(&non_existent), + Err(Error::InvalidPath(_)) + )); + } + + #[test] + fn test_single_file() { + let temp_dir = TempDir::new().unwrap(); + let file_path = temp_dir.path().join("test.txt"); + File::create(&file_path).unwrap(); + + let entries = rec_find_dir_entries(&file_path).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0], file_path); + } + + #[test] + fn test_empty_directory() { + let temp_dir = TempDir::new().unwrap(); + let entries = rec_find_dir_entries(temp_dir.path()).unwrap(); + assert!(entries.is_empty()); + } +} diff --git a/backend/server/src/modules/blog/components/mod.rs b/backend/server/src/modules/blog/components/mod.rs index fb2e2dd..a3e340b 100644 --- a/backend/server/src/modules/blog/components/mod.rs +++ b/backend/server/src/modules/blog/components/mod.rs @@ -1,3 +1,4 @@ pub mod check_update; pub mod consistency_guard; pub mod rag; +mod static_rsc; diff --git a/backend/server/src/modules/blog/components/static_rsc.rs b/backend/server/src/modules/blog/components/static_rsc.rs new file mode 100644 index 0000000..3f4d502 --- /dev/null +++ b/backend/server/src/modules/blog/components/static_rsc.rs @@ -0,0 +1,31 @@ +use std::{ + env, + path::{Path, PathBuf}, + sync::LazyLock, +}; + +use crate::{ + constants::config::general::{default_value, var_name}, + modules::blog::components::check_update::utils::expand_path, +}; + +use regex::Regex; +use tracing::warn; + +pub static BLOG_POST_PATH_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"^posts/\d{4}/[a-zA-Z0-9-]+/index\.md$").unwrap()); + +pub static GIT_WORK_DIR: LazyLock = LazyLock::new(|| { + let path = if let Ok(default_work_path) = env::var(var_name::WORK_DIR) { + expand_path(default_work_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") +}); diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index f536972..b782ef0 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -1,8 +1,6 @@ mod api; mod components; -use crate::modules::blog::components::check_update; -use crate::scheduler::tasks::async_basic::AsyncBasic; use crate::{ modules::{ Module, @@ -12,10 +10,10 @@ use crate::{ get_post, // get_post_list::get_post_list, }, - // cron::check_update, + components::{check_update, consistency_guard}, // cron::check_update, }, }, - scheduler::tasks::CronTask, + scheduler::tasks::{CronTask, async_basic::AsyncBasic}, }; use axum::{Router, routing::get}; @@ -45,11 +43,23 @@ impl Module for Blog { fn get_cron_tasks(&self) -> Vec<(&str, Box)> { let db_handler = self.db_handler.clone(); - vec![( - "Blog Check Update", - AsyncBasic::new(check_update::task(db_handler.clone()), "*/5 * * * * *") + vec![ + ( + "Blog Check Update", + AsyncBasic::new(check_update::task(db_handler.clone()), "*/5 * * * * *") + .unwrap() + .to_task(), + ), + ( + "Consistency Guard", + AsyncBasic::new( + consistency_guard::task(db_handler.clone()), + // "*/15 * * * * *", + "*/15 * * * * *", + ) .unwrap() .to_task(), - )] + ), + ] } } diff --git a/backend/server/src/modules/blog/schema.sql b/backend/server/src/modules/blog/schema.sql index a8d7b35..c6ffbbe 100644 --- a/backend/server/src/modules/blog/schema.sql +++ b/backend/server/src/modules/blog/schema.sql @@ -14,8 +14,7 @@ CREATE TABLE categories CREATE TABLE post_status ( - id SERIAL PRIMARY KEY, - name VARCHAR(20) UNIQUE NOT NULL + name VARCHAR(20) PRIMARY KEY ); INSERT INTO post_status (name) @@ -27,14 +26,15 @@ VALUES ('draft'), CREATE TABLE posts ( id UUID PRIMARY KEY, - identifier VARCHAR(100) UNIQUE NOT NULL, - title VARCHAR(255) NOT NULL, - subtitle VARCHAR(255) NOT NULL, - date_created DATE NOT NULL, + identifier VARCHAR(100) UNIQUE NOT NULL, + title VARCHAR(255) NOT NULL, + subtitle VARCHAR(255) NOT NULL, + date_created DATE NOT NULL, date_updated DATE, - status INTEGER REFERENCES post_status (id) DEFAULT 1 NOT NULL, - content TEXT NOT NULL, - category INTEGER REFERENCES categories (id) NOT NULL + status VARCHAR(20) REFERENCES post_status (name) NOT NULL, + content TEXT NOT NULL, + hash VARCHAR(44) NOT NULL, + category INTEGER REFERENCES categories (id) NOT NULL ); CREATE TABLE post_tag From 2c11af4a25a39c4f9622fbaa31fad511bb2d8513 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Fri, 18 Apr 2025 01:18:25 -0700 Subject: [PATCH 05/13] feat(blog::components::check_update) push frontmatter to database (metadata) --- .../blog/components/check_update/apply.rs | 25 +++++++++++++++++++ .../check_update/sql/create_post_meta.sql | 3 +++ .../components/consistency_guard/utils.rs | 3 ++- backend/server/src/modules/blog/mod.rs | 9 ++----- 4 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 backend/server/src/modules/blog/components/check_update/sql/create_post_meta.sql diff --git a/backend/server/src/modules/blog/components/check_update/apply.rs b/backend/server/src/modules/blog/components/check_update/apply.rs index 808b3c0..ffb1eb9 100644 --- a/backend/server/src/modules/blog/components/check_update/apply.rs +++ b/backend/server/src/modules/blog/components/check_update/apply.rs @@ -18,6 +18,7 @@ const CREATE_RETURN_TAG_ID: &str = include_str!("sql/create_return_tag_id.sql"); const CREATE_RETURN_CATEGORY_ID: &str = include_str!("sql/create_return_category_id.sql"); const CREATE_POST_GENERAL: &str = include_str!("sql/create_post_general.sql"); const CREATE_POST_TAGS: &str = include_str!("sql/create_post_tags.sql"); +const CREATE_POST_META: &str = include_str!("sql/create_post_meta.sql"); pub async fn apply_deltas(db_conn: &PgPool, changes: Vec) { let handlers = changes @@ -118,6 +119,19 @@ async fn handle_create(db: &PgPool, create: Create) { } }; + const KEY_TO_SKIP: &[&str] = &[ + "category", "status", "id", "date", "update", "tags", "title", "subtitle", + ]; + let (post_meta_key, post_meta_value): (Vec, Vec) = + if let Some(kv) = create.payload.meta.as_object() { + kv.iter() + .filter(|(k, _)| !KEY_TO_SKIP.contains(&&***k)) + .map(|(k, v)| (k.clone(), v.as_str().unwrap().to_string())) + .collect() + } else { + (vec![], vec![]) + }; + let meta = create.payload.meta; let post_id = create.uuid; let post_identifier = &identifier; @@ -174,6 +188,17 @@ async fn handle_create(db: &PgPool, create: Create) { return; } + if let Err(e) = query(CREATE_POST_META) + .bind(post_id) + .bind(post_meta_key) + .bind(post_meta_value) + .execute(&mut *tx) + .await + { + error!("failed to insert post {}: {}", &identifier, e); + return; + } + tx.commit().await.unwrap(); } diff --git a/backend/server/src/modules/blog/components/check_update/sql/create_post_meta.sql b/backend/server/src/modules/blog/components/check_update/sql/create_post_meta.sql new file mode 100644 index 0000000..293da0c --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/sql/create_post_meta.sql @@ -0,0 +1,3 @@ +INSERT INTO metadata (pid, meta_key, meta_value) +SELECT $1, key, value +FROM UNNEST($2, $3) AS k(key, value); diff --git a/backend/server/src/modules/blog/components/consistency_guard/utils.rs b/backend/server/src/modules/blog/components/consistency_guard/utils.rs index 7db0dc7..d8e2825 100644 --- a/backend/server/src/modules/blog/components/consistency_guard/utils.rs +++ b/backend/server/src/modules/blog/components/consistency_guard/utils.rs @@ -1,9 +1,10 @@ -use crate::modules::blog::components::consistency_guard::error::Error; use std::{ fs, path::{Path, PathBuf}, }; +use crate::modules::blog::components::consistency_guard::error::Error; + pub fn rec_find_dir_entries(entry: &Path) -> Result, Error> { if !entry.exists() { return Err(Error::InvalidPath(entry.to_path_buf())); diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index b782ef0..707de44 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -5,12 +5,8 @@ use crate::{ modules::{ Module, blog::{ - api::{ - get_page_count, - get_post, - // get_post_list::get_post_list, - }, - components::{check_update, consistency_guard}, // cron::check_update, + api::{get_page_count, get_post}, + components::{check_update, consistency_guard}, }, }, scheduler::tasks::{CronTask, async_basic::AsyncBasic}, @@ -54,7 +50,6 @@ impl Module for Blog { "Consistency Guard", AsyncBasic::new( consistency_guard::task(db_handler.clone()), - // "*/15 * * * * *", "*/15 * * * * *", ) .unwrap() From 80a164625b6765f5c22befe8ee7a1705a7625a40 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 12 May 2025 01:07:23 +0800 Subject: [PATCH 06/13] feat(blog::api) pagination --- .gitignore | 2 +- .../modules/blog/api/get_page_count/mod.rs | 25 ---- .../api/get_page_count/post_page_count.sql | 2 - .../src/modules/blog/api/get_post_list.rs | 130 ------------------ backend/server/src/modules/blog/api/mod.rs | 5 +- .../api/{get_post/mod.rs => posts/get.rs} | 3 - .../get_post_by_str_id.sql | 0 .../{get_post => posts}/get_post_by_uuid.sql | 0 .../server/src/modules/blog/api/posts/mod.rs | 2 + .../modules/blog/api/posts/pagination/mod.rs | 42 ++++++ backend/server/src/modules/blog/mod.rs | 8 +- backend/server/src/utils/axum_response.rs | 31 +++++ backend/server/src/utils/mod.rs | 1 + 13 files changed, 83 insertions(+), 168 deletions(-) delete mode 100644 backend/server/src/modules/blog/api/get_page_count/mod.rs delete mode 100644 backend/server/src/modules/blog/api/get_page_count/post_page_count.sql delete mode 100644 backend/server/src/modules/blog/api/get_post_list.rs rename backend/server/src/modules/blog/api/{get_post/mod.rs => posts/get.rs} (83%) rename backend/server/src/modules/blog/api/{get_post => posts}/get_post_by_str_id.sql (100%) rename backend/server/src/modules/blog/api/{get_post => posts}/get_post_by_uuid.sql (100%) create mode 100644 backend/server/src/modules/blog/api/posts/mod.rs create mode 100644 backend/server/src/modules/blog/api/posts/pagination/mod.rs create mode 100644 backend/server/src/utils/axum_response.rs diff --git a/.gitignore b/.gitignore index 67b7a43..e333228 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ .idea .vscode -# exclude post repo +# exclude content repo /blog_posts_git # rustc ICE dump diff --git a/backend/server/src/modules/blog/api/get_page_count/mod.rs b/backend/server/src/modules/blog/api/get_page_count/mod.rs deleted file mode 100644 index ce20576..0000000 --- a/backend/server/src/modules/blog/api/get_page_count/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::constants::blog::PAGE_SIZE; - -use axum::{Json, extract::State}; -use serde_json::{Value, json}; -use sqlx::{PgPool, Row, query}; -use tracing::error; - -const GET_PAGE_COUNT_QUERY: &str = include_str!("post_page_count.sql"); - -pub async fn handler(State(db_connection): State) -> Result, Json> { - match query(GET_PAGE_COUNT_QUERY) - .bind(PAGE_SIZE as i64) - .fetch_one(&db_connection) - .await - { - Ok(res) => { - let res: i32 = res.get("count"); - Ok(Json::from(json!({"res": res}))) - } - Err(e) => { - error!("failed to get blog page count: {}", e.to_string()); - Err(Json::from(json!({"error": e.to_string()}))) - } - } -} diff --git a/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql b/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql deleted file mode 100644 index 5ccce6b..0000000 --- a/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT cast(ceil(count(*) / cast($1 AS FLOAT)) AS INTEGER) AS count -FROM posts; diff --git a/backend/server/src/modules/blog/api/get_post_list.rs b/backend/server/src/modules/blog/api/get_post_list.rs deleted file mode 100644 index 81800aa..0000000 --- a/backend/server/src/modules/blog/api/get_post_list.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::error::Error; - -use crate::constants::blog::PAGE_SIZE; - -use axum::{extract::Query, extract::State, http::StatusCode, Json}; -use chrono::{DateTime, Utc}; -use futures::future::join_all; -use serde::Deserialize; -use serde_json::{json, Map, Value}; -use sqlx::{postgres::PgRow, query, PgPool, Row}; -use tracing::error; -use uuid::Uuid; - -#[derive(Deserialize)] -pub struct Params { - page: Option, -} - -#[axum::debug_handler] -pub async fn get_post_list( - State(db_connection): State, - Query(params): Query, -) -> Result, (StatusCode, Json)> { - let search_param_page = params.page.unwrap_or(1); - if search_param_page < 1 { - return Err(( - StatusCode::BAD_REQUEST, - Json::from(json!({"error": "`page` should be greater than 0"})), - )); - } - - let offset = PAGE_SIZE * (search_param_page - 1); - - let post_list = sqlx::query("SELECT * FROM post ORDER BY first_update DESC LIMIT $1 OFFSET $2") - .bind(PAGE_SIZE as i32) - .bind(offset as i32) - .fetch_all(&db_connection) - .await; - let post_list = match post_list { - Ok(post_list) => post_list, - Err(e) => { - error!( - "Database query error on fetching post list: {}", - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let mut res = Vec::new(); - let posts: Vec<_> = post_list - .into_iter() - .map(|post| async { analyze_post(&db_connection, post).await }) - .collect(); - let posts = join_all(posts).await; - for post in posts { - match post { - Ok(p) => res.push(p), - Err(e) => { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(("error", e.to_string()))), - )) - } - } - } - - // res.insert("posts".to_string(), Value::from(posts)); - Ok(Json::from(Value::from(res))) -} - -async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result> { - let post_id: Uuid = post.get("post_id"); - let tags = fetch_post_tags(db_connection, post_id).await; - let meta = fetch_post_meta(db_connection, post_id).await.unwrap(); - let header_img = meta.get("header_img"); - let last_update = meta.get("last_update"); - - Ok(json!({ - "post_id": post_id.to_string(), - "title": post.get::("title"), - "category": post.get::("category"), - "header_img": header_img, - "sub_title": post.get::("sub_title"), - "summary": post.get::("summary"), - "last_update": last_update, - "first_update": post.get::, _>("first_update"), - "tags": tags, - })) -} - -async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option { - match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta -FROM meta t -WHERE post_id = $1; - "#, - ) - .bind(post_id) - .fetch_one(db_connection) - .await - { - Ok(value) => Some(value.get::("meta")), - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "post meta data", - "meta", - post_id.to_string(), - e.to_string() - ); - None - } - } -} - -async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec { - match query("SELECT tag FROM Tag WHERE post_id = $1") - .bind(post_id) - .fetch_all(db_connection) - .await - { - Ok(res) => res.iter().map(|tag| tag.get::("tag")).collect(), - Err(_) => vec![], - } -} diff --git a/backend/server/src/modules/blog/api/mod.rs b/backend/server/src/modules/blog/api/mod.rs index 5eb1f92..24fd435 100644 --- a/backend/server/src/modules/blog/api/mod.rs +++ b/backend/server/src/modules/blog/api/mod.rs @@ -1,6 +1,3 @@ -pub mod get_page_count; - // todo: remove exception tag #[allow(dead_code)] -pub mod get_post; -// pub mod get_post_list; +pub mod posts; diff --git a/backend/server/src/modules/blog/api/get_post/mod.rs b/backend/server/src/modules/blog/api/posts/get.rs similarity index 83% rename from backend/server/src/modules/blog/api/get_post/mod.rs rename to backend/server/src/modules/blog/api/posts/get.rs index a2b8df7..ad2b479 100644 --- a/backend/server/src/modules/blog/api/get_post/mod.rs +++ b/backend/server/src/modules/blog/api/posts/get.rs @@ -9,9 +9,6 @@ use serde_json::json; use sqlx::PgPool; use uuid::Uuid; -const GET_POST_BY_UUID: &str = include_str!("get_post_by_uuid.sql"); -const GET_POST_BY_STR_ID: &str = include_str!("get_post_by_str_id.sql"); - pub async fn handler( State(db_connection): State, Path(identifier): Path, diff --git a/backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql b/backend/server/src/modules/blog/api/posts/get_post_by_str_id.sql similarity index 100% rename from backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql rename to backend/server/src/modules/blog/api/posts/get_post_by_str_id.sql diff --git a/backend/server/src/modules/blog/api/get_post/get_post_by_uuid.sql b/backend/server/src/modules/blog/api/posts/get_post_by_uuid.sql similarity index 100% rename from backend/server/src/modules/blog/api/get_post/get_post_by_uuid.sql rename to backend/server/src/modules/blog/api/posts/get_post_by_uuid.sql diff --git a/backend/server/src/modules/blog/api/posts/mod.rs b/backend/server/src/modules/blog/api/posts/mod.rs new file mode 100644 index 0000000..602cb8e --- /dev/null +++ b/backend/server/src/modules/blog/api/posts/mod.rs @@ -0,0 +1,2 @@ +pub mod get; +pub mod pagination; diff --git a/backend/server/src/modules/blog/api/posts/pagination/mod.rs b/backend/server/src/modules/blog/api/posts/pagination/mod.rs new file mode 100644 index 0000000..4512ef3 --- /dev/null +++ b/backend/server/src/modules/blog/api/posts/pagination/mod.rs @@ -0,0 +1,42 @@ +use crate::{ + constants::blog::PAGE_SIZE, + utils::axum_response::{err_resp, succ_resp}, +}; + +use axum::{extract::State, http::StatusCode, response::Response}; +use serde::Serialize; +use serde_json::{Value, json}; +use sqlx::{FromRow, PgPool, Row, query_as}; + +#[derive(FromRow, Serialize)] +struct DBPaginationRow { + page_count: i32, + total_count: i64, +} + +impl From for Value { + fn from(value: DBPaginationRow) -> Self { + json!({ + "page_count": value.page_count, + "total_count": value.total_count + }) + } +} + +pub async fn handler(State(db_connection): State) -> Response { + match query_as::<_, DBPaginationRow>( + r#" + SELECT + cast(ceil(count(*) / cast($1 AS FLOAT)) AS INTEGER) AS page_count, + count(*) AS total_count + FROM posts; + "#, + ) + .bind(PAGE_SIZE as i64) + .fetch_one(&db_connection) + .await + { + Ok(res) => succ_resp(StatusCode::OK, res.into()), + Err(e) => err_resp(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), + } +} diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index 707de44..cc47117 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -5,7 +5,7 @@ use crate::{ modules::{ Module, blog::{ - api::{get_page_count, get_post}, + api::{posts, posts::pagination}, components::{check_update, consistency_guard}, }, }, @@ -28,8 +28,10 @@ impl Module for Blog { // todo: remove debug comment Router::new() // .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count::handler)) - .route("/get_post/{identifier}", get(get_post::handler)) + // .route("/posts/pagination", get(...)) + // .route("/posts", get(...)) // get posts + .route("/posts/pagination", get(pagination::handler)) + .route("/posts/{identifier}", get(posts::get::handler)) .with_state(db_handler) } diff --git a/backend/server/src/utils/axum_response.rs b/backend/server/src/utils/axum_response.rs new file mode 100644 index 0000000..b2d2a9c --- /dev/null +++ b/backend/server/src/utils/axum_response.rs @@ -0,0 +1,31 @@ +use axum::{ + http::{StatusCode, header::CONTENT_TYPE}, + response::Response, +}; +use serde_json::{Value, json}; + +pub fn err_resp(code: StatusCode, err_msg: String) -> Response { + let payload = json!({ + "status": "error", + "data": { + "msg": err_msg, + } + }); + resp_json(code, payload) +} + +pub fn succ_resp(code: StatusCode, payload: Value) -> Response { + let payload = json!({ + "status": "success", + "data": payload + }); + resp_json(code, payload) +} + +pub fn resp_json(code: StatusCode, payload: Value) -> Response { + Response::builder() + .status(code) + .header(CONTENT_TYPE, "application/json") + .body(payload.to_string()) + .unwrap() +} diff --git a/backend/server/src/utils/mod.rs b/backend/server/src/utils/mod.rs index c2bf1c3..9ea0e92 100644 --- a/backend/server/src/utils/mod.rs +++ b/backend/server/src/utils/mod.rs @@ -1 +1,2 @@ +pub mod axum_response; pub mod git; From dc2c294617f19f114344d535a3a354fa5d1808f5 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 12 May 2025 11:24:22 +0800 Subject: [PATCH 07/13] feat(blog::api) update pagination api --- .../modules/blog/api/posts/pagination/mod.rs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/backend/server/src/modules/blog/api/posts/pagination/mod.rs b/backend/server/src/modules/blog/api/posts/pagination/mod.rs index 4512ef3..afed9b0 100644 --- a/backend/server/src/modules/blog/api/posts/pagination/mod.rs +++ b/backend/server/src/modules/blog/api/posts/pagination/mod.rs @@ -4,18 +4,31 @@ use crate::{ }; use axum::{extract::State, http::StatusCode, response::Response}; -use serde::Serialize; use serde_json::{Value, json}; -use sqlx::{FromRow, PgPool, Row, query_as}; +use sqlx::{FromRow, PgPool, query_as}; -#[derive(FromRow, Serialize)] +#[derive(FromRow)] struct DBPaginationRow { page_count: i32, total_count: i64, } -impl From for Value { +struct PaginationResponse { + page_count: i32, + total_count: i64, +} + +impl From for PaginationResponse { fn from(value: DBPaginationRow) -> Self { + Self { + page_count: value.page_count, + total_count: value.total_count, + } + } +} + +impl From for Value { + fn from(value: PaginationResponse) -> Self { json!({ "page_count": value.page_count, "total_count": value.total_count @@ -36,7 +49,7 @@ pub async fn handler(State(db_connection): State) -> Response { .fetch_one(&db_connection) .await { - Ok(res) => succ_resp(StatusCode::OK, res.into()), + Ok(res) => succ_resp(StatusCode::OK, PaginationResponse::from(res).into()), Err(e) => err_resp(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), } } From 2f6cafa543dfe08320ed92bd2473ef0fb59730d8 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Tue, 13 May 2025 17:09:55 +0800 Subject: [PATCH 08/13] feat(blog::api) get_identifier.rs --- .../server/src/modules/blog/api/posts/get.rs | 33 +---- .../modules/blog/api/posts/get_identifier.rs | 130 ++++++++++++++++++ .../blog/api/posts/get_post_adjacent_ids.sql | 13 ++ .../blog/api/posts/get_post_by_str_id.sql | 19 +++ .../blog/api/posts/get_post_by_uuid.sql | 20 ++- .../posts/get_post_with_adjacent_model.sql | 17 +++ .../server/src/modules/blog/api/posts/mod.rs | 1 + .../modules/blog/api/posts/pagination/mod.rs | 2 +- backend/server/src/modules/blog/mod.rs | 8 +- backend/server/src/modules/blog/schema.sql | 1 + backend/server/src/utils/axum_response.rs | 2 +- 11 files changed, 208 insertions(+), 38 deletions(-) create mode 100644 backend/server/src/modules/blog/api/posts/get_identifier.rs create mode 100644 backend/server/src/modules/blog/api/posts/get_post_adjacent_ids.sql create mode 100644 backend/server/src/modules/blog/api/posts/get_post_with_adjacent_model.sql diff --git a/backend/server/src/modules/blog/api/posts/get.rs b/backend/server/src/modules/blog/api/posts/get.rs index ad2b479..704b9eb 100644 --- a/backend/server/src/modules/blog/api/posts/get.rs +++ b/backend/server/src/modules/blog/api/posts/get.rs @@ -1,32 +1,5 @@ -use macros::dev_consume; +use axum::response::IntoResponse; -use axum::{ - Json, - extract::{Path, State}, - response::IntoResponse, -}; -use serde_json::json; -use sqlx::PgPool; -use uuid::Uuid; - -pub async fn handler( - State(db_connection): State, - Path(identifier): Path, -) -> impl IntoResponse { - if let Ok(id) = identifier.parse::() { - select_post_by_uuid(&db_connection, id).await - } else { - select_post_by_str_id(&db_connection, &identifier).await - }; - Json::from(json!({"res": "ok"})).into_response() -} - -async fn select_post_by_uuid(db: &PgPool, id: Uuid) { - dev_consume!(db, id); - todo!() -} - -async fn select_post_by_str_id(db: &PgPool, id: &str) { - dev_consume!(db, id); - todo!() +pub async fn handler() -> impl IntoResponse { + () } diff --git a/backend/server/src/modules/blog/api/posts/get_identifier.rs b/backend/server/src/modules/blog/api/posts/get_identifier.rs new file mode 100644 index 0000000..7226828 --- /dev/null +++ b/backend/server/src/modules/blog/api/posts/get_identifier.rs @@ -0,0 +1,130 @@ +use crate::utils::axum_response::{err_resp, succ_resp}; + +use axum::{ + extract::{Path, State}, + http::StatusCode, + response::IntoResponse, +}; +use chrono::NaiveDate; +use futures::future::OptionFuture; +use serde_json::Value; +use sqlx::{FromRow, PgPool, query_as}; +use uuid::Uuid; + +const SELECT_POST_BY_UUID: &str = include_str!("get_post_by_uuid.sql"); +const SELECT_POST_BY_STR_ID: &str = include_str!("get_post_by_str_id.sql"); +const SELECT_POST_ADJACENT_IDS: &str = include_str!("get_post_adjacent_ids.sql"); +const SELECT_POST_WITH_ADJACENT_MODEL: &str = include_str!("get_post_with_adjacent_model.sql"); + +#[derive(serde::Serialize, FromRow)] +struct SelectedPost { + id: Uuid, + identifier: String, + title: String, + subtitle: String, + status: String, + date_created: NaiveDate, + date_updated: Option, + category: String, + tags: Vec, + content: String, + metadata: Value, +} + +#[derive(serde::Serialize, FromRow)] +struct AdjacentPost { + id: Uuid, + identifier: String, + title: String, + subtitle: String, + date_created: NaiveDate, + category: String, + tags: Vec, + header_img: Option, +} + +#[derive(FromRow, Debug)] +struct AdjacentIds { + prev: Option, + next: Option, +} + +#[derive(serde::Serialize, Default)] +struct AdjacentPosts { + prev: Option, + next: Option, +} + +#[derive(serde::Serialize)] +struct ResponseModel { + post: SelectedPost, + adjacent: AdjacentPosts, +} + +pub async fn handler( + State(db_connection): State, + Path(identifier): Path, +) -> impl IntoResponse { + let post = match if let Ok(id) = identifier.parse::() { + select_post_by_uuid(&db_connection, id).await + } else { + select_post_by_str_id(&db_connection, &identifier).await + } { + Ok(post) => post, + Err(e) => { + return match e { + sqlx::Error::RowNotFound => err_resp(StatusCode::NOT_FOUND, "no matching post"), + _ => err_resp(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + }; + } + }; + + let adj_ids = query_as::<_, AdjacentIds>(SELECT_POST_ADJACENT_IDS) + .bind(post.id) + .fetch_one(&db_connection) + .await + .unwrap(); + let adjacent_prev = OptionFuture::from( + adj_ids + .prev + .map(|id| select_post_with_adjacent_model(&db_connection, id)), + ) + .await + .flatten(); + let adjacent_next = OptionFuture::from( + adj_ids + .next + .map(|id| select_post_with_adjacent_model(&db_connection, id)), + ) + .await + .flatten(); + let adjacent = AdjacentPosts { + prev: adjacent_prev, + next: adjacent_next, + }; + + let response = ResponseModel { post, adjacent }; + succ_resp(StatusCode::OK, serde_json::to_value(response).unwrap()) +} + +async fn select_post_by_uuid(db: &PgPool, id: Uuid) -> Result { + query_as::<_, SelectedPost>(SELECT_POST_BY_UUID) + .bind(id) + .fetch_one(db) + .await +} + +async fn select_post_by_str_id(db: &PgPool, id: &str) -> Result { + query_as::<_, SelectedPost>(SELECT_POST_BY_STR_ID) + .bind(id) + .fetch_one(db) + .await +} + +async fn select_post_with_adjacent_model(db: &PgPool, id: Uuid) -> Option { + query_as(SELECT_POST_WITH_ADJACENT_MODEL) + .bind(id) + .fetch_optional(db) + .await + .unwrap() +} diff --git a/backend/server/src/modules/blog/api/posts/get_post_adjacent_ids.sql b/backend/server/src/modules/blog/api/posts/get_post_adjacent_ids.sql new file mode 100644 index 0000000..33a0f0c --- /dev/null +++ b/backend/server/src/modules/blog/api/posts/get_post_adjacent_ids.sql @@ -0,0 +1,13 @@ +WITH target_create_date AS (SELECT p.date_created AS date FROM posts p WHERE p.id = $1) +SELECT (SELECT p.id + FROM posts p + WHERE p.date_created < t.date + ORDER BY p.date_created DESC + LIMIT 1) AS prev, + (SELECT p.id + FROM posts p + WHERE p.date_created > t.date + ORDER BY p.date_created + LIMIT 1) AS next +FROM target_create_date t + diff --git a/backend/server/src/modules/blog/api/posts/get_post_by_str_id.sql b/backend/server/src/modules/blog/api/posts/get_post_by_str_id.sql index e69de29..3b8e6a5 100644 --- a/backend/server/src/modules/blog/api/posts/get_post_by_str_id.sql +++ b/backend/server/src/modules/blog/api/posts/get_post_by_str_id.sql @@ -0,0 +1,19 @@ +SELECT p.id, + p.identifier, + p.title, + p.subtitle, + p.status, + p.date_created, + p.date_updated, + c.name AS category, + (SELECT array_agg(t.name) + FROM tags t + JOIN post_tag pt ON pt.tag = t.id + WHERE pt.post = p.id) AS tags, + p.content, + (SELECT json_object_agg(m.meta_key, m.meta_value) + FROM metadata m + WHERE m.pid = p.id) AS metadata +FROM posts p + JOIN categories c ON c.id = p.category +WHERE p.identifier = $1; diff --git a/backend/server/src/modules/blog/api/posts/get_post_by_uuid.sql b/backend/server/src/modules/blog/api/posts/get_post_by_uuid.sql index 7dc5107..19522d6 100644 --- a/backend/server/src/modules/blog/api/posts/get_post_by_uuid.sql +++ b/backend/server/src/modules/blog/api/posts/get_post_by_uuid.sql @@ -1 +1,19 @@ -SELECT \ No newline at end of file +SELECT p.id, + p.identifier, + p.title, + p.subtitle, + p.status, + p.date_created, + p.date_updated, + c.name AS category, + (SELECT array_agg(t.name) + FROM tags t + JOIN post_tag pt ON pt.tag = t.id + WHERE pt.post = p.id) AS tags, + p.content, + (SELECT json_object_agg(m.meta_key, m.meta_value) + FROM metadata m + WHERE m.pid = p.id) AS metadata +FROM posts p + JOIN categories c ON c.id = p.category +WHERE p.id = $1; diff --git a/backend/server/src/modules/blog/api/posts/get_post_with_adjacent_model.sql b/backend/server/src/modules/blog/api/posts/get_post_with_adjacent_model.sql new file mode 100644 index 0000000..366ccf0 --- /dev/null +++ b/backend/server/src/modules/blog/api/posts/get_post_with_adjacent_model.sql @@ -0,0 +1,17 @@ +SELECT p.id, + identifier, + title, + subtitle, + date_created, + c.name AS category, + (SELECT array_agg(t.name) + FROM tags t + JOIN post_tag pt ON pt.tag = t.id + WHERE pt.post = p.id) AS tags, + (SELECT m.meta_value + FROM metadata m + WHERE m.pid = p.id + AND m.meta_key = 'header_img') AS header_img +FROM posts p + JOIN public.categories c on c.id = p.category +WHERE p.id = $1 \ No newline at end of file diff --git a/backend/server/src/modules/blog/api/posts/mod.rs b/backend/server/src/modules/blog/api/posts/mod.rs index 602cb8e..41f62d5 100644 --- a/backend/server/src/modules/blog/api/posts/mod.rs +++ b/backend/server/src/modules/blog/api/posts/mod.rs @@ -1,2 +1,3 @@ pub mod get; +pub mod get_identifier; pub mod pagination; diff --git a/backend/server/src/modules/blog/api/posts/pagination/mod.rs b/backend/server/src/modules/blog/api/posts/pagination/mod.rs index afed9b0..0897bd1 100644 --- a/backend/server/src/modules/blog/api/posts/pagination/mod.rs +++ b/backend/server/src/modules/blog/api/posts/pagination/mod.rs @@ -50,6 +50,6 @@ pub async fn handler(State(db_connection): State) -> Response { .await { Ok(res) => succ_resp(StatusCode::OK, PaginationResponse::from(res).into()), - Err(e) => err_resp(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), + Err(e) => err_resp(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), } } diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index cc47117..77e4539 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -5,7 +5,7 @@ use crate::{ modules::{ Module, blog::{ - api::{posts, posts::pagination}, + api::{posts, posts::get, posts::pagination}, components::{check_update, consistency_guard}, }, }, @@ -27,11 +27,9 @@ impl Module for Blog { // todo: move resource init to new func // todo: remove debug comment Router::new() - // .route("/get_post_list", get(get_post_list)) - // .route("/posts/pagination", get(...)) - // .route("/posts", get(...)) // get posts + .route("/posts", get(get::handler)) + .route("/posts/{identifier}", get(posts::get_identifier::handler)) .route("/posts/pagination", get(pagination::handler)) - .route("/posts/{identifier}", get(posts::get::handler)) .with_state(db_handler) } diff --git a/backend/server/src/modules/blog/schema.sql b/backend/server/src/modules/blog/schema.sql index c6ffbbe..1bf1246 100644 --- a/backend/server/src/modules/blog/schema.sql +++ b/backend/server/src/modules/blog/schema.sql @@ -60,6 +60,7 @@ CREATE INDEX idx_categories_name ON categories (name); CREATE INDEX idx_posts_identifier ON posts (identifier); CREATE INDEX idx_posts_status ON posts (status); CREATE INDEX idx_posts_category ON posts (category); +CREATE INDEX idx_posts_date_created ON posts (date_created); CREATE INDEX idx_post_tag_post ON post_tag (post); CREATE INDEX idx_post_tag_tag ON post_tag (tag); CREATE INDEX idx_metadata_pid ON metadata (pid); \ No newline at end of file diff --git a/backend/server/src/utils/axum_response.rs b/backend/server/src/utils/axum_response.rs index b2d2a9c..6cf6bd8 100644 --- a/backend/server/src/utils/axum_response.rs +++ b/backend/server/src/utils/axum_response.rs @@ -4,7 +4,7 @@ use axum::{ }; use serde_json::{Value, json}; -pub fn err_resp(code: StatusCode, err_msg: String) -> Response { +pub fn err_resp(code: StatusCode, err_msg: &str) -> Response { let payload = json!({ "status": "error", "data": { From d15cdf216a99a8a4834b1b80e268e56a33720d86 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 14 May 2025 01:47:21 +0800 Subject: [PATCH 09/13] feat(blog::api) impl GET /blog/posts --- .../server/src/modules/blog/api/posts/get.rs | 111 +++++++++++++++++- .../modules/blog/api/posts/get_identifier.rs | 8 +- .../api/posts/get_post_with_brief_model.sql | 19 +++ .../modules/blog/api/posts/pagination/mod.rs | 7 +- backend/server/src/utils/axum_response.rs | 30 ++++- 5 files changed, 159 insertions(+), 16 deletions(-) create mode 100644 backend/server/src/modules/blog/api/posts/get_post_with_brief_model.sql diff --git a/backend/server/src/modules/blog/api/posts/get.rs b/backend/server/src/modules/blog/api/posts/get.rs index 704b9eb..aa3da53 100644 --- a/backend/server/src/modules/blog/api/posts/get.rs +++ b/backend/server/src/modules/blog/api/posts/get.rs @@ -1,5 +1,110 @@ -use axum::response::IntoResponse; +use crate::{ + constants::blog::PAGE_SIZE, + err_resp_log, + utils::axum_response::{err_resp, succ_resp}, +}; -pub async fn handler() -> impl IntoResponse { - () +use axum::{ + extract::{Query, State}, + http::StatusCode, + response::IntoResponse, +}; +use chrono::NaiveDate; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sqlx::{query_as, FromRow, PgPool}; +use uuid::Uuid; + +#[derive(FromRow, Serialize)] +struct PostBriefModel { + id: Uuid, + identifier: String, + title: String, + subtitle: String, + tags: Vec, + category: String, + date_created: NaiveDate, + date_updated: Option, + header_img: Option, +} + +#[derive(Deserialize, Serialize)] +pub struct Pagination { + page: Option, + page_size: Option, +} + +#[derive(Serialize)] +pub struct RespPagination { + page: u32, + page_size: u32, + page_count: u32, +} + +#[derive(Serialize)] +struct Response { + posts: Vec, + pagination: RespPagination, +} + +#[derive(FromRow)] +struct UuidRow { + id: Uuid, +} + +const GET_POST_WITH_BRIEF_MODEL: &str = include_str!("get_post_with_brief_model.sql"); + +pub async fn handler( + State(db): State, + Query(pagination): Query, +) -> impl IntoResponse { + let page = pagination.page.unwrap_or(1); + let page_size = pagination.page_size.unwrap_or(PAGE_SIZE); + let page_count = match sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM posts") + .fetch_one(&db) + .await + { + Ok(post_count) => ((post_count as f64) / (page_size as f64)).ceil() as u32, + Err(e) => return err_resp_log!(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + }; + + let pagination = RespPagination { + page, + page_size, + page_count, + }; + + if page > page_count { + return err_resp( + StatusCode::BAD_REQUEST, + "page number should not be greater than page count", + Some(json!({"pagination": pagination})), + ); + } + + let offset = page_size * (page - 1); + + let ids = match query_as::<_, UuidRow>( + "SELECT id FROM posts ORDER BY date_created DESC LIMIT $1 OFFSET $2", + ) + .bind(page_size as i32) + .bind(offset as i32) + .fetch_all(&db) + .await + { + Ok(res) => res.into_iter().map(|item| item.id).collect::>(), + Err(e) => return err_resp_log!(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + }; + + let posts = match query_as::<_, PostBriefModel>(GET_POST_WITH_BRIEF_MODEL) + .bind(&ids) + .fetch_all(&db) + .await + { + Ok(posts) => posts, + Err(e) => return err_resp_log!(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + }; + + let resp = Response { posts, pagination }; + succ_resp(StatusCode::OK, serde_json::to_value(resp).unwrap()) } diff --git a/backend/server/src/modules/blog/api/posts/get_identifier.rs b/backend/server/src/modules/blog/api/posts/get_identifier.rs index 7226828..05b103b 100644 --- a/backend/server/src/modules/blog/api/posts/get_identifier.rs +++ b/backend/server/src/modules/blog/api/posts/get_identifier.rs @@ -1,4 +1,4 @@ -use crate::utils::axum_response::{err_resp, succ_resp}; +use crate::{err_resp_log, utils::axum_response::succ_resp}; use axum::{ extract::{Path, State}, @@ -73,8 +73,10 @@ pub async fn handler( Ok(post) => post, Err(e) => { return match e { - sqlx::Error::RowNotFound => err_resp(StatusCode::NOT_FOUND, "no matching post"), - _ => err_resp(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + sqlx::Error::RowNotFound => { + err_resp_log!(StatusCode::NOT_FOUND, "no matching post") + } + _ => err_resp_log!(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), }; } }; diff --git a/backend/server/src/modules/blog/api/posts/get_post_with_brief_model.sql b/backend/server/src/modules/blog/api/posts/get_post_with_brief_model.sql new file mode 100644 index 0000000..33448ea --- /dev/null +++ b/backend/server/src/modules/blog/api/posts/get_post_with_brief_model.sql @@ -0,0 +1,19 @@ +SELECT p.id, + identifier, + title, + subtitle, + date_created, + date_updated, + c.name AS category, + (SELECT array_agg(t.name) + FROM tags t + JOIN post_tag pt ON pt.tag = t.id + WHERE pt.post = p.id) AS tags, + (SELECT m.meta_value + FROM metadata m + WHERE m.pid = p.id + AND m.meta_key = 'header_img') AS header_img +FROM posts p + JOIN public.categories c on c.id = p.category +WHERE p.id = ANY ($1) +ORDER BY p.date_created DESC; \ No newline at end of file diff --git a/backend/server/src/modules/blog/api/posts/pagination/mod.rs b/backend/server/src/modules/blog/api/posts/pagination/mod.rs index 0897bd1..6bb4d6f 100644 --- a/backend/server/src/modules/blog/api/posts/pagination/mod.rs +++ b/backend/server/src/modules/blog/api/posts/pagination/mod.rs @@ -1,7 +1,4 @@ -use crate::{ - constants::blog::PAGE_SIZE, - utils::axum_response::{err_resp, succ_resp}, -}; +use crate::{constants::blog::PAGE_SIZE, err_resp_log, utils::axum_response::succ_resp}; use axum::{extract::State, http::StatusCode, response::Response}; use serde_json::{Value, json}; @@ -50,6 +47,6 @@ pub async fn handler(State(db_connection): State) -> Response { .await { Ok(res) => succ_resp(StatusCode::OK, PaginationResponse::from(res).into()), - Err(e) => err_resp(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + Err(e) => err_resp_log!(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), } } diff --git a/backend/server/src/utils/axum_response.rs b/backend/server/src/utils/axum_response.rs index 6cf6bd8..28bbe27 100644 --- a/backend/server/src/utils/axum_response.rs +++ b/backend/server/src/utils/axum_response.rs @@ -4,13 +4,33 @@ use axum::{ }; use serde_json::{Value, json}; -pub fn err_resp(code: StatusCode, err_msg: &str) -> Response { - let payload = json!({ +#[macro_export] +macro_rules! err_resp_log { + ($code:expr, $msg:expr) => {{ + use tracing::error; + use $crate::utils::axum_response::err_resp; + + error!("{}", $msg); + err_resp($code, $msg, None) + }}; + ($code:expr, $msg:expr, $body:expr) => {{ + use tracing::error; + use $crate::utils::axum_response::err_resp; + + error!("{}", $msg); + err_resp($code, $msg, Some($body)) + }}; +} + +pub fn err_resp(code: StatusCode, err_msg: &str, body: Option) -> Response { + let mut payload = json!({ "status": "error", - "data": { - "msg": err_msg, - } + "msg": err_msg, }); + if let Some(body) = body { + payload["data"] = body; + } + resp_json(code, payload) } From f491bc99a82916c8aba27b4a7b28659e21f564ea Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 14 May 2025 14:18:03 +0800 Subject: [PATCH 10/13] fix(git) exclude .DS_Store --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index e333228..871ac16 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ .idea .vscode +# macOS system files +.DS_Store +**/.DS_Store + # exclude content repo /blog_posts_git From 06a4ab8cf7df2614b898f4480d1a43bdbe71a93a Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 14 May 2025 14:19:31 +0800 Subject: [PATCH 11/13] fix(git) exclude /client/content --- client/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/client/.gitignore b/client/.gitignore index 1c461db..3644a79 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -9,3 +9,4 @@ __pycache__ .cache content/* +/content/ From 45010f030c54869edd3936edf9ec821ddc457def Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Tue, 3 Jun 2025 03:19:49 +0800 Subject: [PATCH 12/13] feat(global) compatibility milestone --- .../server/src/modules/blog/api/posts/get.rs | 2 +- .../modules/blog/api/posts/get_identifier.rs | 2 +- .../modules/blog/api/posts/pagination/mod.rs | 12 +-- frontend/app/(pages)/blog/BlogPostCard.tsx | 45 ++++----- frontend/app/(pages)/blog/PostList.tsx | 30 +++--- frontend/app/(pages)/blog/api.ts | 21 ---- frontend/app/(pages)/blog/page.tsx | 38 ++++++-- .../app/(pages)/blog/post/[id]/Footer.tsx | 23 ++--- .../app/(pages)/blog/post/[id]/Headers.tsx | 5 - .../(pages)/blog/post/[id]/MinifyPostCard.tsx | 4 +- .../app/(pages)/blog/post/[id]/PostCard.tsx | 30 +++--- frontend/app/(pages)/blog/post/[id]/page.tsx | 15 +-- frontend/app/page.tsx | 5 +- frontend/package.json | 6 +- frontend/src/api/blog/models.ts | 0 frontend/src/api/blog/post.ts | 96 +++++++++++++++++++ frontend/src/api/endpoint.ts | 40 ++++++++ frontend/src/lang/_result.ts | 89 +++++++++++++++++ frontend/src/lang/todo.ts | 3 + 19 files changed, 347 insertions(+), 119 deletions(-) delete mode 100644 frontend/app/(pages)/blog/api.ts create mode 100644 frontend/src/api/blog/models.ts create mode 100644 frontend/src/api/blog/post.ts create mode 100644 frontend/src/api/endpoint.ts create mode 100644 frontend/src/lang/_result.ts create mode 100644 frontend/src/lang/todo.ts diff --git a/backend/server/src/modules/blog/api/posts/get.rs b/backend/server/src/modules/blog/api/posts/get.rs index aa3da53..b8c2d4a 100644 --- a/backend/server/src/modules/blog/api/posts/get.rs +++ b/backend/server/src/modules/blog/api/posts/get.rs @@ -12,7 +12,7 @@ use axum::{ use chrono::NaiveDate; use serde::{Deserialize, Serialize}; use serde_json::json; -use sqlx::{query_as, FromRow, PgPool}; +use sqlx::{FromRow, PgPool, query_as}; use uuid::Uuid; #[derive(FromRow, Serialize)] diff --git a/backend/server/src/modules/blog/api/posts/get_identifier.rs b/backend/server/src/modules/blog/api/posts/get_identifier.rs index 05b103b..e377ea7 100644 --- a/backend/server/src/modules/blog/api/posts/get_identifier.rs +++ b/backend/server/src/modules/blog/api/posts/get_identifier.rs @@ -28,7 +28,7 @@ struct SelectedPost { category: String, tags: Vec, content: String, - metadata: Value, + metadata: Option, } #[derive(serde::Serialize, FromRow)] diff --git a/backend/server/src/modules/blog/api/posts/pagination/mod.rs b/backend/server/src/modules/blog/api/posts/pagination/mod.rs index 6bb4d6f..8d9cb6a 100644 --- a/backend/server/src/modules/blog/api/posts/pagination/mod.rs +++ b/backend/server/src/modules/blog/api/posts/pagination/mod.rs @@ -7,19 +7,19 @@ use sqlx::{FromRow, PgPool, query_as}; #[derive(FromRow)] struct DBPaginationRow { page_count: i32, - total_count: i64, + post_count: i64, } struct PaginationResponse { page_count: i32, - total_count: i64, + post_count: i64, } impl From for PaginationResponse { fn from(value: DBPaginationRow) -> Self { Self { page_count: value.page_count, - total_count: value.total_count, + post_count: value.post_count, } } } @@ -28,7 +28,7 @@ impl From for Value { fn from(value: PaginationResponse) -> Self { json!({ "page_count": value.page_count, - "total_count": value.total_count + "post_count": value.post_count }) } } @@ -36,9 +36,9 @@ impl From for Value { pub async fn handler(State(db_connection): State) -> Response { match query_as::<_, DBPaginationRow>( r#" - SELECT + SELECT cast(ceil(count(*) / cast($1 AS FLOAT)) AS INTEGER) AS page_count, - count(*) AS total_count + count(*) AS post_count FROM posts; "#, ) diff --git a/frontend/app/(pages)/blog/BlogPostCard.tsx b/frontend/app/(pages)/blog/BlogPostCard.tsx index 0ca61a8..9e9449d 100644 --- a/frontend/app/(pages)/blog/BlogPostCard.tsx +++ b/frontend/app/(pages)/blog/BlogPostCard.tsx @@ -5,41 +5,42 @@ import Image from 'next/image'; import Link from 'next/link'; import { MdCalendarMonth, MdNorthEast } from 'react-icons/md'; +interface Post { + id: string, + identifier: string, + title: string, + subtitle: string, + tags: string[] + category: string, + date_created: Date, + date_updated: Date | null, + header_img: string | null +} + type Props = { - post: JSON + post: Post } function BlogPostCard({ post }: Props) { - let title = (post as any)['title']; - let sub_title = (post as any)['sub_title']; - // let summary = (post as any)['summary']; - let category = (post as any)['category']; - let tags = (post as any)['tags']; - // let last_update = new Date((post as any)['last_update']); - let first_update = new Date((post as any)['first_update']); - let post_id = (post as any)['post_id']; - let header_img = ((post as any)['header_img'] as string | null); - return ( -
- - { header_img &&
+ { post.header_img &&
{ {
-
+
{ category } -

{ title }

-

{ sub_title }

+ variant={ `solid` }>{ post.category } +

{ post.title }

+

{ post.subtitle }

@@ -63,7 +64,7 @@ function BlogPostCard({ post }: Props) {
    { - tags.map((tag: string, key: any) => ( + post.tags.map((tag: string, key: any) => (
  • { tag }
    • @@ -74,7 +75,7 @@ function BlogPostCard({ post }: Props) {

    - { first_update.toLocaleString('default', { + { post.date_created.toLocaleString('default', { month: `long`, day: `numeric`, year: `numeric` diff --git a/frontend/app/(pages)/blog/PostList.tsx b/frontend/app/(pages)/blog/PostList.tsx index c815d5c..63f7d0c 100644 --- a/frontend/app/(pages)/blog/PostList.tsx +++ b/frontend/app/(pages)/blog/PostList.tsx @@ -1,21 +1,21 @@ 'use server'; import React from 'react'; -import { fetchNocturneApi } from '@/app/(pages)/blog/api'; import BlogPostCard from '@/app/(pages)/blog/BlogPostCard'; -import { Card, CardBody } from '@nextui-org/card'; import * as motion from 'motion/react-client'; +import { fetchPostList } from '@/src/api/blog/post'; const PostList = async ({ page }: { page: number }) => { try { - const res = await fetchNocturneApi(`/blog/get_post_list?page=${ page }`); - const posts = await res.json(); + const postListResponse = await fetchPostList(page); + + const posts = postListResponse.data.posts; return ( // blog list

      - { posts.map((post: JSON, key: number) => ( + { posts.map((post, key: number) => (
    • { ); } catch (e: any) { return ( - // todo: use Error.tsx - - - Fatal: { e.toString() } - - ); + <> +
      +
      +

      + { 'Unexpected Error:' } +

      +

      + { e.message } +

      +
      +
      + + ); + } }; export default PostList; diff --git a/frontend/app/(pages)/blog/api.ts b/frontend/app/(pages)/blog/api.ts deleted file mode 100644 index 9b4c73e..0000000 --- a/frontend/app/(pages)/blog/api.ts +++ /dev/null @@ -1,21 +0,0 @@ -const LOCAL_ENDPOINT = process.env.API_LOCAL_URL; -const REMOTE_ENDPOINT = process.env.API_REMOTE_URL; - -const NOCTURNE_ENDPOINT = ((isServerSide = true) => { - if (process.env.NODE_ENV === 'production') { - if (isServerSide) { - return LOCAL_ENDPOINT; - } else { - return REMOTE_ENDPOINT; - } - } else { - return LOCAL_ENDPOINT; - } -}); - -export async function fetchNocturneApi(path: string, isServerSideRequest = true) { - return await fetch(`${ NOCTURNE_ENDPOINT(isServerSideRequest) }${ path }`, { - cache: 'no-cache' - }); -} - diff --git a/frontend/app/(pages)/blog/page.tsx b/frontend/app/(pages)/blog/page.tsx index f6b4939..ff4b6a4 100644 --- a/frontend/app/(pages)/blog/page.tsx +++ b/frontend/app/(pages)/blog/page.tsx @@ -1,6 +1,7 @@ import PostList from '@/app/(pages)/blog/PostList'; import Pagination from '@/app/(pages)/blog/Pagination'; -import { fetchNocturneApi } from '@/app/(pages)/blog/api'; +import { fetchPostListPagination } from '@/src/api/blog/post'; + async function Blog({ searchParams @@ -10,15 +11,34 @@ async function Blog({ const parsePage = Number((await searchParams)?.page); const currPage = parsePage > 0 ? parsePage : 1; - const ret = await fetchNocturneApi(`/blog/get_page_count`); - const pageCount = (await ret.json() as any)['res'] as number; - return ( -
      - - -
      - ); + + try { + const pagination = await fetchPostListPagination(); + const pageCount = pagination.data.page_count; + + return ( +
      + + +
      + ); + } catch (e: any) { + return ( +
      +
      +

      + { 'Unexpected Error [loading pagination]:' } +

      +

      + { e.message } +

      +
      +
      + ); + } + + } export default Blog; diff --git a/frontend/app/(pages)/blog/post/[id]/Footer.tsx b/frontend/app/(pages)/blog/post/[id]/Footer.tsx index e0ac2b5..f088d14 100644 --- a/frontend/app/(pages)/blog/post/[id]/Footer.tsx +++ b/frontend/app/(pages)/blog/post/[id]/Footer.tsx @@ -3,27 +3,28 @@ import MinifyPostCard from '@/app/(pages)/blog/post/[id]/MinifyPostCard'; import Link from 'next/link'; import { FaCreativeCommons } from 'react-icons/fa6'; +interface AdjacentPosts { + prev: any; + next: any; +} + type Props = { - // post: any - post_id: string; - adjacent_posts: any - perm_link: string | undefined; - author: string | undefined; - author_link: string | undefined; + adjacent_posts: AdjacentPosts + perm_link: string; + author: string; + author_link: string; first_update: string; - last_update: string | undefined; + last_update: string | null; cc: string | undefined; title: string; } const Footer = (props: Props) => { - const { adjacent_posts, perm_link, post_id, first_update, last_update, title } = props; + const { adjacent_posts, perm_link, author_link, author, first_update, last_update, title } = props; const prev_post = adjacent_posts.prev; const next_post = adjacent_posts.next; - const link = perm_link ?? `https://lumither.com/blog/post/${ post_id }`; - const author = props.author ?? 'Tao'; - const author_link = props.author_link ?? '/about/me'; + const link = perm_link; const post_date = new Date(first_update); const update_date: Date | undefined = last_update ? new Date(last_update) : undefined; const cc = props.cc ?? 'by-nc-sa'; diff --git a/frontend/app/(pages)/blog/post/[id]/Headers.tsx b/frontend/app/(pages)/blog/post/[id]/Headers.tsx index 7c83e51..a295173 100644 --- a/frontend/app/(pages)/blog/post/[id]/Headers.tsx +++ b/frontend/app/(pages)/blog/post/[id]/Headers.tsx @@ -3,14 +3,9 @@ import Image from 'next/image'; import { Chip } from '@nextui-org/react'; import { MdCalendarMonth } from 'react-icons/md'; -// type Props = { -// post: any -// }; - type Props = { title: string; subtitle: string; - summary: string | undefined; category: string; tags: string[]; first_update: string; diff --git a/frontend/app/(pages)/blog/post/[id]/MinifyPostCard.tsx b/frontend/app/(pages)/blog/post/[id]/MinifyPostCard.tsx index ac9212f..4352659 100644 --- a/frontend/app/(pages)/blog/post/[id]/MinifyPostCard.tsx +++ b/frontend/app/(pages)/blog/post/[id]/MinifyPostCard.tsx @@ -36,7 +36,7 @@ const MinifyPostCard = (props: Props) => { className="absolute transition-colors bg-zinc-300/80 dark:bg-gray-600/25 dark:backdrop-brightness-75 bottom-0 justify-between rounded-none">

      { post['title'] }

      -

      { post['sub_title'] }

      +

      { post.subtitle }

    @@ -46,7 +46,7 @@ const MinifyPostCard = (props: Props) => {

    { post['title'] }

    -

    { post['sub_title'] }

    +

    { post.subtitle }

    } diff --git a/frontend/app/(pages)/blog/post/[id]/PostCard.tsx b/frontend/app/(pages)/blog/post/[id]/PostCard.tsx index 27b0253..97f9437 100644 --- a/frontend/app/(pages)/blog/post/[id]/PostCard.tsx +++ b/frontend/app/(pages)/blog/post/[id]/PostCard.tsx @@ -3,25 +3,28 @@ import { Card, CardBody } from '@nextui-org/card'; import Headers from '@/app/(pages)/blog/post/[id]/Headers'; import Footer from '@/app/(pages)/blog/post/[id]/Footer'; import MarkdownRenderer from '@/app/(pages)/public/MarkdownRenderer'; +import { FetchPostResponseData } from '@/src/api/blog/post'; type Props = { - post: any + post: FetchPostResponseData } async function PostCard(props: Props) { - let post = props.post; + const post = props.post.post; + const metadata: any | null = post.metadata; + const adj = props.post.adjacent; + return (
    {/* card headers */ } {/* card body */ } @@ -36,14 +39,13 @@ async function PostCard(props: Props) { {/* card footer */ }