From c1ba41ea16abfe0747dd7aabfeb83761a70034a5 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 4 Sep 2024 23:18:41 -0700 Subject: [PATCH 01/13] ref(backend) start of refactor --- .env | 4 +- backend/Cargo.toml | 32 ++------- backend/api_server/Cargo.toml | 12 ++++ backend/api_server/src/init.rs | 22 ++++++ backend/api_server/src/logger.rs | 68 +++++++++++++++++++ backend/api_server/src/main.rs | 7 ++ backend/constants/Cargo.toml | 6 ++ backend/constants/src/blog.rs | 1 + backend/constants/src/lib.rs | 2 + backend/constants/src/time.rs | 1 + backend/legacy/Cargo.toml | 25 +++++++ .../src/api/get/get_page_count.rs | 0 backend/{ => legacy}/src/api/get/get_post.rs | 0 .../{ => legacy}/src/api/get/get_post_list.rs | 0 backend/{ => legacy}/src/api/get/mod.rs | 0 backend/{ => legacy}/src/api/mod.rs | 0 .../{ => legacy}/src/api/post/errors/mod.rs | 0 .../src/api/post/errors/refresh.rs | 0 backend/{ => legacy}/src/api/post/mod.rs | 0 backend/{ => legacy}/src/api/post/refresh.rs | 0 .../src/api/post/utils/build_post_idx.rs | 0 .../{ => legacy}/src/api/post/utils/mod.rs | 0 .../{ => legacy}/src/api/post/utils/post.rs | 0 backend/{ => legacy}/src/constants.rs | 0 backend/{ => legacy}/src/logger.rs | 0 backend/{ => legacy}/src/main.rs | 25 +++---- backend/{ => legacy}/src/markdown/meta.rs | 0 backend/{ => legacy}/src/markdown/mod.rs | 0 backend/{ => legacy}/src/model/blog/hash.sql | 0 backend/{ => legacy}/src/model/blog/meta.sql | 0 backend/{ => legacy}/src/model/blog/mod.rs | 0 backend/{ => legacy}/src/model/blog/post.sql | 0 backend/{ => legacy}/src/model/blog/tag.sql | 0 backend/{ => legacy}/src/model/mod.rs | 0 backend/markdown/Cargo.toml | 6 ++ backend/markdown/src/lib.rs | 14 ++++ 36 files changed, 183 insertions(+), 42 deletions(-) create mode 100644 backend/api_server/Cargo.toml create mode 100644 backend/api_server/src/init.rs create mode 100644 backend/api_server/src/logger.rs create mode 100644 backend/api_server/src/main.rs create mode 100644 backend/constants/Cargo.toml create mode 100644 backend/constants/src/blog.rs create mode 100644 backend/constants/src/lib.rs create mode 100644 backend/constants/src/time.rs create mode 100644 backend/legacy/Cargo.toml rename backend/{ => legacy}/src/api/get/get_page_count.rs (100%) rename backend/{ => legacy}/src/api/get/get_post.rs (100%) rename backend/{ => legacy}/src/api/get/get_post_list.rs (100%) rename backend/{ => legacy}/src/api/get/mod.rs (100%) rename backend/{ => legacy}/src/api/mod.rs (100%) rename backend/{ => legacy}/src/api/post/errors/mod.rs (100%) rename backend/{ => legacy}/src/api/post/errors/refresh.rs (100%) rename backend/{ => legacy}/src/api/post/mod.rs (100%) rename backend/{ => legacy}/src/api/post/refresh.rs (100%) rename backend/{ => legacy}/src/api/post/utils/build_post_idx.rs (100%) rename backend/{ => legacy}/src/api/post/utils/mod.rs (100%) rename backend/{ => legacy}/src/api/post/utils/post.rs (100%) rename backend/{ => legacy}/src/constants.rs (100%) rename backend/{ => legacy}/src/logger.rs (100%) rename backend/{ => legacy}/src/main.rs (81%) rename backend/{ => legacy}/src/markdown/meta.rs (100%) rename backend/{ => legacy}/src/markdown/mod.rs (100%) rename backend/{ => legacy}/src/model/blog/hash.sql (100%) rename backend/{ => legacy}/src/model/blog/meta.sql (100%) rename backend/{ => legacy}/src/model/blog/mod.rs (100%) rename backend/{ => legacy}/src/model/blog/post.sql (100%) rename backend/{ => legacy}/src/model/blog/tag.sql (100%) rename backend/{ => legacy}/src/model/mod.rs (100%) create mode 100644 backend/markdown/Cargo.toml create mode 100644 backend/markdown/src/lib.rs diff --git a/.env b/.env index cf1db3d..b433453 100644 --- a/.env +++ b/.env @@ -7,8 +7,10 @@ POSTGRES_DB=nocturne # backend settings GIT_WORK_DIR=../blog_posts_git -BLOG_ENDPOINT=https://github.com/Lumither/blog-posts.git LOG_ROOT_DIR=/var/log/nocturne + +BLOG_ENDPOINT=https://github.com/Lumither/blog-posts.git +WORD_DIR=~/.nocturne BACKEND_PORT=3001 # frontend settings diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5d3cbd3..1ae871b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,25 +1,7 @@ -[package] -name = "backend" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -axum = "0.7.5" -dotenv = "0.15.0" -serde = { version = "1.0.199", features = ["derive"] } -serde_json = "1.0.116" -tokio = { version = "1.37.0", features = ["full"] } -uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } -sqlx = { version = "0.7.4", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } -regex = "1.10.4" -futures = { version = "0.3.30", features = ["executor"] } -tracing = "0.1.40" -tracing-subscriber = "0.3.18" -tracing-appender = "0.2.3" -log = "0.4.21" -sha2 = "0.10.8" -thiserror = "1.0.63" -base64 = "0.22.1" +[workspace] +resolver = "2" +members = [ + "api_server", + "constants", + "markdown" +] diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml new file mode 100644 index 0000000..2543a40 --- /dev/null +++ b/backend/api_server/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "api_server" +version = "0.1.0" +edition = "2021" + +[dependencies] +constants = { path = "../constants" } +chrono = "0.4.38" +tracing = "0.1.40" +tracing-subscriber = "0.3.18" +tracing-appender = "0.2.3" +dotenv = "0.15.0" diff --git a/backend/api_server/src/init.rs b/backend/api_server/src/init.rs new file mode 100644 index 0000000..8785944 --- /dev/null +++ b/backend/api_server/src/init.rs @@ -0,0 +1,22 @@ +use crate::logger; +use std::env; +use tracing_appender::non_blocking::WorkerGuard; + +pub fn logger_init() -> (WorkerGuard, WorkerGuard) { + logger::init() +} + +pub fn load_env() { + if let Some(env_file) = env::args().nth(1) { + match dotenv::from_filename(&env_file) { + Ok(_) => { + println!("[info] env loaded from {}, starting up", env_file); + } + Err(e) => { + panic!("[fatal] failed to read {}: {}", env_file, e); + } + } + } else { + println!("[info] no .env file referred, starting up") + } +} diff --git a/backend/api_server/src/logger.rs b/backend/api_server/src/logger.rs new file mode 100644 index 0000000..1887753 --- /dev/null +++ b/backend/api_server/src/logger.rs @@ -0,0 +1,68 @@ +use std::{env, io}; + +use chrono::Local; +use tracing::info; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_appender::{non_blocking, rolling}; +use tracing_subscriber::fmt::format::Writer; +use tracing_subscriber::fmt::time::FormatTime; +use tracing_subscriber::fmt::MakeWriter; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::{fmt, Registry}; + +struct LocalTimer; + +impl FormatTime for LocalTimer { + fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { + write!( + w, + "{}", + Local::now().format(constants::time::DEFAULT_TIME_FORMAT) + ) + } +} + +type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; + +pub fn init() -> (WorkerGuard, WorkerGuard) { + let log_path = env::var("LOG_ROOT_DIR").unwrap_or_else(|e| { + panic!("[fatal] failed to load LOG_ROOT_DIR: {}", e); + }); + + let log_path = format!("{}/backend", log_path); + + let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); + + let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); + + let log_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_timer(LocalTimer); + + let subscribers = Registry::default() + .with( + fmt::Layer::default() + .with_writer(info_log_writer) + .with_ansi(false), + ) + .with( + fmt::Layer::default() + .with_writer(err_log_writer) + .with_ansi(false), + ) + .with( + fmt::Layer::default() + .with_writer(io::stdout) + .with_ansi(true) + .event_format(log_format), + ); + + tracing::subscriber::set_global_default(subscribers) + .expect("[fatal] Failed to setup logging system"); + + info!("Module Started: Logger"); + + (info_guard, error_guard) +} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs new file mode 100644 index 0000000..1db4480 --- /dev/null +++ b/backend/api_server/src/main.rs @@ -0,0 +1,7 @@ +mod init; +mod logger; + +fn main() { + init::load_env(); + let _guards = init::logger_init(); +} diff --git a/backend/constants/Cargo.toml b/backend/constants/Cargo.toml new file mode 100644 index 0000000..db345c7 --- /dev/null +++ b/backend/constants/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "constants" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/backend/constants/src/blog.rs b/backend/constants/src/blog.rs new file mode 100644 index 0000000..f0e80e6 --- /dev/null +++ b/backend/constants/src/blog.rs @@ -0,0 +1 @@ +pub const PAGE_SIZE: u32 = 6; diff --git a/backend/constants/src/lib.rs b/backend/constants/src/lib.rs new file mode 100644 index 0000000..f5cd248 --- /dev/null +++ b/backend/constants/src/lib.rs @@ -0,0 +1,2 @@ +pub mod blog; +pub mod time; diff --git a/backend/constants/src/time.rs b/backend/constants/src/time.rs new file mode 100644 index 0000000..89e7f55 --- /dev/null +++ b/backend/constants/src/time.rs @@ -0,0 +1 @@ +pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; diff --git a/backend/legacy/Cargo.toml b/backend/legacy/Cargo.toml new file mode 100644 index 0000000..5d3cbd3 --- /dev/null +++ b/backend/legacy/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "backend" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = "0.7.5" +dotenv = "0.15.0" +serde = { version = "1.0.199", features = ["derive"] } +serde_json = "1.0.116" +tokio = { version = "1.37.0", features = ["full"] } +uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } +sqlx = { version = "0.7.4", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } +chrono = { version = "0.4.38", features = ["serde", "alloc"] } +regex = "1.10.4" +futures = { version = "0.3.30", features = ["executor"] } +tracing = "0.1.40" +tracing-subscriber = "0.3.18" +tracing-appender = "0.2.3" +log = "0.4.21" +sha2 = "0.10.8" +thiserror = "1.0.63" +base64 = "0.22.1" diff --git a/backend/src/api/get/get_page_count.rs b/backend/legacy/src/api/get/get_page_count.rs similarity index 100% rename from backend/src/api/get/get_page_count.rs rename to backend/legacy/src/api/get/get_page_count.rs diff --git a/backend/src/api/get/get_post.rs b/backend/legacy/src/api/get/get_post.rs similarity index 100% rename from backend/src/api/get/get_post.rs rename to backend/legacy/src/api/get/get_post.rs diff --git a/backend/src/api/get/get_post_list.rs b/backend/legacy/src/api/get/get_post_list.rs similarity index 100% rename from backend/src/api/get/get_post_list.rs rename to backend/legacy/src/api/get/get_post_list.rs diff --git a/backend/src/api/get/mod.rs b/backend/legacy/src/api/get/mod.rs similarity index 100% rename from backend/src/api/get/mod.rs rename to backend/legacy/src/api/get/mod.rs diff --git a/backend/src/api/mod.rs b/backend/legacy/src/api/mod.rs similarity index 100% rename from backend/src/api/mod.rs rename to backend/legacy/src/api/mod.rs diff --git a/backend/src/api/post/errors/mod.rs b/backend/legacy/src/api/post/errors/mod.rs similarity index 100% rename from backend/src/api/post/errors/mod.rs rename to backend/legacy/src/api/post/errors/mod.rs diff --git a/backend/src/api/post/errors/refresh.rs b/backend/legacy/src/api/post/errors/refresh.rs similarity index 100% rename from backend/src/api/post/errors/refresh.rs rename to backend/legacy/src/api/post/errors/refresh.rs diff --git a/backend/src/api/post/mod.rs b/backend/legacy/src/api/post/mod.rs similarity index 100% rename from backend/src/api/post/mod.rs rename to backend/legacy/src/api/post/mod.rs diff --git a/backend/src/api/post/refresh.rs b/backend/legacy/src/api/post/refresh.rs similarity index 100% rename from backend/src/api/post/refresh.rs rename to backend/legacy/src/api/post/refresh.rs diff --git a/backend/src/api/post/utils/build_post_idx.rs b/backend/legacy/src/api/post/utils/build_post_idx.rs similarity index 100% rename from backend/src/api/post/utils/build_post_idx.rs rename to backend/legacy/src/api/post/utils/build_post_idx.rs diff --git a/backend/src/api/post/utils/mod.rs b/backend/legacy/src/api/post/utils/mod.rs similarity index 100% rename from backend/src/api/post/utils/mod.rs rename to backend/legacy/src/api/post/utils/mod.rs diff --git a/backend/src/api/post/utils/post.rs b/backend/legacy/src/api/post/utils/post.rs similarity index 100% rename from backend/src/api/post/utils/post.rs rename to backend/legacy/src/api/post/utils/post.rs diff --git a/backend/src/constants.rs b/backend/legacy/src/constants.rs similarity index 100% rename from backend/src/constants.rs rename to backend/legacy/src/constants.rs diff --git a/backend/src/logger.rs b/backend/legacy/src/logger.rs similarity index 100% rename from backend/src/logger.rs rename to backend/legacy/src/logger.rs diff --git a/backend/src/main.rs b/backend/legacy/src/main.rs similarity index 81% rename from backend/src/main.rs rename to backend/legacy/src/main.rs index eaf806d..7fef8f3 100644 --- a/backend/src/main.rs +++ b/backend/legacy/src/main.rs @@ -4,13 +4,12 @@ use std::num::ParseIntError; use std::process::exit; use std::str::FromStr; -use crate::api::get::get_page_count::get_page_count; -use crate::api::get::get_post::get_post; -use crate::api::get::get_post_list::get_post_list; -use crate::api::post::refresh::refresh; +use crate::api::{ + get::{get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list}, + post::refresh::refresh, +}; use axum::routing::{get, post}; use axum::Router; -use dotenv::dotenv; use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; use sqlx::ConnectOptions; use tracing::{error, info, warn}; @@ -26,21 +25,15 @@ async fn main() -> Result<(), Box> { if let Some(env_file) = env::args().nth(1) { match dotenv::from_filename(&env_file) { Ok(_) => { - println!("[Info] env loaded from {}", env_file); + println!("[Info] env loaded from {}, starting up", env_file); } Err(e) => { panic!("[Fatal] failed to read {}: {}", env_file, e); } } - }; - match dotenv() { - Ok(_) => { - println!("[Info] env loaded from .env file, starting up...") - } - Err(_) => { - println!("[Info] .env not detected, starting up...") - } - }; + } else { + println!("[info] no .env file referred, starting up") + } let _guards = logger::init(); @@ -52,7 +45,7 @@ async fn main() -> Result<(), Box> { Err(_) => 3001, }; - let db_connect_option = PgConnectOptions::from_str(dbg!(&parse_db_uri())) + let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) .unwrap() .disable_statement_logging(); diff --git a/backend/src/markdown/meta.rs b/backend/legacy/src/markdown/meta.rs similarity index 100% rename from backend/src/markdown/meta.rs rename to backend/legacy/src/markdown/meta.rs diff --git a/backend/src/markdown/mod.rs b/backend/legacy/src/markdown/mod.rs similarity index 100% rename from backend/src/markdown/mod.rs rename to backend/legacy/src/markdown/mod.rs diff --git a/backend/src/model/blog/hash.sql b/backend/legacy/src/model/blog/hash.sql similarity index 100% rename from backend/src/model/blog/hash.sql rename to backend/legacy/src/model/blog/hash.sql diff --git a/backend/src/model/blog/meta.sql b/backend/legacy/src/model/blog/meta.sql similarity index 100% rename from backend/src/model/blog/meta.sql rename to backend/legacy/src/model/blog/meta.sql diff --git a/backend/src/model/blog/mod.rs b/backend/legacy/src/model/blog/mod.rs similarity index 100% rename from backend/src/model/blog/mod.rs rename to backend/legacy/src/model/blog/mod.rs diff --git a/backend/src/model/blog/post.sql b/backend/legacy/src/model/blog/post.sql similarity index 100% rename from backend/src/model/blog/post.sql rename to backend/legacy/src/model/blog/post.sql diff --git a/backend/src/model/blog/tag.sql b/backend/legacy/src/model/blog/tag.sql similarity index 100% rename from backend/src/model/blog/tag.sql rename to backend/legacy/src/model/blog/tag.sql diff --git a/backend/src/model/mod.rs b/backend/legacy/src/model/mod.rs similarity index 100% rename from backend/src/model/mod.rs rename to backend/legacy/src/model/mod.rs diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml new file mode 100644 index 0000000..6f30a74 --- /dev/null +++ b/backend/markdown/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "markdown" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs new file mode 100644 index 0000000..b93cf3f --- /dev/null +++ b/backend/markdown/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} From 3baf011be96cf74f74cfb2fc498f8be1053fa4fe Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Sun, 8 Sep 2024 00:05:14 -0700 Subject: [PATCH 02/13] feat(backend) improve logger.rs --- .env | 7 ++-- backend/api_server/Cargo.toml | 3 +- backend/api_server/src/logger.rs | 56 ++++++++++++++++++++++++++------ 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/.env b/.env index b433453..daaad72 100644 --- a/.env +++ b/.env @@ -6,11 +6,12 @@ POSTGRES_USER= POSTGRES_DB=nocturne # backend settings -GIT_WORK_DIR=../blog_posts_git -LOG_ROOT_DIR=/var/log/nocturne + +#GIT_WORK_DIR=../blog_posts_git +#LOG_ROOT_DIR=/var/log/nocturne BLOG_ENDPOINT=https://github.com/Lumither/blog-posts.git -WORD_DIR=~/.nocturne +WORK_DIR=~/.nocturne BACKEND_PORT=3001 # frontend settings diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index 2543a40..73014cf 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" constants = { path = "../constants" } chrono = "0.4.38" tracing = "0.1.40" -tracing-subscriber = "0.3.18" +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } tracing-appender = "0.2.3" dotenv = "0.15.0" +dirs = "5.0.1" diff --git a/backend/api_server/src/logger.rs b/backend/api_server/src/logger.rs index 1887753..7721323 100644 --- a/backend/api_server/src/logger.rs +++ b/backend/api_server/src/logger.rs @@ -1,14 +1,16 @@ -use std::{env, io}; - use chrono::Local; +use dirs::home_dir; +use std::path::Path; +use std::{env, io}; use tracing::info; use tracing_appender::non_blocking::WorkerGuard; use tracing_appender::{non_blocking, rolling}; +use tracing_subscriber::filter::LevelFilter; use tracing_subscriber::fmt::format::Writer; use tracing_subscriber::fmt::time::FormatTime; use tracing_subscriber::fmt::MakeWriter; use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::{fmt, Registry}; +use tracing_subscriber::{fmt, Layer, Registry}; struct LocalTimer; @@ -25,44 +27,78 @@ impl FormatTime for LocalTimer { type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; pub fn init() -> (WorkerGuard, WorkerGuard) { - let log_path = env::var("LOG_ROOT_DIR").unwrap_or_else(|e| { - panic!("[fatal] failed to load LOG_ROOT_DIR: {}", e); + let log_path = &env::var("LOG_ROOT_DIR").unwrap_or_else(|e| { + if let Ok(default_log_path) = env::var("WORK_DIR") { + if default_log_path.starts_with('~') { + let home_path = home_dir() + .expect("[fatal] unable to locate home dir with parsing env `WORK_DIR`, try to use absolute path instead"); + home_path + .join(default_log_path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .expect("[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead") + } else { + default_log_path + } + } else { + panic!("[fatal] failed to parse env `LOG_ROOT_DIR`: {}", e); + } }); - let log_path = format!("{}/backend", log_path); + let log_path = Path::new(log_path) + .join("log") + .join("backend") + .into_os_string() + .into_string() + .unwrap_or_else(|e| panic!("[fatal] failed to parse {:?}", e)); let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); - let log_format = tracing_subscriber::fmt::format() + let info_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let err_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let stdout_format = tracing_subscriber::fmt::format() .with_level(true) .with_target(true) .with_source_location(true) .with_timer(LocalTimer); + // let default_filter = tracing_subscriber::filter::EnvFilter::try_from_default_env() + // .unwrap_or_else(|_| "tower_http=debug,axum::rejection=trace".into()); + let subscribers = Registry::default() .with( fmt::Layer::default() .with_writer(info_log_writer) - .with_ansi(false), + .event_format(info_format), ) .with( fmt::Layer::default() .with_writer(err_log_writer) - .with_ansi(false), + .event_format(err_format) + .with_filter(LevelFilter::ERROR), ) .with( fmt::Layer::default() .with_writer(io::stdout) .with_ansi(true) - .event_format(log_format), + .event_format(stdout_format), ); tracing::subscriber::set_global_default(subscribers) .expect("[fatal] Failed to setup logging system"); info!("Module Started: Logger"); + info!("Logging to file {}", log_path); (info_guard, error_guard) } From 3dab27323a178464a32da92408e1b6f2f92025bc Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Tue, 29 Oct 2024 18:37:14 -0700 Subject: [PATCH 03/13] ref, feat(scheduler) global project refactor, add scheduler --- .env | 2 - .gitignore | 3 +- backend/Cargo.toml | 3 +- backend/api_server/Cargo.toml | 11 +- .../src/blog/api}/get_page_count.rs | 3 +- .../src/blog/api}/get_post.rs | 5 +- .../src/blog/api}/get_post_list.rs | 3 +- .../get => api_server/src/blog/api}/mod.rs | 1 + backend/api_server/src/blog/api/refresh.rs | 190 ++++++++++++ backend/api_server/src/blog/mod.rs | 19 ++ backend/api_server/src/blog/scheduler/mod.rs | 184 +++++++++++ .../src/blog/scheduler/scheduler_error.rs | 44 +++ .../api_server/src/blog/scheduler/sleeper.rs | 77 +++++ .../src/blog/scheduler/task_func.rs | 3 + .../src/blog/scheduler/tasks/basic.rs | 30 ++ .../src/blog/scheduler/tasks/mod.rs | 9 + backend/api_server/src/init.rs | 4 +- backend/api_server/src/logger.rs | 24 +- backend/api_server/src/main.rs | 7 +- .../src/main.rs => api_server/src/server.rs} | 54 +--- backend/legacy/Cargo.toml | 25 -- backend/legacy/src/api/mod.rs | 2 - backend/legacy/src/api/post/errors/mod.rs | 1 - backend/legacy/src/api/post/errors/refresh.rs | 61 ---- backend/legacy/src/api/post/mod.rs | 3 - backend/legacy/src/api/post/refresh.rs | 192 ------------ .../src/api/post/utils/build_post_idx.rs | 288 ------------------ backend/legacy/src/api/post/utils/mod.rs | 2 - backend/legacy/src/api/post/utils/post.rs | 120 -------- backend/legacy/src/constants.rs | 3 - backend/legacy/src/logger.rs | 67 ---- backend/legacy/src/markdown/meta.rs | 55 ---- backend/legacy/src/markdown/mod.rs | 1 - backend/legacy/src/model/blog/hash.sql | 5 - backend/legacy/src/model/blog/meta.sql | 79 ----- backend/legacy/src/model/blog/mod.rs | 38 --- backend/legacy/src/model/blog/post.sql | 12 - backend/legacy/src/model/blog/tag.sql | 5 - backend/legacy/src/model/mod.rs | 1 - backend/macros/Cargo.toml | 6 + backend/macros/src/lib.rs | 14 + backend/markdown/Cargo.toml | 8 + backend/markdown/src/lib.rs | 59 +++- backend/markdown/src/utils/front_matter.rs | 77 +++++ backend/markdown/src/utils/mod.rs | 2 + backend/markdown/src/utils/yaml.rs | 29 ++ frontend/app/layout.tsx | 2 +- 47 files changed, 796 insertions(+), 1037 deletions(-) rename backend/{legacy/src/api/get => api_server/src/blog/api}/get_page_count.rs (95%) rename backend/{legacy/src/api/get => api_server/src/blog/api}/get_post.rs (99%) rename backend/{legacy/src/api/get => api_server/src/blog/api}/get_post_list.rs (98%) rename backend/{legacy/src/api/get => api_server/src/blog/api}/mod.rs (79%) create mode 100644 backend/api_server/src/blog/api/refresh.rs create mode 100644 backend/api_server/src/blog/mod.rs create mode 100644 backend/api_server/src/blog/scheduler/mod.rs create mode 100644 backend/api_server/src/blog/scheduler/scheduler_error.rs create mode 100644 backend/api_server/src/blog/scheduler/sleeper.rs create mode 100644 backend/api_server/src/blog/scheduler/task_func.rs create mode 100644 backend/api_server/src/blog/scheduler/tasks/basic.rs create mode 100644 backend/api_server/src/blog/scheduler/tasks/mod.rs rename backend/{legacy/src/main.rs => api_server/src/server.rs} (52%) delete mode 100644 backend/legacy/Cargo.toml delete mode 100644 backend/legacy/src/api/mod.rs delete mode 100644 backend/legacy/src/api/post/errors/mod.rs delete mode 100644 backend/legacy/src/api/post/errors/refresh.rs delete mode 100644 backend/legacy/src/api/post/mod.rs delete mode 100644 backend/legacy/src/api/post/refresh.rs delete mode 100644 backend/legacy/src/api/post/utils/build_post_idx.rs delete mode 100644 backend/legacy/src/api/post/utils/mod.rs delete mode 100644 backend/legacy/src/api/post/utils/post.rs delete mode 100644 backend/legacy/src/constants.rs delete mode 100644 backend/legacy/src/logger.rs delete mode 100644 backend/legacy/src/markdown/meta.rs delete mode 100644 backend/legacy/src/markdown/mod.rs delete mode 100644 backend/legacy/src/model/blog/hash.sql delete mode 100644 backend/legacy/src/model/blog/meta.sql delete mode 100644 backend/legacy/src/model/blog/mod.rs delete mode 100644 backend/legacy/src/model/blog/post.sql delete mode 100644 backend/legacy/src/model/blog/tag.sql delete mode 100644 backend/legacy/src/model/mod.rs create mode 100644 backend/macros/Cargo.toml create mode 100644 backend/macros/src/lib.rs create mode 100644 backend/markdown/src/utils/front_matter.rs create mode 100644 backend/markdown/src/utils/mod.rs create mode 100644 backend/markdown/src/utils/yaml.rs diff --git a/.env b/.env index daaad72..3708ad5 100644 --- a/.env +++ b/.env @@ -17,5 +17,3 @@ BACKEND_PORT=3001 # frontend settings API_LOCAL_URL=http://localhost:3001 API_REMOTE_URL=https://lumither.com/api - - diff --git a/.gitignore b/.gitignore index 6299755..3694827 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ .env.* /container/*/.env -# idea related +# IDE related .idea +.vscode # exclude post repo /blog_posts_git diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 1ae871b..6c75b0a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -3,5 +3,6 @@ resolver = "2" members = [ "api_server", "constants", - "markdown" + "macros", + "markdown", ] diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index 73014cf..bb2012c 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -5,9 +5,18 @@ edition = "2021" [dependencies] constants = { path = "../constants" } -chrono = "0.4.38" +chrono = { version = "0.4.38", features = ["serde", "alloc"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } tracing-appender = "0.2.3" dotenv = "0.15.0" dirs = "5.0.1" +tokio = { version = "1.40.0", features = ["full"] } +sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } +axum = "0.7.5" +serde = { version = "1.0.199", features = ["derive"] } +serde_json = "1.0.128" +uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } +futures = { version = "0.3.30", features = ["executor"] } +sha2 = "0.10.8" +cron = "0.12.1" \ No newline at end of file diff --git a/backend/legacy/src/api/get/get_page_count.rs b/backend/api_server/src/blog/api/get_page_count.rs similarity index 95% rename from backend/legacy/src/api/get/get_page_count.rs rename to backend/api_server/src/blog/api/get_page_count.rs index 887e360..db540af 100644 --- a/backend/legacy/src/api/get/get_page_count.rs +++ b/backend/api_server/src/blog/api/get_page_count.rs @@ -1,4 +1,5 @@ -use crate::constants::PAGE_SIZE; +use constants::blog::PAGE_SIZE; + use axum::extract::State; use axum::Json; use serde_json::{json, Value}; diff --git a/backend/legacy/src/api/get/get_post.rs b/backend/api_server/src/blog/api/get_post.rs similarity index 99% rename from backend/legacy/src/api/get/get_post.rs rename to backend/api_server/src/blog/api/get_post.rs index a38eb89..2374dec 100644 --- a/backend/legacy/src/api/get/get_post.rs +++ b/backend/api_server/src/blog/api/get_post.rs @@ -1,12 +1,11 @@ -use std::str::FromStr; - use axum::{ extract::{Path, State}, http::StatusCode, Json, }; use serde_json::{json, Value}; -use sqlx::{Error, PgPool, query, Row}; +use sqlx::{query, Error, PgPool, Row}; +use std::str::FromStr; use tracing::error; use uuid::Uuid; diff --git a/backend/legacy/src/api/get/get_post_list.rs b/backend/api_server/src/blog/api/get_post_list.rs similarity index 98% rename from backend/legacy/src/api/get/get_post_list.rs rename to backend/api_server/src/blog/api/get_post_list.rs index 1b55610..854f495 100644 --- a/backend/legacy/src/api/get/get_post_list.rs +++ b/backend/api_server/src/blog/api/get_post_list.rs @@ -1,4 +1,5 @@ -use crate::constants::PAGE_SIZE; +use constants::blog::PAGE_SIZE; + use axum::{extract::Query, extract::State, http::StatusCode, Json}; use chrono::{DateTime, Utc}; use futures::future::join_all; diff --git a/backend/legacy/src/api/get/mod.rs b/backend/api_server/src/blog/api/mod.rs similarity index 79% rename from backend/legacy/src/api/get/mod.rs rename to backend/api_server/src/blog/api/mod.rs index 5e23c3d..e946d12 100644 --- a/backend/legacy/src/api/get/mod.rs +++ b/backend/api_server/src/blog/api/mod.rs @@ -1,3 +1,4 @@ pub mod get_page_count; pub mod get_post; pub mod get_post_list; +pub mod refresh; diff --git a/backend/api_server/src/blog/api/refresh.rs b/backend/api_server/src/blog/api/refresh.rs new file mode 100644 index 0000000..cc7b331 --- /dev/null +++ b/backend/api_server/src/blog/api/refresh.rs @@ -0,0 +1,190 @@ +use std::path::Path; +use std::str::FromStr; + +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::Json; +use serde::Deserialize; +use serde_json::{Map, Value}; +use sqlx::PgPool; + +// use crate::api::post::errors::refresh::PostIdxError; +// use crate::api::post::utils::{build_post_idx, post}; +// use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL}; + +#[derive(Deserialize)] +pub struct Params { + /// refresh level + level: Option, +} + +pub async fn refresh( + State(db_connection): State, + Query(params): Query, +) -> Result)> { + // let is_force_refresh = params.level.unwrap_or("".to_string()) == "force"; + // + // // search post + // let data_repo_dir = match env::var("GIT_WORK_DIR") { + // Ok(value) => value, + // Err(_) => { + // return Err(( + // StatusCode::INTERNAL_SERVER_ERROR, + // Json::from(json!(( + // "error", + // "error at parsing environment variable: GIT_WORK_DIR" + // ))), + // )); + // } + // }; + // let post_dir = Path::new(&data_repo_dir).join("posts"); + // + // if is_force_refresh { + // if let Err(e) = query("DROP TABLE IF EXISTS hash, meta, post, tag;") + // .execute(&db_connection) + // .await + // { + // error!( + // "DB query error when deleting table: `hash`, `meta`, `post`, `tag`: {}", + // e.to_string() + // ); + // return Err(( + // StatusCode::INTERNAL_SERVER_ERROR, + // Json::from(json!(("error", e.to_string()))), + // )); + // } + // } + // + // // generate post idx + // let post_path = search_md(&post_dir); + // + // let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = post_path + // .into_iter() + // .filter_map(|post_path| match post::from_path(&post_path) { + // Ok(post) => match fs::read(&post_path) { + // Ok(bytes) => { + // let mut hasher = Sha256::new(); + // hasher.update(bytes); + // let hash = hasher.finalize().to_vec(); + // Some((post, hash)) + // } + // Err(e) => { + // error!( + // "Failed to read file while hashing the post <{}>, skipping: {}", + // post_path, + // e.to_string() + // ); + // None + // } + // }, + // Err(e) => { + // error!( + // "Failed to parse post <{}>, skipping: {}", + // post_path, + // e.to_string() + // ); + // None + // } + // }) + // .collect(); + // + // // db create + // query(POST_BASE_MODEL) + // .execute(&db_connection) + // .await + // .unwrap(); + // query(POST_META_MODEL) + // .execute(&db_connection) + // .await + // .unwrap(); + // query(TAG_MODEL).execute(&db_connection).await.unwrap(); + // query(POST_HASH_MODEL) + // .execute(&db_connection) + // .await + // .unwrap(); + // + // let res: Vec<_> = stream::iter(post_list_with_hash) + // .map(|(post, hash)| { + // let db_clone = db_connection.clone(); + // async move { build_post_idx(&db_clone, post, hash).await } + // }) + // .buffer_unordered(usize::MAX) + // .collect() + // .await; + // + // let errs: Vec<_> = res + // .into_iter() + // .filter_map(|res| res.err()) + // .map(|e| e.to_string()) + // .collect(); + // + // if !errs.is_empty() { + // Err(( + // StatusCode::INTERNAL_SERVER_ERROR, + // Json::from(json!({"error": errs})), + // )) + // } else { + // Ok(StatusCode::OK) + // } + todo!() +} + +/// Search a dir, return a list of markdown files +/// +/// # Arguments +/// * `entry`: dir path +/// +/// # Returns +/// Vec +/// * `String`: path of a markdown file +/// +fn search_md(entry: &Path) -> Vec { + if entry.ends_with(".git") { + return vec![]; + } + if entry.is_dir() { + if let Ok(files) = entry.read_dir() { + files + .flat_map(|file| { + let file_path = file.unwrap().path(); + search_md(file_path.as_path()) + }) + .collect() + } else { + vec![] + } + } else if entry.extension().is_some_and(|ext| ext == "md") { + vec![entry.to_str().unwrap().to_string()] + } else { + vec![] + } +} + +async fn build_post_idx( + db_connection: &PgPool, + post: Map, + hash: Vec, + // ) -> Result<(), PostIdxError> { +) -> Result<(), ()> { + todo!() + // let meta = &post["meta"]; + // + // // extract post id + // let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { + // Ok(id) => id, + // Err(e) => { + // return Err(PostIdxError::InvalidUUID { + // id: post["post_id"].as_str().unwrap().to_string(), + // err_msg: e.to_string(), + // }) + // } + // }; + // + // if build_post_idx::hash(db_connection, post_id, hash).await? { + // build_post_idx::base(db_connection, &post, post_id, meta).await?; + // build_post_idx::tag(db_connection, post_id, meta).await?; + // build_post_idx::meta(db_connection, post_id, meta).await?; + // } + // + // Ok(()) +} diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs new file mode 100644 index 0000000..abfb8e6 --- /dev/null +++ b/backend/api_server/src/blog/mod.rs @@ -0,0 +1,19 @@ +mod api; +mod scheduler; + +use crate::blog::api::{ + get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, +}; + +use crate::blog::api::refresh::refresh; +use axum::routing::{get, post}; +use axum::Router; +use sqlx::{Pool, Postgres}; + +pub fn get_router() -> Router> { + Router::new() + .route("/refresh_posts", post(refresh)) + .route("/get_post_list", get(get_post_list)) + .route("/get_page_count", get(get_page_count)) + .route("/get_post/:post_id", get(get_post)) +} diff --git a/backend/api_server/src/blog/scheduler/mod.rs b/backend/api_server/src/blog/scheduler/mod.rs new file mode 100644 index 0000000..ef73764 --- /dev/null +++ b/backend/api_server/src/blog/scheduler/mod.rs @@ -0,0 +1,184 @@ +use crate::blog::{ + scheduler::scheduler_error::SchedulerError, + scheduler::scheduler_error::SchedulerError::IdAlreadyRunning, + scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper, + scheduler::tasks::Task, +}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::RwLock; +use std::{collections::HashMap, sync::Arc, thread}; +use uuid::Uuid; + +pub mod scheduler_error; +mod sleeper; +mod task_func; +pub mod tasks; + +struct TaskStatus { + sleeper: Option>, + is_alive: AtomicBool, +} + +#[derive(Default)] +pub struct Scheduler { + tasks: Arc>>>>, + running: Arc>>>, + stopped: Arc>>, +} + +impl Scheduler { + /// Start all tasks in the scheduler + pub fn start(&self) -> Result<(), SchedulerError> { + for (k, _v) in self.tasks.read()?.iter() { + self.run_non_blocking(k)? + } + Ok(()) + } + + /// Terminate all tasks in the scheduler + pub fn halt() {} + + fn run_task( + running: Arc>>>, + stopped: Arc>>, + id: Uuid, + task: Arc>, + ) -> Result<(), SchedulerError> { + if running.read()?.get(&id).is_some() { + return Err(IdAlreadyRunning); + } else { + running.write()?.insert( + id, + TaskStatus { + sleeper: None, + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + loop { + if let Some(next_execution) = task.get_next_execution() { + // guarantee is alive + let task_status = { + let running_reading_guard = running.read()?; + running_reading_guard.get(&id).cloned().clone() + }; + if let Some(task_status) = task_status { + if !&task_status.is_alive.load(Ordering::SeqCst) { + { + let mut running_lock_guard = running.write()?; + running_lock_guard.remove(&id); + } + stopped.write()?.push(id); + return Ok(()); + } else { + let sleeper = Arc::from(SkippableSleeper::from(next_execution)); + { + let mut running_lock_guard = running.write()?; + running_lock_guard.insert( + id, + TaskStatus { + sleeper: Some(sleeper.clone()), + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + sleeper.start(); + task.call(); + } + } + } else { + stopped.write()?.push(id); + break; + } + } + Ok(()) + } + + /// Run a task with `id` + pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + Self::run_task(self.running.clone(), self.stopped.clone(), id, task.clone()) + } else { + Err(IdNotFound) + } + } + + pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + let running_clone = self.running.clone(); + let stopped_clone = self.stopped.clone(); + let task_clone = task.clone(); + thread::spawn(move || Self::run_task(running_clone, stopped_clone, id, task_clone)); + Ok(()) + } else { + Err(IdNotFound) + } + } + + fn get_running(&self, id: &Uuid) -> Option> { + let ret = self.running.read().unwrap().get(id).cloned(); + ret + } + + /// Stop a task with `id` + pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + status.is_alive.store(false, Ordering::SeqCst); + if let Some(s) = &status.sleeper { + s.skip(); + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + if let Some(s) = &status.sleeper { + s.skip() + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn insert(&self, task: Box) -> Result { + let id = Uuid::new_v4(); + self.tasks.write()?.insert(id, Arc::new(task)); + Ok(id) + } +} + +#[cfg(test)] +mod tests { + use crate::blog::scheduler::tasks::basic::BasicTask; + use crate::blog::scheduler::Scheduler; + use std::error::Error; + use std::thread; + use std::time::Duration; + + #[test] + fn test_run() -> Result<(), Box> { + let scheduler = Scheduler::default(); + let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; + let id = scheduler.insert(Box::new(task))?; + scheduler.run_non_blocking(&id)?; + thread::sleep(Duration::from_secs(4)); + scheduler.stop(&id)?; + // test termination + thread::sleep(Duration::from_secs(5)); + Ok(()) + } +} diff --git a/backend/api_server/src/blog/scheduler/scheduler_error.rs b/backend/api_server/src/blog/scheduler/scheduler_error.rs new file mode 100644 index 0000000..54615be --- /dev/null +++ b/backend/api_server/src/blog/scheduler/scheduler_error.rs @@ -0,0 +1,44 @@ +use crate::blog::scheduler::scheduler_error::SchedulerError::MutexPoison; +use std::error::Error; +use std::fmt::{Debug, Display, Formatter}; +use std::sync::PoisonError; + +pub enum SchedulerError { + IdNotFound, + IdAlreadyRunning, + MutexPoison(String), +} + +impl From> for SchedulerError { + fn from(value: PoisonError) -> Self { + MutexPoison(value.to_string()) + } +} + +impl Debug for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) + } +} + +impl Display for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + SchedulerError::IdNotFound => { + "SchedulerError::IdNotFound".to_string() + } + SchedulerError::IdAlreadyRunning => { + "SchedulerError::IdAlreadyRunning".to_string() + } + MutexPoison(str) => { + format!("SchedulerError::MutexPoison: {}", str) + } + } + ) + } +} + +impl Error for SchedulerError {} diff --git a/backend/api_server/src/blog/scheduler/sleeper.rs b/backend/api_server/src/blog/scheduler/sleeper.rs new file mode 100644 index 0000000..dd84bd6 --- /dev/null +++ b/backend/api_server/src/blog/scheduler/sleeper.rs @@ -0,0 +1,77 @@ +use chrono::{DateTime, Utc}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; +use std::time::Duration; + +#[derive(Clone, Debug, Default)] +pub struct SkippableSleeper { + pub duration: Duration, + state: Arc<(Mutex, Condvar)>, +} + +impl SkippableSleeper { + pub fn new(duration: Duration) -> Self { + SkippableSleeper { + duration, + state: Arc::new((Mutex::new(false), Condvar::new())), + } + } + + // Blocks until the duration has passed or skip() is called + pub fn start(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + let state_clone = Arc::clone(&self.state); + let duration_clone = self.duration; + thread::spawn(move || { + thread::sleep(duration_clone); + let (lock, cvar) = &*state_clone; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + }); + while !*received { + received = cvar.wait(received).unwrap(); + } + } + + // Skip the blocking delay + pub fn skip(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + } +} + +impl From> for SkippableSleeper { + fn from(value: DateTime) -> Self { + let from_utc_timestamp = value.timestamp() as u64; + let curr_utc_timestamp = Utc::now().timestamp() as u64; + SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) + } +} + +#[cfg(test)] +mod test { + use crate::blog::scheduler::sleeper::SkippableSleeper; + use std::thread; + use std::time::Duration; + + #[test] + fn usage() { + let sleeper = SkippableSleeper::new(Duration::new(10, 0)); + thread::spawn({ + let sleeper = sleeper.clone(); + move || { + println!("Sleeping..."); + sleeper.start(); + println!("Woke up!"); + } + }); + println!("Start Sleeping..."); + thread::sleep(Duration::from_secs(4)); + println!("Skipping sleep..."); + sleeper.skip(); + } +} diff --git a/backend/api_server/src/blog/scheduler/task_func.rs b/backend/api_server/src/blog/scheduler/task_func.rs new file mode 100644 index 0000000..af27ded --- /dev/null +++ b/backend/api_server/src/blog/scheduler/task_func.rs @@ -0,0 +1,3 @@ +pub trait TaskFunc: Fn() + Send + Sync + 'static {} + +impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} diff --git a/backend/api_server/src/blog/scheduler/tasks/basic.rs b/backend/api_server/src/blog/scheduler/tasks/basic.rs new file mode 100644 index 0000000..d18e60c --- /dev/null +++ b/backend/api_server/src/blog/scheduler/tasks/basic.rs @@ -0,0 +1,30 @@ +use crate::blog::scheduler::task_func::TaskFunc; +use crate::blog::scheduler::tasks::Task; +use chrono::{DateTime, Utc}; +use cron::Schedule; +use std::error::Error; +use std::str::FromStr; + +pub struct BasicTask { + schedule: Schedule, + func: Box, +} + +impl BasicTask { + pub fn new(func: fn(), cron_expr: &str) -> Result> { + Ok(BasicTask { + schedule: Schedule::from_str(cron_expr)?, + func: Box::new(func), + }) + } +} + +impl Task for BasicTask { + fn get_next_execution(&self) -> Option> { + self.schedule.upcoming(Utc).next() + } + + fn call(&self) { + (self.func)() + } +} diff --git a/backend/api_server/src/blog/scheduler/tasks/mod.rs b/backend/api_server/src/blog/scheduler/tasks/mod.rs new file mode 100644 index 0000000..898bc9e --- /dev/null +++ b/backend/api_server/src/blog/scheduler/tasks/mod.rs @@ -0,0 +1,9 @@ +use crate::blog::scheduler::task_func::TaskFunc; +use chrono::{DateTime, Utc}; + +pub mod basic; + +pub trait Task: Send + Sync { + fn get_next_execution(&self) -> Option>; + fn call(&self); +} diff --git a/backend/api_server/src/init.rs b/backend/api_server/src/init.rs index 8785944..809b977 100644 --- a/backend/api_server/src/init.rs +++ b/backend/api_server/src/init.rs @@ -1,5 +1,7 @@ -use crate::logger; use std::env; + +use crate::logger; + use tracing_appender::non_blocking::WorkerGuard; pub fn logger_init() -> (WorkerGuard, WorkerGuard) { diff --git a/backend/api_server/src/logger.rs b/backend/api_server/src/logger.rs index 7721323..7dc26e2 100644 --- a/backend/api_server/src/logger.rs +++ b/backend/api_server/src/logger.rs @@ -1,16 +1,14 @@ -use chrono::Local; -use dirs::home_dir; use std::path::Path; use std::{env, io}; + +use chrono::Local; +use dirs::home_dir; use tracing::info; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_appender::{non_blocking, rolling}; -use tracing_subscriber::filter::LevelFilter; -use tracing_subscriber::fmt::format::Writer; -use tracing_subscriber::fmt::time::FormatTime; -use tracing_subscriber::fmt::MakeWriter; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::{fmt, Layer, Registry}; +use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; +use tracing_subscriber::{ + filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, + layer::SubscriberExt, Layer, Registry, +}; struct LocalTimer; @@ -95,10 +93,10 @@ pub fn init() -> (WorkerGuard, WorkerGuard) { ); tracing::subscriber::set_global_default(subscribers) - .expect("[fatal] Failed to setup logging system"); + .expect("[fatal] failed to setup logging system"); - info!("Module Started: Logger"); - info!("Logging to file {}", log_path); + info!("module started: logger"); + info!("logging to dir: {}", log_path); (info_guard, error_guard) } diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index 1db4480..7187196 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -1,7 +1,12 @@ +mod blog; mod init; mod logger; +mod server; -fn main() { +#[tokio::main] +async fn main() { init::load_env(); let _guards = init::logger_init(); + + server::start().await; } diff --git a/backend/legacy/src/main.rs b/backend/api_server/src/server.rs similarity index 52% rename from backend/legacy/src/main.rs rename to backend/api_server/src/server.rs index 7fef8f3..e23e63b 100644 --- a/backend/legacy/src/main.rs +++ b/backend/api_server/src/server.rs @@ -1,45 +1,20 @@ use std::env; -use std::error::Error; use std::num::ParseIntError; use std::process::exit; use std::str::FromStr; -use crate::api::{ - get::{get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list}, - post::refresh::refresh, -}; -use axum::routing::{get, post}; +use crate::blog; use axum::Router; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; -use sqlx::ConnectOptions; +use sqlx::{ + postgres::{PgConnectOptions, PgPoolOptions}, + ConnectOptions, +}; use tracing::{error, info, warn}; -mod api; -mod constants; -mod logger; -mod markdown; -mod model; - -#[tokio::main] -async fn main() -> Result<(), Box> { - if let Some(env_file) = env::args().nth(1) { - match dotenv::from_filename(&env_file) { - Ok(_) => { - println!("[Info] env loaded from {}, starting up", env_file); - } - Err(e) => { - panic!("[Fatal] failed to read {}: {}", env_file, e); - } - } - } else { - println!("[info] no .env file referred, starting up") - } - - let _guards = logger::init(); - +pub async fn start() { let port: u32 = match env::var("BACKEND_PORT") { Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { - warn!("Failed to parse BACKEND_PORT: {}", e.to_string()); + warn!("failed to parse BACKEND_PORT: {}", e.to_string()); 3001 }), Err(_) => 3001, @@ -51,35 +26,30 @@ async fn main() -> Result<(), Box> { let db_pool = match PgPoolOptions::new().connect_with(db_connect_option).await { Ok(pool) => { - info!("Database connected"); + info!("database connected"); pool } Err(e) => { - error!("Failed to load database: {}", e.to_string()); + error!("failed to load database: {}", e.to_string()); exit(1); } }; let app = Router::new() - .route("/refresh_posts", post(refresh)) - .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count)) - .route("/get_post/:post_id", get(get_post)) + .nest("blog", blog::get_router()) .with_state(db_pool); let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)) .await .unwrap_or_else(|e| { - error!("Failed to build TCP listener: {}", e.to_string()); + error!("failed to build TCP listener: {}", e.to_string()); exit(1); }); axum::serve(listener, app).await.unwrap_or_else(|e| { - error!("Failed to start axum: {}", e.to_string()); + error!("failed to start axum: {}", e.to_string()); exit(1); }); - - Ok(()) } fn parse_db_uri() -> String { diff --git a/backend/legacy/Cargo.toml b/backend/legacy/Cargo.toml deleted file mode 100644 index 5d3cbd3..0000000 --- a/backend/legacy/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "backend" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -axum = "0.7.5" -dotenv = "0.15.0" -serde = { version = "1.0.199", features = ["derive"] } -serde_json = "1.0.116" -tokio = { version = "1.37.0", features = ["full"] } -uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } -sqlx = { version = "0.7.4", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } -regex = "1.10.4" -futures = { version = "0.3.30", features = ["executor"] } -tracing = "0.1.40" -tracing-subscriber = "0.3.18" -tracing-appender = "0.2.3" -log = "0.4.21" -sha2 = "0.10.8" -thiserror = "1.0.63" -base64 = "0.22.1" diff --git a/backend/legacy/src/api/mod.rs b/backend/legacy/src/api/mod.rs deleted file mode 100644 index 69e599d..0000000 --- a/backend/legacy/src/api/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod get; -pub mod post; diff --git a/backend/legacy/src/api/post/errors/mod.rs b/backend/legacy/src/api/post/errors/mod.rs deleted file mode 100644 index 1d2e0f5..0000000 --- a/backend/legacy/src/api/post/errors/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod refresh; diff --git a/backend/legacy/src/api/post/errors/refresh.rs b/backend/legacy/src/api/post/errors/refresh.rs deleted file mode 100644 index 1c4224a..0000000 --- a/backend/legacy/src/api/post/errors/refresh.rs +++ /dev/null @@ -1,61 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum PostIdxError { - #[error("Failed to parse UUID <{}>: {}", id, err_msg)] - InvalidUUID { id: String, err_msg: String }, - - #[error( - "Failed to parse `last update time` for <{}> when creating post index: {}", - id, - err_msg - )] - InvalidLastUpdate { id: String, err_msg: String }, - - #[error( - "Failed to parse `first update time` for <{}> when creating post index: {}", - id, - err_msg - )] - InvalidFirstUpdate { id: String, err_msg: String }, - - #[error( - "Failed to parse `tag` for <{}> when creating post index: <{}>: {}", - id, - invalid_string, - err_msg - )] - InvalidTagString { - id: String, - invalid_string: String, - err_msg: String, - }, - - #[error( - "Database error on writing `{}` to table `{}`: <{}>: {}", - data_desc, - db_table, - id, - err_msg - )] - DBWriteFailure { - data_desc: String, - db_table: String, - id: String, - err_msg: String, - }, - - #[error( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - data_desc, - db_table, - id, - err_msg - )] - DBReadFailure { - data_desc: String, - db_table: String, - id: String, - err_msg: String, - }, -} diff --git a/backend/legacy/src/api/post/mod.rs b/backend/legacy/src/api/post/mod.rs deleted file mode 100644 index 1bba1ec..0000000 --- a/backend/legacy/src/api/post/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod errors; -pub mod refresh; -mod utils; diff --git a/backend/legacy/src/api/post/refresh.rs b/backend/legacy/src/api/post/refresh.rs deleted file mode 100644 index 65d7be3..0000000 --- a/backend/legacy/src/api/post/refresh.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::path::Path; -use std::str::FromStr; -use std::{env, fs}; - -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::Json; -use futures::{stream, StreamExt}; -use serde::Deserialize; -use serde_json::{json, Map, Value}; -use sha2::{Digest, Sha256}; -use sqlx::{query, PgPool}; -use tracing::error; -use uuid::Uuid; - -use crate::api::post::errors::refresh::PostIdxError; -use crate::api::post::utils::{build_post_idx, post}; -use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL}; - -#[derive(Deserialize)] -pub struct Params { - /// refresh level - level: Option, -} - -pub async fn refresh( - State(db_connection): State, - Query(params): Query, -) -> Result)> { - let is_force_refresh = params.level.unwrap_or("".to_string()) == "force"; - - // search post - let data_repo_dir = match env::var("GIT_WORK_DIR") { - Ok(value) => value, - Err(_) => { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(( - "error", - "error at parsing environment variable: GIT_WORK_DIR" - ))), - )); - } - }; - let post_dir = Path::new(&data_repo_dir).join("posts"); - - if is_force_refresh { - if let Err(e) = query("DROP TABLE IF EXISTS hash, meta, post, tag;") - .execute(&db_connection) - .await - { - error!( - "DB query error when deleting table: `hash`, `meta`, `post`, `tag`: {}", - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(("error", e.to_string()))), - )); - } - } - - // generate post idx - let post_path = search_md(&post_dir); - - let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = post_path - .into_iter() - .filter_map(|post_path| match post::from_path(&post_path) { - Ok(post) => match fs::read(&post_path) { - Ok(bytes) => { - let mut hasher = Sha256::new(); - hasher.update(bytes); - let hash = hasher.finalize().to_vec(); - Some((post, hash)) - } - Err(e) => { - error!( - "Failed to read file while hashing the post <{}>, skipping: {}", - post_path, - e.to_string() - ); - None - } - }, - Err(e) => { - error!( - "Failed to parse post <{}>, skipping: {}", - post_path, - e.to_string() - ); - None - } - }) - .collect(); - - // db create - query(POST_BASE_MODEL) - .execute(&db_connection) - .await - .unwrap(); - query(POST_META_MODEL) - .execute(&db_connection) - .await - .unwrap(); - query(TAG_MODEL).execute(&db_connection).await.unwrap(); - query(POST_HASH_MODEL) - .execute(&db_connection) - .await - .unwrap(); - - let res: Vec<_> = stream::iter(post_list_with_hash) - .map(|(post, hash)| { - let db_clone = db_connection.clone(); - async move { build_post_idx(&db_clone, post, hash).await } - }) - .buffer_unordered(usize::MAX) - .collect() - .await; - - let errs: Vec<_> = res - .into_iter() - .filter_map(|res| res.err()) - .map(|e| e.to_string()) - .collect(); - - if !errs.is_empty() { - Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": errs})), - )) - } else { - Ok(StatusCode::OK) - } -} - -/// Search a dir, return a list of markdown files -/// -/// # Arguments -/// * `entry`: dir path -/// -/// # Returns -/// Vec -/// * `String`: path of a markdown file -/// -fn search_md(entry: &Path) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path.as_path()) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry.to_str().unwrap().to_string()] - } else { - vec![] - } -} - -async fn build_post_idx( - db_connection: &PgPool, - post: Map, - hash: Vec, -) -> Result<(), PostIdxError> { - let meta = &post["meta"]; - - // extract post id - let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { - Ok(id) => id, - Err(e) => { - return Err(PostIdxError::InvalidUUID { - id: post["post_id"].as_str().unwrap().to_string(), - err_msg: e.to_string(), - }) - } - }; - - if build_post_idx::hash(db_connection, post_id, hash).await? { - build_post_idx::base(db_connection, &post, post_id, meta).await?; - build_post_idx::tag(db_connection, post_id, meta).await?; - build_post_idx::meta(db_connection, post_id, meta).await?; - } - - Ok(()) -} diff --git a/backend/legacy/src/api/post/utils/build_post_idx.rs b/backend/legacy/src/api/post/utils/build_post_idx.rs deleted file mode 100644 index 47b2692..0000000 --- a/backend/legacy/src/api/post/utils/build_post_idx.rs +++ /dev/null @@ -1,288 +0,0 @@ -use base64::{Engine, engine::general_purpose}; -use chrono::DateTime; -use serde_json::{Map, Value}; -use sqlx::{PgPool, query, Row}; -use uuid::Uuid; - -use crate::api::post::errors::refresh::PostIdxError; -use crate::constants::GLOBAL_TIME_FORMAT; - -pub async fn base( - db_connection: &PgPool, - post: &Map, - post_id: Uuid, - meta: &Value, -) -> Result<(), PostIdxError> { - let title = post["title"].as_str(); - let summary = post - .get("summary") - .and_then(|summary| summary.as_str()) - .unwrap_or(""); - let content = post["content"].as_str(); - let last_update = - match DateTime::parse_from_str(post["last_update"].as_str().unwrap(), GLOBAL_TIME_FORMAT) { - Ok(time) => time, - Err(e) => { - return Err(PostIdxError::InvalidLastUpdate { - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - }; - let first_update = match DateTime::parse_from_str( - post["first_update"].as_str().unwrap(), - GLOBAL_TIME_FORMAT, - ) { - Ok(time) => time, - Err(e) => { - return Err(PostIdxError::InvalidFirstUpdate { - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - }; - let sub_title = meta - .get("sub_title") - .and_then(|sub_title| sub_title.as_str()) - .unwrap_or(""); - let category = meta - .get("category") - .and_then(|category| category.as_str()) - .unwrap_or("N/A"); - let header_img = meta - .get("header_img") - .and_then(|header_img| header_img.as_str()) - .unwrap_or(""); - - match query( - r##" -INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) -ON CONFLICT (post_id) DO -UPDATE SET - title = EXCLUDED.title, - summary = EXCLUDED.summary, - content = EXCLUDED.content, - last_update = EXCLUDED.last_update, - first_update = EXCLUDED.first_update, - sub_title = EXCLUDED.sub_title, - category = EXCLUDED.category, - header_img = EXCLUDED.header_img; - "##, - ) - .bind(post_id) - .bind(title) - .bind(summary) - .bind(content) - .bind(last_update) - .bind(first_update) - .bind(sub_title) - .bind(category) - .bind(header_img) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err( - PostIdxError::DBWriteFailure { - data_desc: "post base information".to_string(), - db_table: "Post".to_string(), - id: post_id.to_string(), - err_msg: e.to_string() - } - ) - } - } - - Ok(()) -} - -pub async fn tag(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> { - if let Some(tags_string) = meta.get("tags").to_owned() { - // parse tags string - let parsed: Value = match serde_json::from_str(tags_string.as_str().unwrap()) { - Ok(value) => value, - Err(e) => { - return Err(PostIdxError::InvalidTagString { - id: post_id.to_string(), - invalid_string: tags_string.to_string().to_string(), - err_msg: e.to_string(), - }) - } - }; - let string_array = parsed.as_array().unwrap(); - let tags: Vec = string_array - .iter() - .map(|value| value.as_str().unwrap().to_string()) - .collect(); - - // remove old record - if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") - .bind(post_id) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - - // add new record - for tag in tags { - match query("INSERT INTO Tag VALUES ($1, $2);") - .bind(post_id) - .bind(tag) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - } - } - } - - Ok(()) -} - -/// # Arguments -/// -/// * `db_connection`: database connection -/// * `post_id`: UUID of the post -/// * `hash`: Hex String -/// -/// # Returns -/// Result -/// - `bool`: if the hash is different from database, return true, else false -pub async fn hash( - db_connection: &PgPool, - post_id: Uuid, - hash: Vec, -) -> Result { - let base64_form = general_purpose::STANDARD.encode(hash); - match query("SELECT hash FROM Hash WHERE post_id = $1") - .bind(post_id) - .fetch_optional(db_connection) - .await - { - Ok(row) => { - let db_record: String = match row { - None => "".to_string(), - Some(row) => row.try_get("hash").unwrap_or("".to_string()), - }; - if base64_form == db_record { - Ok(false) - } else { - if let Err(e) = query( - r#" -INSERT INTO Hash (post_id, hash) VALUES ($1, $2) - ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, - ) - .bind(post_id) - .bind(&base64_form) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - Ok(true) - } - } - Err(e) => Err(PostIdxError::DBReadFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }), - } -} - -pub async fn meta(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> { - let mut err_trails = 5; - loop { - if let Err(e) = query( - format!( - r#" -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY['tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY(_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - "#, post_id.to_string().as_str(), meta.to_string() - ).as_str() - ) - .execute(db_connection) - .await { - if err_trails >= 0 { - err_trails -= 1; - continue - } else { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post meta data".to_string(), - db_table: "Meta".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - } else { - return Ok(()) - } - } -} diff --git a/backend/legacy/src/api/post/utils/mod.rs b/backend/legacy/src/api/post/utils/mod.rs deleted file mode 100644 index faee48e..0000000 --- a/backend/legacy/src/api/post/utils/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod build_post_idx; -pub mod post; diff --git a/backend/legacy/src/api/post/utils/post.rs b/backend/legacy/src/api/post/utils/post.rs deleted file mode 100644 index 901d799..0000000 --- a/backend/legacy/src/api/post/utils/post.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::{env, fs}; -use std::error::Error; -use std::process::Command; -use std::str; -use std::str::FromStr; - -use chrono::{DateTime, Utc}; -use serde_json::{Map, Value}; -use tracing::{error, warn}; -use uuid::Uuid; - -use crate::constants::GLOBAL_TIME_FORMAT; -use crate::markdown::meta::parse_meta; - -pub fn from_path(path: &str) -> Result, Box> { - let mut res = Map::new(); - - // field `content` - let md_content = fs::read_to_string(path)?; - res.insert("content".to_string(), Value::from(md_content.to_owned())); - - // field `meta` - let meta = parse_meta(&md_content); - let meta_map: Map = meta - .clone() - .into_iter() - .map(|(k, v)| (k, Value::from(v))) - .collect(); - res.insert("meta".to_string(), Value::from(meta_map)); - - // field `post_id` - let post_id = match meta.get("id") { - None => Uuid::new_v4(), - Some(id) => match Uuid::from_str(id.as_str()) { - Ok(id) => id, - Err(e) => { - error!("Failed to parse UUID <{}>: {}", id.as_str(), e.to_string()); - return Err(Box::new(e)); - } - }, - }; - res.insert("post_id".to_string(), Value::from(post_id.to_string())); - - // field `title` - let title = match meta.get("title") { - None => "Title Not Found".to_string(), - Some(title) => title.to_owned(), - }; - res.insert("title".to_string(), Value::from(title)); - - // field `summary` - let summary = match meta.get("summary") { - None => "".to_string(), - Some(summary) => summary.to_owned(), - }; - res.insert("summary".to_string(), Value::from(summary)); - - // field `last_update` - let git_output = match Command::new("git") - .arg("--no-pager") - .arg("log") - .arg("-1") - .arg("--format=%cI") - .arg("--") - .current_dir(env::var("GIT_WORK_DIR")?) - .arg(path) - .output() - { - Ok(res) => res, - Err(e) => { - error!("Failed to execute `git` command: {}", e.to_string()); - return Err(Box::new(e)); - } - }; - let last_update: DateTime = if git_output.status.success() { - chrono::DateTime::parse_from_str( - str::from_utf8(&git_output.stdout).unwrap().trim_end(), - GLOBAL_TIME_FORMAT, - ) - .unwrap_or_else(|_| DateTime::from(Utc::now())) - .to_utc() - } else { - Utc::now().to_utc() - }; - res.insert( - "last_update".to_string(), - Value::from(last_update.format(GLOBAL_TIME_FORMAT).to_string()), - ); - - // field `first_update` - let first_update: DateTime = match meta.get("date") { - None => { - warn!( - "Failed to parse first update time for post {}, using current time instead", - post_id.to_string() - ); - Utc::now().to_utc() - } - Some(date) => DateTime::from( - match DateTime::parse_from_str(format!("{}T000000", date).as_str(), "%Y-%m-%d%zT%H%M%S") - { - Ok(time) => time, - Err(e) => { - warn!( - "Failed to parse first update time for post {}, using current time instead: {}", - post_id.to_string(), - e.to_string() - ); - DateTime::from(Utc::now()) - } - }, - ), - }; - res.insert( - "first_update".to_string(), - Value::from(first_update.format(GLOBAL_TIME_FORMAT).to_string()), - ); - - Ok(res) -} diff --git a/backend/legacy/src/constants.rs b/backend/legacy/src/constants.rs deleted file mode 100644 index 5dd57e4..0000000 --- a/backend/legacy/src/constants.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub const GLOBAL_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; - -pub const PAGE_SIZE: u32 = 6; diff --git a/backend/legacy/src/logger.rs b/backend/legacy/src/logger.rs deleted file mode 100644 index 974a706..0000000 --- a/backend/legacy/src/logger.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::{env, io}; - -use chrono::Local; -use tracing::info; -use tracing_appender::{non_blocking, rolling}; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_subscriber::{fmt, Registry}; -use tracing_subscriber::fmt::format::Writer; -use tracing_subscriber::fmt::MakeWriter; -use tracing_subscriber::fmt::time::FormatTime; -use tracing_subscriber::layer::SubscriberExt; - -use crate::constants::GLOBAL_TIME_FORMAT; - -struct LocalTimer; - -impl FormatTime for LocalTimer { - fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { - write!(w, "{}", Local::now().format(GLOBAL_TIME_FORMAT)) - } -} - -type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; - -pub fn init() -> (WorkerGuard, WorkerGuard) { - let log_path = env::var("LOG_ROOT_DIR").unwrap_or_else(|e| { - panic!("[Fatal] failed to load LOG_ROOT_DIR: {}", e); - }); - - let log_path = format!("{}/backend", log_path); - - let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); - - let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); - - let log_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_timer(LocalTimer); - - let subscribers = Registry::default() - .with( - fmt::Layer::default() - .with_writer(info_log_writer) - .with_ansi(false), - ) - .with( - fmt::Layer::default() - .with_writer(err_log_writer) - .with_ansi(false), - ) - .with( - fmt::Layer::default() - .with_writer(io::stdout) - .with_ansi(true) - .event_format(log_format), - ); - - tracing::subscriber::set_global_default(subscribers) - .expect("[Fatal] Failed to setup logging system"); - - info!("Module Started: Logger"); - - // error!("test"); - (info_guard, error_guard) -} diff --git a/backend/legacy/src/markdown/meta.rs b/backend/legacy/src/markdown/meta.rs deleted file mode 100644 index b2d7cac..0000000 --- a/backend/legacy/src/markdown/meta.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::collections::HashMap; - -use regex::Regex; - -/// return the meta info contained in the .md file front matter -pub fn parse_meta(md_content: &str) -> HashMap { - let mut fields: HashMap = HashMap::new(); - let match_regex = Regex::new(r"(?s)---\n(.*\n)---\n").unwrap(); - if let Some(front_matter) = match_regex.captures(md_content) { - let fm_content = front_matter.get(1).unwrap().as_str(); - for line in fm_content.lines() { - if let Some((k, v)) = extract_kv(line) { - fields.insert(k, v); - } - } - } - fields -} - -fn extract_kv(line: &str) -> Option<(String, String)> { - let split: Vec<&str> = line.splitn(2, ':').map(|s| s.trim()).collect(); - if split.len() == 2 { - let key = split[0].to_string(); - let value = split[1].to_string(); - Some((key, value)) - } else { - None - } -} - -#[cfg(test)] -mod meta_test { - use std::error::Error; - - use crate::markdown::meta::parse_meta; - - #[test] - fn test_parse_meta_front_matter() -> Result<(), Box> { - let parsed_meta = parse_meta( - r##" ---- -title: Hello World! -tags: [ "test", "hello_world" ] ---- - "##, - ); - dbg!(&parsed_meta); - assert_eq!(parsed_meta.get("title"), Some(&"Hello World!".to_string())); - assert_eq!( - parsed_meta.get("tags"), - Some(&r#"[ "test", "hello_world" ]"#.to_string()) - ); - Ok(()) - } -} diff --git a/backend/legacy/src/markdown/mod.rs b/backend/legacy/src/markdown/mod.rs deleted file mode 100644 index 6b50816..0000000 --- a/backend/legacy/src/markdown/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod meta; diff --git a/backend/legacy/src/model/blog/hash.sql b/backend/legacy/src/model/blog/hash.sql deleted file mode 100644 index 7dc91de..0000000 --- a/backend/legacy/src/model/blog/hash.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); \ No newline at end of file diff --git a/backend/legacy/src/model/blog/meta.sql b/backend/legacy/src/model/blog/meta.sql deleted file mode 100644 index 6cf903a..0000000 --- a/backend/legacy/src/model/blog/meta.sql +++ /dev/null @@ -1,79 +0,0 @@ -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); - -DROP table meta; - -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY ['id', 'tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY (_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - -SELECT jsonb_strip_nulls(to_jsonb(t)) AS meta -FROM meta t -WHERE post_id = '8ca3fa2b-f040-448c-8885-81aab56db9bb'; - -WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = '8ca3fa2b-f040-448c-8885-81aab56db9bb'), - prev AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date < (SELECT date FROM target_post) - ORDER BY t.date DESC - LIMIT 1), - next AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date > (SELECT date FROM target_post) - ORDER BY t.date - LIMIT 1) -SELECT json_strip_nulls(json_build_object( - 'prev', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM prev), - 'next', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM next) - )) AS result; - diff --git a/backend/legacy/src/model/blog/mod.rs b/backend/legacy/src/model/blog/mod.rs deleted file mode 100644 index c282af7..0000000 --- a/backend/legacy/src/model/blog/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -pub const POST_BASE_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - header_img TEXT, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, - first_update TIMESTAMPTZ NOT NULL -); -"##; - -// base mode, will add more cols in runtime -pub const POST_META_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); -"##; - -pub const POST_HASH_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); -"##; - -pub const TAG_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); -"##; diff --git a/backend/legacy/src/model/blog/post.sql b/backend/legacy/src/model/blog/post.sql deleted file mode 100644 index 1afc493..0000000 --- a/backend/legacy/src/model/blog/post.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - header_img TEXT, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, - first_update TIMESTAMPTZ NOT NULL -); diff --git a/backend/legacy/src/model/blog/tag.sql b/backend/legacy/src/model/blog/tag.sql deleted file mode 100644 index 6c84b76..0000000 --- a/backend/legacy/src/model/blog/tag.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); diff --git a/backend/legacy/src/model/mod.rs b/backend/legacy/src/model/mod.rs deleted file mode 100644 index 21aa2c6..0000000 --- a/backend/legacy/src/model/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod blog; diff --git a/backend/macros/Cargo.toml b/backend/macros/Cargo.toml new file mode 100644 index 0000000..a11722a --- /dev/null +++ b/backend/macros/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "macros" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs new file mode 100644 index 0000000..b93cf3f --- /dev/null +++ b/backend/macros/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index 6f30a74..896c0c8 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -4,3 +4,11 @@ version = "0.1.0" edition = "2021" [dependencies] +uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } +chrono = { version = "0.4.38", features = ["serde", "alloc"] } +serde = { version = "1.0.209", features = ["derive"] } +serde_json = "1.0.128" +regex = "1.10.4" +yaml-rust2 = "0.8.1" +sha2 = "0.10.8" +base64 = "0.22.1" diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index b93cf3f..ce6345f 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -1,14 +1,55 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right +mod utils; + +use std::error::Error; +use std::fs; +use std::path::Path; +use std::str::FromStr; + +use base64::{engine::general_purpose, Engine}; +use serde::Serialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + +#[derive(Default, Debug, Serialize)] +pub struct MdFile { + file_id: Uuid, + file_name: String, + meta: Value, + content: String, + hash: String, } -#[cfg(test)] -mod tests { - use super::*; +impl MdFile { + pub async fn from_file(file_path: &str) -> Result> { + let raw_file = fs::read_to_string(file_path)?; + let raw_file_bytes = raw_file.as_bytes().to_vec(); + let (front_matter_str, content) = split_md_front_matter(raw_file); + let front_matter = parse_front_matter(&front_matter_str)?; + let hash = calculate_hash_base64(&raw_file_bytes); + let file_id = Uuid::from_str(front_matter["id"].to_string().as_str())?; + let file_name = Path::new(file_path) + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); + Ok(MdFile { + file_id, + file_name, + meta: front_matter, + content, + hash, + }) } } + +fn calculate_hash_base64(bytes: &Vec) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let hash = hasher.finalize().to_vec(); + general_purpose::STANDARD.encode(hash) +} diff --git a/backend/markdown/src/utils/front_matter.rs b/backend/markdown/src/utils/front_matter.rs new file mode 100644 index 0000000..5bca291 --- /dev/null +++ b/backend/markdown/src/utils/front_matter.rs @@ -0,0 +1,77 @@ +use crate::utils::yaml::yaml_to_serde; +use regex::Regex; +use serde_json::Value; +use std::error::Error; +use yaml_rust2::YamlLoader; + +pub fn split_md_front_matter(raw_file: String) -> (String, String) { + let re = Regex::new(r"(?s)^(---\n.*?\n---\n)(.*)").unwrap(); + if let Some(captures) = re.captures(raw_file.as_str()) { + let front_matter = captures.get(1).map_or("", |m| m.as_str()).to_string(); + let main_content = captures.get(2).map_or("", |m| m.as_str()).to_string(); + (front_matter, main_content) + } else { + (String::new(), raw_file) + } +} + +pub fn parse_front_matter(front_matter: &str) -> Result> { + let parsed_f_matter = YamlLoader::load_from_str(front_matter)?[0].clone(); + Ok(Value::from(yaml_to_serde(&parsed_f_matter))) +} + +#[cfg(test)] +mod tests { + use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + + #[test] + fn front_matter_to_string() { + let (front_matter, main_content) = split_md_front_matter( + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- + +test body + +"## + .to_string(), + ); + assert_eq!( + front_matter, + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- +"## + ); + assert_eq!(main_content, "\ntest body\n\n"); + } + + #[test] + fn test_front_matter_parser() { + parse_front_matter( + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- +"##, + ) + .expect("Failed to parse front matter"); + } +} diff --git a/backend/markdown/src/utils/mod.rs b/backend/markdown/src/utils/mod.rs new file mode 100644 index 0000000..27be997 --- /dev/null +++ b/backend/markdown/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod front_matter; +mod yaml; diff --git a/backend/markdown/src/utils/yaml.rs b/backend/markdown/src/utils/yaml.rs new file mode 100644 index 0000000..8c52f4a --- /dev/null +++ b/backend/markdown/src/utils/yaml.rs @@ -0,0 +1,29 @@ +use serde_json::Value; +use std::str::FromStr; +use yaml_rust2::Yaml; + +pub fn yaml_to_serde(yaml: &Yaml) -> Value { + match yaml { + Yaml::Null => Value::Null, + Yaml::Boolean(b) => Value::Bool(*b), + Yaml::Integer(i) => Value::Number(serde_json::Number::from(*i)), + Yaml::Real(s) => Value::Number(serde_json::Number::from_str(s).unwrap()), + Yaml::String(s) => Value::String(s.clone()), + Yaml::Array(arr) => Value::Array(arr.iter().map(yaml_to_serde).collect()), + Yaml::Hash(hash) => { + let obj: serde_json::map::Map = hash + .iter() + .map(|(k, v)| { + let key = match k { + Yaml::String(s) => s.clone(), + _ => "INVALID_KEY".to_string(), + }; + (key, yaml_to_serde(v)) + }) + .collect(); + Value::Object(obj) + } + Yaml::Alias(_) => Value::Null, + Yaml::BadValue => Value::Null, + } +} diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index 3e71f6c..1c4b3c9 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -8,7 +8,7 @@ import Footer from '@/components/Footer'; export const metadata: Metadata = { title: 'Lumitherの酒馆', - description: '一间位于时间夹缝中的旅馆，静候下一位旅者的到来...' + description: '一间位于时间夹缝中的酒馆，静候下一位旅者的到来...' }; export default function RootLayout({ From 6f45a140ba91b2828d8f2faa9b9db6ee08fa5622 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 30 Oct 2024 22:54:39 -0700 Subject: [PATCH 04/13] ref(scheduler) move module --- backend/api_server/src/blog/mod.rs | 1 - backend/api_server/src/main.rs | 1 + backend/api_server/src/{blog => }/scheduler/mod.rs | 6 +++--- .../api_server/src/{blog => }/scheduler/scheduler_error.rs | 2 +- backend/api_server/src/{blog => }/scheduler/sleeper.rs | 2 +- backend/api_server/src/{blog => }/scheduler/task_func.rs | 0 backend/api_server/src/{blog => }/scheduler/tasks/basic.rs | 4 ++-- backend/api_server/src/{blog => }/scheduler/tasks/mod.rs | 1 - 8 files changed, 8 insertions(+), 9 deletions(-) rename backend/api_server/src/{blog => }/scheduler/mod.rs (97%) rename backend/api_server/src/{blog => }/scheduler/scheduler_error.rs (93%) rename backend/api_server/src/{blog => }/scheduler/sleeper.rs (97%) rename backend/api_server/src/{blog => }/scheduler/task_func.rs (100%) rename backend/api_server/src/{blog => }/scheduler/tasks/basic.rs (86%) rename backend/api_server/src/{blog => }/scheduler/tasks/mod.rs (76%) diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs index abfb8e6..06ac819 100644 --- a/backend/api_server/src/blog/mod.rs +++ b/backend/api_server/src/blog/mod.rs @@ -1,5 +1,4 @@ mod api; -mod scheduler; use crate::blog::api::{ get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index 7187196..520be42 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -1,6 +1,7 @@ mod blog; mod init; mod logger; +mod scheduler; mod server; #[tokio::main] diff --git a/backend/api_server/src/blog/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs similarity index 97% rename from backend/api_server/src/blog/scheduler/mod.rs rename to backend/api_server/src/scheduler/mod.rs index ef73764..018cd5c 100644 --- a/backend/api_server/src/blog/scheduler/mod.rs +++ b/backend/api_server/src/scheduler/mod.rs @@ -1,4 +1,4 @@ -use crate::blog::{ +use crate::{ scheduler::scheduler_error::SchedulerError, scheduler::scheduler_error::SchedulerError::IdAlreadyRunning, scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper, @@ -163,8 +163,8 @@ impl Scheduler { #[cfg(test)] mod tests { - use crate::blog::scheduler::tasks::basic::BasicTask; - use crate::blog::scheduler::Scheduler; + use crate::scheduler::tasks::basic::BasicTask; + use crate::scheduler::Scheduler; use std::error::Error; use std::thread; use std::time::Duration; diff --git a/backend/api_server/src/blog/scheduler/scheduler_error.rs b/backend/api_server/src/scheduler/scheduler_error.rs similarity index 93% rename from backend/api_server/src/blog/scheduler/scheduler_error.rs rename to backend/api_server/src/scheduler/scheduler_error.rs index 54615be..43f30dd 100644 --- a/backend/api_server/src/blog/scheduler/scheduler_error.rs +++ b/backend/api_server/src/scheduler/scheduler_error.rs @@ -1,4 +1,4 @@ -use crate::blog::scheduler::scheduler_error::SchedulerError::MutexPoison; +use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; use std::error::Error; use std::fmt::{Debug, Display, Formatter}; use std::sync::PoisonError; diff --git a/backend/api_server/src/blog/scheduler/sleeper.rs b/backend/api_server/src/scheduler/sleeper.rs similarity index 97% rename from backend/api_server/src/blog/scheduler/sleeper.rs rename to backend/api_server/src/scheduler/sleeper.rs index dd84bd6..4155e57 100644 --- a/backend/api_server/src/blog/scheduler/sleeper.rs +++ b/backend/api_server/src/scheduler/sleeper.rs @@ -54,7 +54,7 @@ impl From> for SkippableSleeper { #[cfg(test)] mod test { - use crate::blog::scheduler::sleeper::SkippableSleeper; + use crate::scheduler::sleeper::SkippableSleeper; use std::thread; use std::time::Duration; diff --git a/backend/api_server/src/blog/scheduler/task_func.rs b/backend/api_server/src/scheduler/task_func.rs similarity index 100% rename from backend/api_server/src/blog/scheduler/task_func.rs rename to backend/api_server/src/scheduler/task_func.rs diff --git a/backend/api_server/src/blog/scheduler/tasks/basic.rs b/backend/api_server/src/scheduler/tasks/basic.rs similarity index 86% rename from backend/api_server/src/blog/scheduler/tasks/basic.rs rename to backend/api_server/src/scheduler/tasks/basic.rs index d18e60c..0318899 100644 --- a/backend/api_server/src/blog/scheduler/tasks/basic.rs +++ b/backend/api_server/src/scheduler/tasks/basic.rs @@ -1,5 +1,5 @@ -use crate::blog::scheduler::task_func::TaskFunc; -use crate::blog::scheduler::tasks::Task; +use crate::scheduler::task_func::TaskFunc; +use crate::scheduler::tasks::Task; use chrono::{DateTime, Utc}; use cron::Schedule; use std::error::Error; diff --git a/backend/api_server/src/blog/scheduler/tasks/mod.rs b/backend/api_server/src/scheduler/tasks/mod.rs similarity index 76% rename from backend/api_server/src/blog/scheduler/tasks/mod.rs rename to backend/api_server/src/scheduler/tasks/mod.rs index 898bc9e..7e3b87a 100644 --- a/backend/api_server/src/blog/scheduler/tasks/mod.rs +++ b/backend/api_server/src/scheduler/tasks/mod.rs @@ -1,4 +1,3 @@ -use crate::blog::scheduler::task_func::TaskFunc; use chrono::{DateTime, Utc}; pub mod basic; From d98aa3f745f9a624f4c668be1e979059f68dbbe2 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 30 Oct 2024 23:07:22 -0700 Subject: [PATCH 05/13] fix(scheduler) remove unnecessary field --- backend/api_server/src/scheduler/mod.rs | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/backend/api_server/src/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs index 018cd5c..21678f4 100644 --- a/backend/api_server/src/scheduler/mod.rs +++ b/backend/api_server/src/scheduler/mod.rs @@ -7,6 +7,7 @@ use crate::{ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::RwLock; use std::{collections::HashMap, sync::Arc, thread}; +use tracing::log::error; use uuid::Uuid; pub mod scheduler_error; @@ -23,7 +24,6 @@ struct TaskStatus { pub struct Scheduler { tasks: Arc>>>>, running: Arc>>>, - stopped: Arc>>, } impl Scheduler { @@ -40,7 +40,6 @@ impl Scheduler { fn run_task( running: Arc>>>, - stopped: Arc>>, id: Uuid, task: Arc>, ) -> Result<(), SchedulerError> { @@ -69,7 +68,6 @@ impl Scheduler { let mut running_lock_guard = running.write()?; running_lock_guard.remove(&id); } - stopped.write()?.push(id); return Ok(()); } else { let sleeper = Arc::from(SkippableSleeper::from(next_execution)); @@ -89,7 +87,7 @@ impl Scheduler { } } } else { - stopped.write()?.push(id); + error!("unexpected error "); break; } } @@ -100,7 +98,7 @@ impl Scheduler { pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { let id = id.to_owned(); if let Some(task) = self.tasks.read()?.get(&id) { - Self::run_task(self.running.clone(), self.stopped.clone(), id, task.clone()) + Self::run_task(self.running.clone(), id, task.clone()) } else { Err(IdNotFound) } @@ -110,9 +108,8 @@ impl Scheduler { let id = id.to_owned(); if let Some(task) = self.tasks.read()?.get(&id) { let running_clone = self.running.clone(); - let stopped_clone = self.stopped.clone(); let task_clone = task.clone(); - thread::spawn(move || Self::run_task(running_clone, stopped_clone, id, task_clone)); + thread::spawn(move || Self::run_task(running_clone, id, task_clone)); Ok(()) } else { Err(IdNotFound) @@ -174,11 +171,16 @@ mod tests { let scheduler = Scheduler::default(); let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; let id = scheduler.insert(Box::new(task))?; - scheduler.run_non_blocking(&id)?; - thread::sleep(Duration::from_secs(4)); - scheduler.stop(&id)?; - // test termination - thread::sleep(Duration::from_secs(5)); + for i in 1..=2 { + println!("Loop {} start", i); + println!("Run {}", i); + scheduler.run_non_blocking(&id)?; + thread::sleep(Duration::from_secs(4)); + scheduler.stop(&id)?; + println!("Stop {}", i); + thread::sleep(Duration::from_secs(2)); + println!("Loop {} finished", i); + } Ok(()) } } From a123ff9f3fdba97417327bb772aecb31dd869b8a Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Sun, 17 Nov 2024 21:06:36 -0800 Subject: [PATCH 06/13] fix(scheduler) fn type --- backend/api_server/Cargo.toml | 3 +- .../api_server/src/blog/cron/check_update.rs | 24 ++++++++++++ backend/api_server/src/blog/cron/mod.rs | 1 + backend/api_server/src/blog/mod.rs | 1 + backend/api_server/src/init.rs | 38 +++++++++++++++++-- backend/api_server/src/main.rs | 9 ++++- backend/api_server/src/scheduler/mod.rs | 4 ++ .../api_server/src/scheduler/tasks/basic.rs | 9 ++++- backend/api_server/src/server.rs | 18 ++++++++- backend/markdown/Cargo.toml | 2 +- backend/markdown/src/utils/front_matter.rs | 4 +- 11 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 backend/api_server/src/blog/cron/check_update.rs create mode 100644 backend/api_server/src/blog/cron/mod.rs diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index bb2012c..09883f5 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -19,4 +19,5 @@ serde_json = "1.0.128" uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } futures = { version = "0.3.30", features = ["executor"] } sha2 = "0.10.8" -cron = "0.12.1" \ No newline at end of file +cron = "0.12.1" +git2 = "0.19.0" diff --git a/backend/api_server/src/blog/cron/check_update.rs b/backend/api_server/src/blog/cron/check_update.rs new file mode 100644 index 0000000..f2fe6a9 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update.rs @@ -0,0 +1,24 @@ +use dirs::home_dir; +use std::env; + +pub fn task() { + // get dir + let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| { + if let Ok(default_log_path) = env::var("WORK_DIR") { + if default_log_path.starts_with('~') { + let home_path = home_dir().unwrap_or_else(|| panic!("Cannot find home directory")); + home_path + .join(default_log_path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .expect( + "[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead", + ) + } else { + default_log_path + } + } else { + panic!("[fatal] failed to parse env `GIT_WORK_DIR`: {}", e); + } + }); +} diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs new file mode 100644 index 0000000..be1305e --- /dev/null +++ b/backend/api_server/src/blog/cron/mod.rs @@ -0,0 +1 @@ +pub mod check_update; diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs index 06ac819..8b61780 100644 --- a/backend/api_server/src/blog/mod.rs +++ b/backend/api_server/src/blog/mod.rs @@ -1,4 +1,5 @@ mod api; +pub mod cron; use crate::blog::api::{ get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, diff --git a/backend/api_server/src/init.rs b/backend/api_server/src/init.rs index 809b977..73e53fd 100644 --- a/backend/api_server/src/init.rs +++ b/backend/api_server/src/init.rs @@ -1,11 +1,17 @@ -use std::env; - use crate::logger; +use std::{ + backtrace::{Backtrace, BacktraceStatus}, + env, panic, + panic::PanicHookInfo, +}; +use tracing::{error, event, span, Level}; use tracing_appender::non_blocking::WorkerGuard; pub fn logger_init() -> (WorkerGuard, WorkerGuard) { - logger::init() + let ret = logger::init(); + panic::set_hook(Box::new(panic_hook)); + ret } pub fn load_env() { @@ -22,3 +28,29 @@ pub fn load_env() { println!("[info] no .env file referred, starting up") } } + +fn panic_hook(panic_info: &PanicHookInfo) { + let payload = panic_info.payload(); + // panic_info. + + let payload = if let Some(s) = payload.downcast_ref::<&str>() { + Some(&**s) + } else { + payload.downcast_ref::().map(|s| s.as_str()) + }; + + // let location = panic_info.location().map(|l| l.to_string()); + // let backtrace = Backtrace::capture(); + // let note = (backtrace.status() == BacktraceStatus::Disabled) + // .then_some("run with RUST_BACKTRACE=1 environment variable to display a backtrace"); + // (Some(backtrace), note) + // }; + // + // panic_info. + + if let Some(location) = panic_info.location() { + eprintln!("err filename: {}", location.file()) + } + + event!(target: "test", Level::ERROR, "panic hook: {:?}", panic_info); +} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index 520be42..411e673 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -9,5 +9,12 @@ async fn main() { init::load_env(); let _guards = init::logger_init(); - server::start().await; + // panic!("test"); + + match server::start().await { + Ok(_) => {} + Err(e) => { + panic!("server start failed: {}", e); + } + }; } diff --git a/backend/api_server/src/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs index 21678f4..08f018c 100644 --- a/backend/api_server/src/scheduler/mod.rs +++ b/backend/api_server/src/scheduler/mod.rs @@ -156,6 +156,10 @@ impl Scheduler { self.tasks.write()?.insert(id, Arc::new(task)); Ok(id) } + + pub fn new() -> Self { + Self::default() + } } #[cfg(test)] diff --git a/backend/api_server/src/scheduler/tasks/basic.rs b/backend/api_server/src/scheduler/tasks/basic.rs index 0318899..71fc3a6 100644 --- a/backend/api_server/src/scheduler/tasks/basic.rs +++ b/backend/api_server/src/scheduler/tasks/basic.rs @@ -11,12 +11,19 @@ pub struct BasicTask { } impl BasicTask { - pub fn new(func: fn(), cron_expr: &str) -> Result> { + pub fn new(func: F, cron_expr: &str) -> Result> + where + F: TaskFunc, + { Ok(BasicTask { schedule: Schedule::from_str(cron_expr)?, func: Box::new(func), }) } + + pub fn to_task(self) -> Box { + Box::new(self) + } } impl Task for BasicTask { diff --git a/backend/api_server/src/server.rs b/backend/api_server/src/server.rs index e23e63b..6dfb8e8 100644 --- a/backend/api_server/src/server.rs +++ b/backend/api_server/src/server.rs @@ -1,9 +1,12 @@ use std::env; +use std::error::Error; use std::num::ParseIntError; use std::process::exit; use std::str::FromStr; use crate::blog; +use crate::scheduler::tasks::basic::BasicTask; +use crate::scheduler::Scheduler; use axum::Router; use sqlx::{ postgres::{PgConnectOptions, PgPoolOptions}, @@ -11,7 +14,7 @@ use sqlx::{ }; use tracing::{error, info, warn}; -pub async fn start() { +pub async fn start() -> Result<(), Box> { let port: u32 = match env::var("BACKEND_PORT") { Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { warn!("failed to parse BACKEND_PORT: {}", e.to_string()); @@ -35,6 +38,18 @@ pub async fn start() { } }; + // cron task + let scheduler = Scheduler::new(); + let test_str = "hello world".to_string(); + // scheduler.insert( + // BasicTask::new( + // move || blog::cron::check_update::task(test_str.clone()), + // "* * * * * *", + // )? + // .to_task(), + // )?; + + // http server let app = Router::new() .nest("blog", blog::get_router()) .with_state(db_pool); @@ -50,6 +65,7 @@ pub async fn start() { error!("failed to start axum: {}", e.to_string()); exit(1); }); + Ok(()) } fn parse_db_uri() -> String { diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index 896c0c8..0d0b372 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -9,6 +9,6 @@ chrono = { version = "0.4.38", features = ["serde", "alloc"] } serde = { version = "1.0.209", features = ["derive"] } serde_json = "1.0.128" regex = "1.10.4" -yaml-rust2 = "0.8.1" +yaml-rust2 = "0.9.0" sha2 = "0.10.8" base64 = "0.22.1" diff --git a/backend/markdown/src/utils/front_matter.rs b/backend/markdown/src/utils/front_matter.rs index 5bca291..f38f029 100644 --- a/backend/markdown/src/utils/front_matter.rs +++ b/backend/markdown/src/utils/front_matter.rs @@ -60,7 +60,7 @@ tags: [ "test1", "test2" ] #[test] fn test_front_matter_parser() { - parse_front_matter( + dbg!(parse_front_matter( r##"--- title: test_title date: 0000-00-00+0000 @@ -71,7 +71,7 @@ summary: summary tags: [ "test1", "test2" ] --- "##, - ) + )) .expect("Failed to parse front matter"); } } From c7e3b12856558a04e44455d81373cc7d7bc08180 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Tue, 19 Nov 2024 00:33:09 -0800 Subject: [PATCH 07/13] feat(macros) add `panic_with_log` --- backend/api_server/Cargo.toml | 1 + backend/api_server/src/init.rs | 37 ++------------------------ backend/api_server/src/main.rs | 7 ++--- backend/api_server/src/server.rs | 31 +++++++++++++--------- backend/macros/Cargo.toml | 2 ++ backend/macros/src/lib.rs | 15 +---------- backend/macros/src/log.rs | 45 ++++++++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 64 deletions(-) create mode 100644 backend/macros/src/log.rs diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index 09883f5..e140b1e 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +macros = { path = "../macros" } constants = { path = "../constants" } chrono = { version = "0.4.38", features = ["serde", "alloc"] } tracing = "0.1.40" diff --git a/backend/api_server/src/init.rs b/backend/api_server/src/init.rs index 73e53fd..c240d89 100644 --- a/backend/api_server/src/init.rs +++ b/backend/api_server/src/init.rs @@ -1,17 +1,10 @@ use crate::logger; -use std::{ - backtrace::{Backtrace, BacktraceStatus}, - env, panic, - panic::PanicHookInfo, -}; -use tracing::{error, event, span, Level}; +use std::{env, panic}; use tracing_appender::non_blocking::WorkerGuard; pub fn logger_init() -> (WorkerGuard, WorkerGuard) { - let ret = logger::init(); - panic::set_hook(Box::new(panic_hook)); - ret + logger::init() } pub fn load_env() { @@ -28,29 +21,3 @@ pub fn load_env() { println!("[info] no .env file referred, starting up") } } - -fn panic_hook(panic_info: &PanicHookInfo) { - let payload = panic_info.payload(); - // panic_info. - - let payload = if let Some(s) = payload.downcast_ref::<&str>() { - Some(&**s) - } else { - payload.downcast_ref::().map(|s| s.as_str()) - }; - - // let location = panic_info.location().map(|l| l.to_string()); - // let backtrace = Backtrace::capture(); - // let note = (backtrace.status() == BacktraceStatus::Disabled) - // .then_some("run with RUST_BACKTRACE=1 environment variable to display a backtrace"); - // (Some(backtrace), note) - // }; - // - // panic_info. - - if let Some(location) = panic_info.location() { - eprintln!("err filename: {}", location.file()) - } - - event!(target: "test", Level::ERROR, "panic hook: {:?}", panic_info); -} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index 411e673..c1431c4 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -1,3 +1,6 @@ +use macros::panic_with_log; +use tracing::Level; + mod blog; mod init; mod logger; @@ -9,12 +12,10 @@ async fn main() { init::load_env(); let _guards = init::logger_init(); - // panic!("test"); - match server::start().await { Ok(_) => {} Err(e) => { - panic!("server start failed: {}", e); + panic_with_log!(Level::ERROR, "failed to start server: {}", e); } }; } diff --git a/backend/api_server/src/server.rs b/backend/api_server/src/server.rs index 6dfb8e8..047eae1 100644 --- a/backend/api_server/src/server.rs +++ b/backend/api_server/src/server.rs @@ -5,14 +5,13 @@ use std::process::exit; use std::str::FromStr; use crate::blog; -use crate::scheduler::tasks::basic::BasicTask; -use crate::scheduler::Scheduler; use axum::Router; +use macros::panic_with_log; use sqlx::{ postgres::{PgConnectOptions, PgPoolOptions}, ConnectOptions, }; -use tracing::{error, info, warn}; +use tracing::{error, info, warn, Level}; pub async fn start() -> Result<(), Box> { let port: u32 = match env::var("BACKEND_PORT") { @@ -39,8 +38,8 @@ pub async fn start() -> Result<(), Box> { }; // cron task - let scheduler = Scheduler::new(); - let test_str = "hello world".to_string(); + // let scheduler = Scheduler::new(); + // let test_str = "hello world".to_string(); // scheduler.insert( // BasicTask::new( // move || blog::cron::check_update::task(test_str.clone()), @@ -51,15 +50,23 @@ pub async fn start() -> Result<(), Box> { // http server let app = Router::new() - .nest("blog", blog::get_router()) + .nest("/blog", blog::get_router()) .with_state(db_pool); - let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)) - .await - .unwrap_or_else(|e| { - error!("failed to build TCP listener: {}", e.to_string()); - exit(1); - }); + let listener = match tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)).await { + Ok(listener) => { + info!("server started on 0.0.0.0:{}", port); + listener + } + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to start server on 0.0.0.0:{}: {}", + port, + e.to_string() + ); + } + }; axum::serve(listener, app).await.unwrap_or_else(|e| { error!("failed to start axum: {}", e.to_string()); diff --git a/backend/macros/Cargo.toml b/backend/macros/Cargo.toml index a11722a..6291521 100644 --- a/backend/macros/Cargo.toml +++ b/backend/macros/Cargo.toml @@ -4,3 +4,5 @@ version = "0.1.0" edition = "2021" [dependencies] +tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs index b93cf3f..aad31d7 100644 --- a/backend/macros/src/lib.rs +++ b/backend/macros/src/lib.rs @@ -1,14 +1 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +mod log; diff --git a/backend/macros/src/log.rs b/backend/macros/src/log.rs new file mode 100644 index 0000000..f7dd357 --- /dev/null +++ b/backend/macros/src/log.rs @@ -0,0 +1,45 @@ +/// log the message at specific log level and exit(1) +/// +/// # Example +/// ```rust +/// +/// use macros::panic_with_log; +/// use tracing::Level; +/// +/// panic_with_log!(Level::ERROR, "Test panicked: {}", 42); +/// ``` +#[macro_export] +macro_rules! panic_with_log { + ($level:expr, $msg:literal $(, $($arg:tt)*)?) => {{ + use std::process; + use tracing::event; + event!($level, $msg $(, $($arg)*)?); + process::exit(1); + }}; +} + +#[cfg(test)] +mod tests { + #[test] + fn test_panic_with_log() { + use std::io; + use tracing::Level; + use tracing_subscriber::prelude::*; + use tracing_subscriber::{fmt, Registry}; + + let format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true); + + let subscriber = Registry::default().with( + fmt::Layer::default() + .with_writer(io::stderr) + .event_format(format), + ); + + tracing::subscriber::set_global_default(subscriber).unwrap(); + + panic_with_log!(Level::ERROR, "Test panicked: {}", 42); + } +} From e26003f4301973cec0bce33942e6ecbae8361ec4 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 20 Nov 2024 00:57:54 -0800 Subject: [PATCH 08/13] ref(global) move modules, extract constants --- .../api_server/src/blog/cron/check_update.rs | 3 +- backend/api_server/src/init.rs | 23 ----- backend/api_server/src/init/database.rs | 38 ++++++++ backend/api_server/src/{ => init}/logger.rs | 60 +++++++------ backend/api_server/src/init/mod.rs | 45 ++++++++++ backend/api_server/src/init/server.rs | 52 +++++++++++ backend/api_server/src/main.rs | 15 +--- backend/api_server/src/server.rs | 88 ------------------- backend/constants/src/config.rs | 43 +++++++++ backend/constants/src/lib.rs | 1 + 10 files changed, 216 insertions(+), 152 deletions(-) delete mode 100644 backend/api_server/src/init.rs create mode 100644 backend/api_server/src/init/database.rs rename backend/api_server/src/{ => init}/logger.rs (63%) create mode 100644 backend/api_server/src/init/mod.rs create mode 100644 backend/api_server/src/init/server.rs delete mode 100644 backend/api_server/src/server.rs create mode 100644 backend/constants/src/config.rs diff --git a/backend/api_server/src/blog/cron/check_update.rs b/backend/api_server/src/blog/cron/check_update.rs index f2fe6a9..5f016d6 100644 --- a/backend/api_server/src/blog/cron/check_update.rs +++ b/backend/api_server/src/blog/cron/check_update.rs @@ -1,6 +1,7 @@ -use dirs::home_dir; use std::env; +use dirs::home_dir; + pub fn task() { // get dir let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| { diff --git a/backend/api_server/src/init.rs b/backend/api_server/src/init.rs deleted file mode 100644 index c240d89..0000000 --- a/backend/api_server/src/init.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::logger; -use std::{env, panic}; - -use tracing_appender::non_blocking::WorkerGuard; - -pub fn logger_init() -> (WorkerGuard, WorkerGuard) { - logger::init() -} - -pub fn load_env() { - if let Some(env_file) = env::args().nth(1) { - match dotenv::from_filename(&env_file) { - Ok(_) => { - println!("[info] env loaded from {}, starting up", env_file); - } - Err(e) => { - panic!("[fatal] failed to read {}: {}", env_file, e); - } - } - } else { - println!("[info] no .env file referred, starting up") - } -} diff --git a/backend/api_server/src/init/database.rs b/backend/api_server/src/init/database.rs new file mode 100644 index 0000000..e403269 --- /dev/null +++ b/backend/api_server/src/init/database.rs @@ -0,0 +1,38 @@ +use std::env; +use std::error::Error; +use std::str::FromStr; + +use constants::config::database::{default_value, var_name}; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, Pool, Postgres}; +use tracing::info; + +pub async fn init() -> Result, Box> { + let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) + .unwrap() + .disable_statement_logging(); + Ok(PgPoolOptions::new().connect_with(db_connect_option).await?) +} + +fn parse_db_uri() -> String { + env::var(var_name::URI).unwrap_or_else(|_| { + info!( + "`{}` env var is not set, \ + using `{}`, `{}`, `{}`, `{}` and `{}`", + var_name::URI, + var_name::USERNAME, + var_name::PASSWORD, + var_name::HOST, + var_name::PORT, + var_name::DBNAME + ); + let postgres_user = env::var(var_name::USERNAME).unwrap_or(default_value::USERNAME.into()); + let postgres_password = + env::var(var_name::PASSWORD).unwrap_or(default_value::PASSWORD.into()); + let db_host = env::var(var_name::HOST).unwrap_or(default_value::HOST.into()); + let db_port = env::var(var_name::PORT).unwrap_or(default_value::PORT.into()); + let postgres_db = env::var(var_name::DBNAME).unwrap_or(default_value::DBNAME.into()); + let db_type = default_value::DATABASE_TYPE; + format!("{db_type}://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") + }) +} diff --git a/backend/api_server/src/logger.rs b/backend/api_server/src/init/logger.rs similarity index 63% rename from backend/api_server/src/logger.rs rename to backend/api_server/src/init/logger.rs index 7dc26e2..5fd56b4 100644 --- a/backend/api_server/src/logger.rs +++ b/backend/api_server/src/init/logger.rs @@ -2,6 +2,7 @@ use std::path::Path; use std::{env, io}; use chrono::Local; +use constants::config::general::{default_value, var_name}; use dirs::home_dir; use tracing::info; use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; @@ -25,33 +26,26 @@ impl FormatTime for LocalTimer { type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; pub fn init() -> (WorkerGuard, WorkerGuard) { - let log_path = &env::var("LOG_ROOT_DIR").unwrap_or_else(|e| { - if let Ok(default_log_path) = env::var("WORK_DIR") { - if default_log_path.starts_with('~') { - let home_path = home_dir() - .expect("[fatal] unable to locate home dir with parsing env `WORK_DIR`, try to use absolute path instead"); - home_path - .join(default_log_path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .expect("[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead") + let log_path = match env::var(var_name::LOG_DIR) { + Ok(path) => Path::new(&path).to_path_buf(), + Err(_) => { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_home_path(default_log_path) } else { - default_log_path - } - } else { - panic!("[fatal] failed to parse env `LOG_ROOT_DIR`: {}", e); + let fallback_path = expand_home_path(default_value::WORK_DIR.to_string()); + println!( + "[warn] unset env var `{}` and `{}`, trying to log on fallback path: {}/log", + var_name::LOG_DIR, + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("log") } - }); - - let log_path = Path::new(log_path) - .join("log") - .join("backend") - .into_os_string() - .into_string() - .unwrap_or_else(|e| panic!("[fatal] failed to parse {:?}", e)); + }; let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); - let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); let info_format = tracing_subscriber::fmt::format() @@ -70,9 +64,6 @@ pub fn init() -> (WorkerGuard, WorkerGuard) { .with_source_location(true) .with_timer(LocalTimer); - // let default_filter = tracing_subscriber::filter::EnvFilter::try_from_default_env() - // .unwrap_or_else(|_| "tower_http=debug,axum::rejection=trace".into()); - let subscribers = Registry::default() .with( fmt::Layer::default() @@ -95,8 +86,21 @@ pub fn init() -> (WorkerGuard, WorkerGuard) { tracing::subscriber::set_global_default(subscribers) .expect("[fatal] failed to setup logging system"); - info!("module started: logger"); - info!("logging to dir: {}", log_path); + info!("logging to dir: {}", log_path.to_str().unwrap()); (info_guard, error_guard) } + +fn expand_home_path(path: String) -> String { + if path.starts_with('~') { + let home_path = home_dir() + .unwrap_or_else(|| panic!("[fatal] unable to locate home dir when parsing env `{}`, try to use absolute path instead", var_name::WORK_DIR)); + home_path + .join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + } else { + path + } +} diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs new file mode 100644 index 0000000..e65de59 --- /dev/null +++ b/backend/api_server/src/init/mod.rs @@ -0,0 +1,45 @@ +mod database; +pub mod logger; +pub mod server; + +use macros::panic_with_log; +use std::{env, panic}; +use tracing::{info, Level}; + +pub fn load_env() { + if let Some(env_file) = env::args().nth(1) { + match dotenv::from_filename(&env_file) { + Ok(_) => { + println!("[info] env loaded from {}, starting up", env_file); + } + Err(e) => { + panic!("[fatal] failed to read {}: {}", env_file, e); + } + } + } else { + println!("[info] no .env file referred, starting up") + } +} + +pub async fn start() { + let db_pool = match database::init().await { + Ok(pool) => { + info!("database connection pool initialized"); + pool + } + Err(e) => { + panic_with_log!( + Level::ERROR, + "database connection pool failed to initialize: {}", + e + ); + } + }; + + match server::start(db_pool).await { + Ok(_) => {} + Err(e) => { + panic_with_log!(Level::ERROR, "server panicked: {}", e) + } + } +} diff --git a/backend/api_server/src/init/server.rs b/backend/api_server/src/init/server.rs new file mode 100644 index 0000000..703a043 --- /dev/null +++ b/backend/api_server/src/init/server.rs @@ -0,0 +1,52 @@ +use std::env; +use std::error::Error; +use std::num::ParseIntError; +use std::process::exit; + +use crate::blog; +use axum::Router; +use constants::config::server::{default_value, var_name}; +use macros::panic_with_log; +use sqlx::{Pool, Postgres}; +use tracing::{error, info, warn, Level}; + +pub async fn start(db_pool: Pool) -> Result<(), Box> { + let app = Router::new() + .nest("/blog", blog::get_router()) + .with_state(db_pool); + + let port: u32 = match env::var(var_name::PORT) { + Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { + warn!( + "failed to parse `{}`, using default port: {}", + var_name::PORT, + e.to_string() + ); + default_value::PORT + }), + Err(_) => default_value::PORT, + }; + + let listener = + match tokio::net::TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await { + Ok(listener) => { + info!("server started on {}:{}", default_value::HOST, port); + listener + } + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to start server on {}:{}: {}", + default_value::HOST, + port, + e.to_string() + ); + } + }; + + axum::serve(listener, app).await.unwrap_or_else(|e| { + error!("failed to start axum: {}", e.to_string()); + exit(1); + }); + Ok(()) +} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index c1431c4..5cdd2e5 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -1,21 +1,12 @@ -use macros::panic_with_log; -use tracing::Level; - mod blog; mod init; -mod logger; mod scheduler; -mod server; #[tokio::main] async fn main() { init::load_env(); - let _guards = init::logger_init(); - match server::start().await { - Ok(_) => {} - Err(e) => { - panic_with_log!(Level::ERROR, "failed to start server: {}", e); - } - }; + let _guards = init::logger::init(); + + init::start().await; } diff --git a/backend/api_server/src/server.rs b/backend/api_server/src/server.rs deleted file mode 100644 index 047eae1..0000000 --- a/backend/api_server/src/server.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::env; -use std::error::Error; -use std::num::ParseIntError; -use std::process::exit; -use std::str::FromStr; - -use crate::blog; -use axum::Router; -use macros::panic_with_log; -use sqlx::{ - postgres::{PgConnectOptions, PgPoolOptions}, - ConnectOptions, -}; -use tracing::{error, info, warn, Level}; - -pub async fn start() -> Result<(), Box> { - let port: u32 = match env::var("BACKEND_PORT") { - Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { - warn!("failed to parse BACKEND_PORT: {}", e.to_string()); - 3001 - }), - Err(_) => 3001, - }; - - let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) - .unwrap() - .disable_statement_logging(); - - let db_pool = match PgPoolOptions::new().connect_with(db_connect_option).await { - Ok(pool) => { - info!("database connected"); - pool - } - Err(e) => { - error!("failed to load database: {}", e.to_string()); - exit(1); - } - }; - - // cron task - // let scheduler = Scheduler::new(); - // let test_str = "hello world".to_string(); - // scheduler.insert( - // BasicTask::new( - // move || blog::cron::check_update::task(test_str.clone()), - // "* * * * * *", - // )? - // .to_task(), - // )?; - - // http server - let app = Router::new() - .nest("/blog", blog::get_router()) - .with_state(db_pool); - - let listener = match tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)).await { - Ok(listener) => { - info!("server started on 0.0.0.0:{}", port); - listener - } - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to start server on 0.0.0.0:{}: {}", - port, - e.to_string() - ); - } - }; - - axum::serve(listener, app).await.unwrap_or_else(|e| { - error!("failed to start axum: {}", e.to_string()); - exit(1); - }); - Ok(()) -} - -fn parse_db_uri() -> String { - let postgres_user = env::var("POSTGRES_USER").unwrap_or("".to_string()); - let postgres_password = env::var("POSTGRES_PASSWORD").unwrap_or("".to_string()); - let db_host = env::var("DB_HOST").unwrap_or("localhost".to_string()); - let db_port = env::var("DB_PORT") - .unwrap_or("5432".to_string()) - .to_string(); - let postgres_db = env::var("POSTGRES_DB").unwrap_or("nocturne".to_string()); - - format!("postgresql://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") -} diff --git a/backend/constants/src/config.rs b/backend/constants/src/config.rs new file mode 100644 index 0000000..4584402 --- /dev/null +++ b/backend/constants/src/config.rs @@ -0,0 +1,43 @@ +// Database related +pub mod database { + pub mod var_name { + pub const URI: &str = "DATABASE_URL"; + pub const USERNAME: &str = "POSTGRES_USER"; + pub const PASSWORD: &str = "POSTGRES_PASSWORD"; + pub const HOST: &str = "POSTGRES_HOST"; + pub const PORT: &str = "POSTGRES_PORT"; + pub const DBNAME: &str = "POSTGRES_DB"; + } + + pub mod default_value { + pub const USERNAME: &str = ""; + pub const PASSWORD: &str = ""; + pub const HOST: &str = "localhost"; + pub const PORT: &str = "5432"; + pub const DBNAME: &str = "nocturne"; + pub const DATABASE_TYPE: &str = "postgresql"; + } +} + +pub mod server { + pub mod var_name { + pub const PORT: &str = "BACKEND_PORT"; + pub const HOST: &str = "BACKEND_HOST"; + } + + pub mod default_value { + pub const PORT: u32 = 3001; + pub const HOST: &str = "0.0.0.0"; + } +} + +pub mod general { + pub mod var_name { + pub const LOG_DIR: &str = "LOG_DIR"; + pub const WORK_DIR: &str = "WORK_DIR"; + } + + pub mod default_value { + pub const WORK_DIR: &str = "~/.nocturne"; + } +} diff --git a/backend/constants/src/lib.rs b/backend/constants/src/lib.rs index f5cd248..1b58b94 100644 --- a/backend/constants/src/lib.rs +++ b/backend/constants/src/lib.rs @@ -1,2 +1,3 @@ pub mod blog; +pub mod config; pub mod time; From 7f400470ffc06075991c4d20819a9373ec8e4683 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 20 Nov 2024 19:26:33 -0800 Subject: [PATCH 09/13] feat(cron) call the module --- .../api_server/src/blog/cron/check_update.rs | 41 +++++++++---------- backend/api_server/src/blog/cron/mod.rs | 9 ++++ backend/api_server/src/init/cron.rs | 11 +++++ backend/api_server/src/init/mod.rs | 8 ++++ backend/api_server/src/scheduler/mod.rs | 4 ++ 5 files changed, 51 insertions(+), 22 deletions(-) create mode 100644 backend/api_server/src/init/cron.rs diff --git a/backend/api_server/src/blog/cron/check_update.rs b/backend/api_server/src/blog/cron/check_update.rs index 5f016d6..3a9f6b8 100644 --- a/backend/api_server/src/blog/cron/check_update.rs +++ b/backend/api_server/src/blog/cron/check_update.rs @@ -1,25 +1,22 @@ -use std::env; - -use dirs::home_dir; - pub fn task() { // get dir - let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| { - if let Ok(default_log_path) = env::var("WORK_DIR") { - if default_log_path.starts_with('~') { - let home_path = home_dir().unwrap_or_else(|| panic!("Cannot find home directory")); - home_path - .join(default_log_path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .expect( - "[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead", - ) - } else { - default_log_path - } - } else { - panic!("[fatal] failed to parse env `GIT_WORK_DIR`: {}", e); - } - }); + // let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| { + // if let Ok(default_log_path) = env::var("WORK_DIR") { + // if default_log_path.starts_with('~') { + // let home_path = home_dir().unwrap_or_else(|| panic!("Cannot find home directory")); + // home_path + // .join(default_log_path.strip_prefix("~/").unwrap()) + // .into_os_string() + // .into_string() + // .expect( + // "[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead", + // ) + // } else { + // default_log_path + // } + // } else { + // panic!("[fatal] failed to parse env `GIT_WORK_DIR`: {}", e); + // } + // }); + println!("check update"); } diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs index be1305e..5f06dc5 100644 --- a/backend/api_server/src/blog/cron/mod.rs +++ b/backend/api_server/src/blog/cron/mod.rs @@ -1 +1,10 @@ +use crate::scheduler::tasks::basic::BasicTask; +use crate::scheduler::tasks::Task; + pub mod check_update; + +pub fn get_tasks() -> Vec> { + vec![BasicTask::new(check_update::task, "* * * * * *") + .unwrap() + .to_task()] +} diff --git a/backend/api_server/src/init/cron.rs b/backend/api_server/src/init/cron.rs new file mode 100644 index 0000000..5b17651 --- /dev/null +++ b/backend/api_server/src/init/cron.rs @@ -0,0 +1,11 @@ +use crate::blog; +use crate::scheduler::Scheduler; + +pub fn start() { + let scheduler = Scheduler::new(); + let _ = vec![blog::cron::get_tasks()] + .into_iter() + .map(|list| scheduler.insert_list(list)) + .collect::>(); + scheduler.start().unwrap() +} diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs index e65de59..f2d02fb 100644 --- a/backend/api_server/src/init/mod.rs +++ b/backend/api_server/src/init/mod.rs @@ -1,3 +1,4 @@ +mod cron; mod database; pub mod logger; pub mod server; @@ -22,6 +23,7 @@ pub fn load_env() { } pub async fn start() { + // connect database let db_pool = match database::init().await { Ok(pool) => { info!("database connection pool initialized"); @@ -36,6 +38,12 @@ pub async fn start() { } }; + // schema init + + // load cron task + cron::start(); + + // api serve match server::start(db_pool).await { Ok(_) => {} Err(e) => { diff --git a/backend/api_server/src/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs index 08f018c..ce86797 100644 --- a/backend/api_server/src/scheduler/mod.rs +++ b/backend/api_server/src/scheduler/mod.rs @@ -157,6 +157,10 @@ impl Scheduler { Ok(id) } + pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { + tasks.into_iter().map(|t| self.insert(t)).collect() + } + pub fn new() -> Self { Self::default() } From 99a19d8d9e7b43256d427ce7bf1b7e714cd9025e Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 20 Nov 2024 21:38:36 -0800 Subject: [PATCH 10/13] feat(init::schema) init database schema --- .../api_server/src/blog/cron/check_update.rs | 4 +- backend/api_server/src/blog/cron/mod.rs | 12 +++-- backend/api_server/src/init/cron.rs | 5 +- backend/api_server/src/init/mod.rs | 4 +- backend/api_server/src/init/schema.rs | 49 +++++++++++++++++++ backend/markdown/src/lib.rs | 10 ++-- 6 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 backend/api_server/src/init/schema.rs diff --git a/backend/api_server/src/blog/cron/check_update.rs b/backend/api_server/src/blog/cron/check_update.rs index 3a9f6b8..ef1eca0 100644 --- a/backend/api_server/src/blog/cron/check_update.rs +++ b/backend/api_server/src/blog/cron/check_update.rs @@ -1,4 +1,6 @@ -pub fn task() { +use sqlx::{Pool, Postgres}; + +pub fn task(db_connection: &Pool) { // get dir // let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| { // if let Ok(default_log_path) = env::var("WORK_DIR") { diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs index 5f06dc5..71581de 100644 --- a/backend/api_server/src/blog/cron/mod.rs +++ b/backend/api_server/src/blog/cron/mod.rs @@ -1,10 +1,14 @@ use crate::scheduler::tasks::basic::BasicTask; use crate::scheduler::tasks::Task; +use sqlx::{Pool, Postgres}; pub mod check_update; -pub fn get_tasks() -> Vec> { - vec![BasicTask::new(check_update::task, "* * * * * *") - .unwrap() - .to_task()] +pub fn get_tasks(db_connection: Pool) -> Vec> { + let db_conn_clone = db_connection.clone(); + vec![ + BasicTask::new(move || check_update::task(&db_conn_clone), "* * * * * *") + .unwrap() + .to_task(), + ] } diff --git a/backend/api_server/src/init/cron.rs b/backend/api_server/src/init/cron.rs index 5b17651..429537a 100644 --- a/backend/api_server/src/init/cron.rs +++ b/backend/api_server/src/init/cron.rs @@ -1,9 +1,10 @@ use crate::blog; use crate::scheduler::Scheduler; +use sqlx::{Pool, Postgres}; -pub fn start() { +pub fn start(db_pool: Pool) { let scheduler = Scheduler::new(); - let _ = vec![blog::cron::get_tasks()] + let _ = vec![blog::cron::get_tasks(db_pool.clone())] .into_iter() .map(|list| scheduler.insert_list(list)) .collect::>(); diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs index f2d02fb..2332914 100644 --- a/backend/api_server/src/init/mod.rs +++ b/backend/api_server/src/init/mod.rs @@ -1,6 +1,7 @@ mod cron; mod database; pub mod logger; +mod schema; pub mod server; use macros::panic_with_log; @@ -39,9 +40,10 @@ pub async fn start() { }; // schema init + schema::init(&db_pool).await; // load cron task - cron::start(); + cron::start(db_pool.clone()); // api serve match server::start(db_pool).await { diff --git a/backend/api_server/src/init/schema.rs b/backend/api_server/src/init/schema.rs new file mode 100644 index 0000000..41ccb34 --- /dev/null +++ b/backend/api_server/src/init/schema.rs @@ -0,0 +1,49 @@ +use sqlx::{query, Pool, Postgres}; +use tracing::info; + +const POST_BASE_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Post +( + post_id UUID PRIMARY KEY, + title VARCHAR(255) NOT NULL, + category VARCHAR(255) NOT NULL, + header_img TEXT, + summary TEXT, + sub_title TEXT, + content TEXT NOT NULL, + last_update TIMESTAMPTZ NOT NULL, + first_update TIMESTAMPTZ NOT NULL +); +"##; + +// base model, will add more cols at runtime +const POST_META_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Meta +( + post_id UUID PRIMARY KEY +); +"##; + +const POST_HASH_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Hash +( + post_id UUID PRIMARY KEY, + hash VARCHAR(44) +); +"##; + +const TAG_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Tag +( + post_id UUID NOT NULL, + tag VARCHAR(255) NOT NULL +); +"##; + +pub async fn init(db_connection: &Pool) { + query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); + query(POST_META_MODEL).execute(db_connection).await.unwrap(); + query(TAG_MODEL).execute(db_connection).await.unwrap(); + query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); + info!("database schema initialized"); +} diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index ce6345f..b1e3562 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -15,11 +15,11 @@ use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; #[derive(Default, Debug, Serialize)] pub struct MdFile { - file_id: Uuid, - file_name: String, - meta: Value, - content: String, - hash: String, + pub file_id: Uuid, + pub file_name: String, + pub meta: Value, + pub content: String, + pub hash: String, } impl MdFile { From bacaea80a308160052daea390c19172d9f0b6364 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 27 Nov 2024 00:18:08 -0800 Subject: [PATCH 11/13] feat(cron::update) auto update git repo and update db index --- backend/api_server/Cargo.toml | 5 +- .../api_server/src/blog/cron/check_update.rs | 24 -- .../src/blog/cron/check_update/error.rs | 49 +++ .../src/blog/cron/check_update/index.rs | 333 ++++++++++++++++++ .../src/blog/cron/check_update/mod.rs | 144 ++++++++ .../src/blog/cron/check_update/utils.rs | 86 +++++ backend/api_server/src/init/misc.rs | 14 + backend/api_server/src/init/mod.rs | 3 + backend/api_server/src/init/schema.rs | 4 +- backend/api_server/src/main.rs | 1 + backend/api_server/src/utils/git.rs | 195 ++++++++++ backend/api_server/src/utils/mod.rs | 1 + backend/constants/src/config.rs | 5 + backend/constants/src/time.rs | 1 + backend/markdown/Cargo.toml | 1 + backend/markdown/src/error.rs | 16 + backend/markdown/src/lib.rs | 31 +- backend/markdown/src/utils/front_matter.rs | 8 +- 18 files changed, 882 insertions(+), 39 deletions(-) delete mode 100644 backend/api_server/src/blog/cron/check_update.rs create mode 100644 backend/api_server/src/blog/cron/check_update/error.rs create mode 100644 backend/api_server/src/blog/cron/check_update/index.rs create mode 100644 backend/api_server/src/blog/cron/check_update/mod.rs create mode 100644 backend/api_server/src/blog/cron/check_update/utils.rs create mode 100644 backend/api_server/src/init/misc.rs create mode 100644 backend/api_server/src/utils/git.rs create mode 100644 backend/api_server/src/utils/mod.rs create mode 100644 backend/markdown/src/error.rs diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index e140b1e..6307f72 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] macros = { path = "../macros" } constants = { path = "../constants" } +markdown = { path = "../markdown" } chrono = { version = "0.4.38", features = ["serde", "alloc"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } @@ -19,6 +20,6 @@ serde = { version = "1.0.199", features = ["derive"] } serde_json = "1.0.128" uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } futures = { version = "0.3.30", features = ["executor"] } -sha2 = "0.10.8" -cron = "0.12.1" +cron = "0.13.0" git2 = "0.19.0" +thiserror = "2.0.3" diff --git a/backend/api_server/src/blog/cron/check_update.rs b/backend/api_server/src/blog/cron/check_update.rs deleted file mode 100644 index ef1eca0..0000000 --- a/backend/api_server/src/blog/cron/check_update.rs +++ /dev/null @@ -1,24 +0,0 @@ -use sqlx::{Pool, Postgres}; - -pub fn task(db_connection: &Pool) { - // get dir - // let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| { - // if let Ok(default_log_path) = env::var("WORK_DIR") { - // if default_log_path.starts_with('~') { - // let home_path = home_dir().unwrap_or_else(|| panic!("Cannot find home directory")); - // home_path - // .join(default_log_path.strip_prefix("~/").unwrap()) - // .into_os_string() - // .into_string() - // .expect( - // "[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead", - // ) - // } else { - // default_log_path - // } - // } else { - // panic!("[fatal] failed to parse env `GIT_WORK_DIR`: {}", e); - // } - // }); - println!("check update"); -} diff --git a/backend/api_server/src/blog/cron/check_update/error.rs b/backend/api_server/src/blog/cron/check_update/error.rs new file mode 100644 index 0000000..0d27c29 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/error.rs @@ -0,0 +1,49 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PostIdxError { + #[error("missing essential field `{}` for file `{}`", field, filename)] + MissingField { field: String, filename: String }, + + #[error( + "invalid field `{}` for file `{}`, <{}>: {}", + field_name, + filename, + err_field, + msg + )] + InvalidField { + field_name: String, + err_field: String, + filename: String, + msg: String, + }, + + #[error( + "database error on writing `{}` to table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBWriteFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, + + #[error( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBReadFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, +} diff --git a/backend/api_server/src/blog/cron/check_update/index.rs b/backend/api_server/src/blog/cron/check_update/index.rs new file mode 100644 index 0000000..8bfe006 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/index.rs @@ -0,0 +1,333 @@ +use std::error::Error; +use std::path::Path; + +use crate::blog::cron::check_update::utils::get_md_file_basename; +use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date}; +use markdown::MdFile; + +use sqlx::{query, PgPool, Pool, Postgres, Row}; +use tokio::runtime::Runtime; +use tracing::{info, warn}; + +async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + let post_id = post.file_id; + let title = match post.meta.get("title") { + None => { + return Err(PostIdxError::MissingField { + field: "title".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid title for `{}`", &post.filename); + "invalid title" + }), + }; + let sub_title = post + .meta + .get("sub_title") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let summary = post + .meta + .get("summary") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let content = post.content.as_str(); + let category = match post.meta.get("category") { + None => { + return Err(PostIdxError::MissingField { + field: "category".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid category for `{}`", &post.filename); + "invalid category" + }), + }; + let last_update = { + if let Some(last_update) = post.meta.get("last_update") { + match parse_date(last_update.as_str().unwrap()) { + Ok(date) => Some(date), + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "last_update".to_string(), + err_field: last_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + } + } else { + None + } + }; + let first_update = match post.meta.get("date") { + None => { + return Err(PostIdxError::MissingField { + field: "date".to_string(), + filename: post.filename.clone(), + }) + } + Some(first_update) => match parse_date(first_update.as_str().unwrap()) { + Ok(date) => date, + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "date".to_string(), + err_field: first_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + }, + }; + let filename = get_md_file_basename(&post.filename); + + if let Err(e) = query( + r##" +INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (post_id) DO +UPDATE SET + title = EXCLUDED.title, + summary = EXCLUDED.summary, + content = EXCLUDED.content, + last_update = EXCLUDED.last_update, + first_update = EXCLUDED.first_update, + sub_title = EXCLUDED.sub_title, + category = EXCLUDED.category, + filename = EXCLUDED.filename; + "##, + ) + .bind(post_id) + .bind(title) + .bind(summary) + .bind(content) + .bind(last_update) + .bind(first_update) + .bind(sub_title) + .bind(category) + .bind(filename) + .execute(db_connection) + .await + { + return Err( + PostIdxError::DBWriteFailure { + data_desc: "post base information".to_string(), + db_table: "Post".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + } + ); + } + Ok(()) +} + +async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Some(tags_string) = post.meta.get("tags").to_owned() { + let tags = match tags_string.as_array().map(|vec| { + vec.iter() + .map(|val| val.as_str().unwrap()) + .collect::>() + }) { + None => { + return Err(PostIdxError::MissingField { + field: "tags".to_string(), + filename: post.filename.clone(), + }) + } + Some(val) => val, + }; + + let post_id = post.file_id; + + // remove old record + if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") + .bind(post_id) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }); + } + + // add new record + for tag in tags { + match query("INSERT INTO Tag VALUES ($1, $2);") + .bind(post_id) + .bind(tag) + .execute(db_connection) + .await + { + Ok(_) => {} + Err(e) => { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }) + } + } + } + } + + Ok(()) +} + +async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { + let base64_form = post.hash.clone(); + match query("SELECT hash FROM Hash WHERE post_id = $1") + .bind(post.file_id) + .fetch_optional(db_connection) + .await + { + Ok(row) => { + let db_record: String = match row { + None => "".to_string(), + Some(row) => row.try_get("hash").unwrap_or("".to_string()), + }; + if base64_form == db_record { + Ok(false) + } else { + if let Err(e) = query( + r#" +INSERT INTO Hash (post_id, hash) VALUES ($1, $2) + ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, + ) + .bind(post.file_id) + .bind(&base64_form) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }); + } + Ok(true) + } + } + Err(e) => Err(PostIdxError::DBReadFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }), + } +} + +async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Err(e) = query( + // todo: sanitization + format!( + r#" +DO +$$ + DECLARE + _post_id UUID := '{}'; + _meta_cols JSONB := '{}'; + _exists BOOLEAN; + _col_name TEXT; + _col_value TEXT; + _sql TEXT; + _insert_columns TEXT := 'post_id'; + _insert_values TEXT := quote_literal(_post_id); + _update_set TEXT := ''; + _blacklist TEXT[] := ARRAY['tags']; + BEGIN + SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; + + FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) + LOOP + IF _col_name = ANY(_blacklist) THEN + CONTINUE; + END IF; + + IF NOT EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'meta' + AND column_name = _col_name + -- may change in the future + AND table_schema = 'public') THEN + _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; + EXECUTE _sql; + END IF; + _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); + _insert_values := _insert_values || ', ' || quote_literal(_col_value); + _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); + END LOOP; + + _update_set := substring(_update_set FROM 3); + + IF _exists THEN + _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); + EXECUTE _sql; + ELSE + _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; + EXECUTE _sql; + END IF; + END; +$$ LANGUAGE plpgsql; + "#, post.file_id.to_string().as_str(), post.meta + ).as_str() + ) + .execute(db_connection) + .await { + Err(PostIdxError::DBWriteFailure { + data_desc: "post meta data".to_string(), + db_table: "Meta".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }) + } else { + Ok(()) + } +} + +pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { + let md_file = MdFile::from_file(path)?; + + base(db_conn, &md_file).await?; + tag(db_conn, &md_file).await?; + meta(db_conn, &md_file).await?; + hash(db_conn, &md_file).await?; + + info!("indexed markdown file: {}", path.display()); + Ok(()) +} + +pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + if let Err(e) = query( + r#" +WITH deleted_post AS ( + DELETE FROM post + WHERE id = (SELECT id FROM post WHERE filename = $1) + RETURNING id +) +DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); + "#, + ) + .bind(file_basename) + .execute(db_conn) + .await + { + Err(Box::from( + format!("failed to drop index: {}", e).to_string(), + )) + } else { + Ok(()) + } + }) +} diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs new file mode 100644 index 0000000..63ca30e --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/mod.rs @@ -0,0 +1,144 @@ +use std::env; +use std::fs::remove_dir_all; +use std::path::{Path, PathBuf}; + +use crate::{ + blog::cron::check_update::{ + index::drop_index, + utils::{expand_path, get_md_file_basename, index_files, search_md}, + }, + utils::git, +}; +use constants::config::general::{default_value, var_name}; +use macros::panic_with_log; + +use git2::{Delta, Repository}; +use sqlx::{Pool, Postgres}; +use tracing::{error, info, warn, Level}; + +mod error; +mod index; +mod utils; + +pub fn task(db_connection: &Pool) { + // init + let git_work_dir = { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_path(default_log_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") + }; + + // clone & check update + + let git_url = match env::var(var_name::BLOG_GIT_URL) { + Ok(url) => url, + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to read `{}`: {}", + var_name::BLOG_GIT_URL, + e + ) + } + }; + let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) + .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); + let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) + .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); + + match Repository::open(&git_work_dir) { + Ok(repo) => { + // prevent git remote url mismatch (clean cache) + match repo.find_remote(&git_remote_name) { + Ok(remote) => { + if remote.url().is_some_and(|url| url != git_url) { + remove_dir_all(&git_work_dir).unwrap(); + warn!("blog content git({}) url mismatch, reinitialized", &git_url); + return; + } + } + Err(e) => { + remove_dir_all(&git_work_dir).unwrap(); + error!( + "unexpected error on indexing blog content git({}), reinitialized: {}", + &git_url, e + ); + } + } + + match git::sync(&git_remote_name, &git_remote_branch, &repo) { + Ok(updates) => { + if !updates.is_empty() { + // incremental update + info!( + "blog content git({}) update detected: {:?}", + &git_url, updates + ); + + let mut update_md: Vec = vec![]; + + for update in updates { + if [Delta::Deleted, Delta::Untracked].contains(&update.status) { + if let Some(path) = &update.old_path { + if drop_index( + &get_md_file_basename( + path.file_name().unwrap().to_str().unwrap(), + ), + db_connection, + ) + .is_ok() + { + info!( + "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", + &git_url, + &path.display() + ); + } else { + error!( + "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", + &git_url, + &path.display() + ) + } + } + } else if let Some(path) = &update.new_path { + if path.extension().is_some_and(|ext| ext == "md") { + update_md.push(path.to_path_buf()); + } + } + } + + index_files(&update_md, db_connection); + } + } + Err(e) => { + error!( + "failed to sync blog content git({}) updates: {}", + &git_url, e + ); + } + } + } + Err(_) => { + match Repository::clone(&git_url, &git_work_dir) { + Ok(_repo) => { + // full repo index + let files = search_md(git_work_dir); + index_files(&files, db_connection); + } + Err(e) => { + error!("failed to clone blog content git({}): {}", &git_url, e); + } + } + } + }; +} diff --git a/backend/api_server/src/blog/cron/check_update/utils.rs b/backend/api_server/src/blog/cron/check_update/utils.rs new file mode 100644 index 0000000..d0da864 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/utils.rs @@ -0,0 +1,86 @@ +use dirs::home_dir; +use std::path::PathBuf; + +use crate::blog::cron::check_update::index::index_file; +use chrono::{DateTime, FixedOffset, ParseResult}; +use constants::config::general::{default_value, var_name}; +use constants::time::DEFAULT_DATE_FORMAT; + +use futures::executor::block_on; +use futures::future::join_all; +use sqlx::{Pool, Postgres}; +use tokio::runtime::Runtime; +use tokio::spawn; +use tracing::{error, warn}; + +pub fn search_md(entry: PathBuf) -> Vec { + if entry.ends_with(".git") { + return vec![]; + } + if entry.is_dir() { + if let Ok(files) = entry.read_dir() { + files + .flat_map(|file| { + let file_path = file.unwrap().path(); + search_md(file_path) + }) + .collect() + } else { + vec![] + } + } else if entry.extension().is_some_and(|ext| ext == "md") { + vec![entry] + } else { + vec![] + } +} + +pub fn index_files(files: &[PathBuf], db_conn: &Pool) { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let files_clone = files.to_owned(); + let handlers = files_clone + .into_iter() + .map(|file: PathBuf| { + let conn_clone = db_conn.clone(); + spawn(async move { + if let Err(e) = index_file(&file, &conn_clone).await { + error!("failed to index markdown file `{}`: {}", file.display(), e); + } + }) + }) + .collect::>(); + + block_on(join_all(handlers)); + }); +} + +pub fn expand_path(path: String) -> String { + if path.starts_with('~') { + home_dir() + .map(|dir| { + dir.join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + }) + .unwrap_or_else(|| { + warn!( + "failed to parse env var `{}`, using default value `{}`", + var_name::WORK_DIR, + default_value::WORK_DIR + ); + default_value::WORK_DIR.to_string() + }) + } else { + path + } +} + +pub fn parse_date(date_str: &str) -> ParseResult> { + DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) +} + +pub fn get_md_file_basename(f_name: &str) -> String { + f_name.replace(".md", "") +} diff --git a/backend/api_server/src/init/misc.rs b/backend/api_server/src/init/misc.rs new file mode 100644 index 0000000..6a08bc2 --- /dev/null +++ b/backend/api_server/src/init/misc.rs @@ -0,0 +1,14 @@ +use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds}; +use tracing::warn; + +pub fn init() { + // git2 option + unsafe { + if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) { + warn!("failed to set git2 server connect timeout: {}", e); + } + if let Err(e) = set_server_timeout_in_milliseconds(60_000) { + warn!("failed to set git2 server timeout: {}", e); + } + } +} diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs index 2332914..507d52a 100644 --- a/backend/api_server/src/init/mod.rs +++ b/backend/api_server/src/init/mod.rs @@ -1,6 +1,7 @@ mod cron; mod database; pub mod logger; +mod misc; mod schema; pub mod server; @@ -24,6 +25,8 @@ pub fn load_env() { } pub async fn start() { + misc::init(); + // connect database let db_pool = match database::init().await { Ok(pool) => { diff --git a/backend/api_server/src/init/schema.rs b/backend/api_server/src/init/schema.rs index 41ccb34..ef9def0 100644 --- a/backend/api_server/src/init/schema.rs +++ b/backend/api_server/src/init/schema.rs @@ -7,11 +7,11 @@ CREATE TABLE IF NOT EXISTS Post post_id UUID PRIMARY KEY, title VARCHAR(255) NOT NULL, category VARCHAR(255) NOT NULL, - header_img TEXT, + filename TEXT UNIQUE NOT NULL, summary TEXT, sub_title TEXT, content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, + last_update TIMESTAMPTZ, first_update TIMESTAMPTZ NOT NULL ); "##; diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index 5cdd2e5..8aab2c7 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -1,6 +1,7 @@ mod blog; mod init; mod scheduler; +mod utils; #[tokio::main] async fn main() { diff --git a/backend/api_server/src/utils/git.rs b/backend/api_server/src/utils/git.rs new file mode 100644 index 0000000..6743058 --- /dev/null +++ b/backend/api_server/src/utils/git.rs @@ -0,0 +1,195 @@ +/// git merge +/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs +/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 +/// +use git2::{ + build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, +}; +use std::path::PathBuf; +use tracing::{error, warn}; + + +#[derive(Debug, Eq, PartialEq)] +pub struct FileDelta { + pub old_path: Option, + pub new_path: Option, + pub status: Delta, +} + +fn do_fetch<'a>( + repo: &'a Repository, + refs: &[&str], + remote: &'a mut Remote, +) -> Result, git2::Error> { + let mut fetch_option = FetchOptions::new(); + + // Always fetch all tags. + // Perform a download and also update tips + fetch_option.download_tags(AutotagOption::All); + remote.fetch(refs, Some(&mut fetch_option), None)?; + + let fetch_head = repo.find_reference("FETCH_HEAD")?; + repo.reference_to_annotated_commit(&fetch_head) +} + +fn fast_forward( + repo: &Repository, + lb: &mut Reference, + rc: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let name = match lb.name() { + Some(s) => s.to_string(), + None => String::from_utf8_lossy(lb.name_bytes()).to_string(), + }; + let msg = format!("fast-forward({}) -> {}", name, rc.id()); + // println!("{}", msg); + lb.set_target(rc.id(), &msg)?; + repo.set_head(&name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + // For some reason, the force is required to make the working directory actually get updated + // I suspect we should be adding some logic to handle dirty working directory states, + // but this is just an example so maybe not. + .force(), + ))?; + Ok(()) +} + +fn normal_merge( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let ancestor = repo + .find_commit(repo.merge_base(local.id(), remote.id())?)? + .tree()?; + let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?; + + if idx.has_conflicts() { + warn!("merge conflicts detected"); + repo.checkout_index(Some(&mut idx), None)?; + return Ok(()); + } + let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; + // now create the merge commit + let msg = format!("merge({}) -> {}", remote.id(), local.id()); + let sig = repo.signature()?; + let local_commit = repo.find_commit(local.id())?; + let remote_commit = repo.find_commit(remote.id())?; + + // Do our merge commit and set the current branch head to that commit. + let _merge_commit = repo.commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &result_tree, + &[&local_commit, &remote_commit], + )?; + // Set working tree to match head. + repo.checkout_head(None)?; + Ok(()) +} + +fn do_merge<'a>( + repo: &'a Repository, + remote_branch: &str, + fetch_commit: AnnotatedCommit<'a>, +) -> Result, git2::Error> { + // 1. do a merge analysis + let analysis = repo.merge_analysis(&[&fetch_commit])?; + + let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; + + // 2. Do the appropriate merge + if analysis.0.is_fast_forward() { + // println!("Doing a fast-forward"); + // do a fast-forward + let ref_name = format!("refs/heads/{}", remote_branch); + match repo.find_reference(&ref_name) { + Ok(mut r) => { + fast_forward(repo, &mut r, &fetch_commit)?; + } + Err(_) => { + // The branch doesn't exist, so set the reference to the + // commit directly. Usually this is because you are pulling + // into an empty repository. + repo.reference( + &ref_name, + fetch_commit.id(), + true, + &format!("({}) -> {}", remote_branch, fetch_commit.id()), + )?; + repo.set_head(&ref_name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + .allow_conflicts(true) + .conflict_style_merge(true) + .force(), + ))?; + } + }; + } else if analysis.0.is_normal() { + // do a normal merge + normal_merge(repo, &head_commit, &fetch_commit)?; + } + // else { + // // already up-to-date + // println!("Nothing to do..."); + // } + + calculate_diff(repo, &head_commit, &fetch_commit) +} + +fn calculate_diff( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result, git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?; + + let mut updated_files: Vec = Vec::new(); + + if let Err(e) = diff.foreach( + &mut |d, _| { + // let delta = d.status(); + // + // let n_path = d.new_file().path(); + // let o_path = d.old_file().path(); + // + // if let Some(path) = n_path { + // updated_files.insert(path.to_owned()); + // } + // if let Some(path) = o_path { + // updated_files.insert(path.to_owned()); + // } + updated_files.push(FileDelta { + old_path: d.old_file().path().map(PathBuf::from), + new_path: d.new_file().path().map(PathBuf::from), + status: d.status(), + }); + true // Continue + }, + None, + None, + None, + ) { + error!("error at calculating diff: {:?}", e); + } + + Ok(updated_files.into_iter().collect()) +} + +pub fn sync( + remote_name: &str, + remote_branch: &str, + repo: &Repository, +) -> Result, git2::Error> { + let mut remote = repo.find_remote(remote_name)?; + let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; + do_merge(repo, remote_branch, fetch_commit) +} diff --git a/backend/api_server/src/utils/mod.rs b/backend/api_server/src/utils/mod.rs new file mode 100644 index 0000000..c2bf1c3 --- /dev/null +++ b/backend/api_server/src/utils/mod.rs @@ -0,0 +1 @@ +pub mod git; diff --git a/backend/constants/src/config.rs b/backend/constants/src/config.rs index 4584402..bf6ba77 100644 --- a/backend/constants/src/config.rs +++ b/backend/constants/src/config.rs @@ -35,9 +35,14 @@ pub mod general { pub mod var_name { pub const LOG_DIR: &str = "LOG_DIR"; pub const WORK_DIR: &str = "WORK_DIR"; + pub const BLOG_GIT_URL: &str = "BLOG_GIT_URL"; + pub const BLOG_GIT_REMOTE_BRANCH: &str = "BLOG_GIT_REMOTE_BRANCH"; + pub const BLOG_GIT_REMOTE_NAME: &str = "BLOG_GIT_REMOTE_NAME"; } pub mod default_value { pub const WORK_DIR: &str = "~/.nocturne"; + pub const BLOG_GIT_REMOTE_BRANCH: &str = "master"; + pub const BLOG_GIT_REMOTE_NAME: &str = "origin"; } } diff --git a/backend/constants/src/time.rs b/backend/constants/src/time.rs index 89e7f55..d7b29ac 100644 --- a/backend/constants/src/time.rs +++ b/backend/constants/src/time.rs @@ -1 +1,2 @@ pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; +pub const DEFAULT_DATE_FORMAT: &str = "%Y-%m-%d%zT%H%M%S"; diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index 0d0b372..a7d8aef 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -12,3 +12,4 @@ regex = "1.10.4" yaml-rust2 = "0.9.0" sha2 = "0.10.8" base64 = "0.22.1" +thiserror = "2.0.3" diff --git a/backend/markdown/src/error.rs b/backend/markdown/src/error.rs new file mode 100644 index 0000000..a93585f --- /dev/null +++ b/backend/markdown/src/error.rs @@ -0,0 +1,16 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("missing front-matter field: id")] + MissingID, + + #[error("invalid file id: {}", msg)] + InvalidID { msg: String }, + + #[error("invalid front-matter: {}", msg)] + InvalidFrontMatter { msg: String }, + + #[error("fs error: {}", msg)] + FSError { msg: String }, +} diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index b1e3562..518d162 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -1,35 +1,48 @@ +mod error; mod utils; -use std::error::Error; use std::fs; use std::path::Path; use std::str::FromStr; +use crate::error::Error; +use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + use base64::{engine::general_purpose, Engine}; use serde::Serialize; use serde_json::Value; use sha2::{Digest, Sha256}; use uuid::Uuid; -use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; - #[derive(Default, Debug, Serialize)] pub struct MdFile { pub file_id: Uuid, - pub file_name: String, + pub filename: String, pub meta: Value, pub content: String, pub hash: String, } impl MdFile { - pub async fn from_file(file_path: &str) -> Result> { - let raw_file = fs::read_to_string(file_path)?; + pub fn from_file(file_path: &Path) -> Result { + let raw_file = match fs::read_to_string(file_path) { + Ok(file) => file, + Err(e) => return Err(Error::FSError { msg: e.to_string() }), + }; let raw_file_bytes = raw_file.as_bytes().to_vec(); let (front_matter_str, content) = split_md_front_matter(raw_file); - let front_matter = parse_front_matter(&front_matter_str)?; + let front_matter = match parse_front_matter(&front_matter_str) { + Ok(front_matter) => front_matter, + Err(e) => return Err(Error::InvalidFrontMatter { msg: e.to_string() }), + }; let hash = calculate_hash_base64(&raw_file_bytes); - let file_id = Uuid::from_str(front_matter["id"].to_string().as_str())?; + let file_id = match front_matter["id"].as_str() { + Some(id) => match Uuid::from_str(id) { + Ok(id) => id, + Err(e) => return Err(Error::InvalidID { msg: e.to_string() }), + }, + None => return Err(Error::MissingID), + }; let file_name = Path::new(file_path) .file_name() .unwrap() @@ -39,7 +52,7 @@ impl MdFile { Ok(MdFile { file_id, - file_name, + filename: file_name, meta: front_matter, content, hash, diff --git a/backend/markdown/src/utils/front_matter.rs b/backend/markdown/src/utils/front_matter.rs index f38f029..5bf77d2 100644 --- a/backend/markdown/src/utils/front_matter.rs +++ b/backend/markdown/src/utils/front_matter.rs @@ -16,8 +16,12 @@ pub fn split_md_front_matter(raw_file: String) -> (String, String) { } pub fn parse_front_matter(front_matter: &str) -> Result> { - let parsed_f_matter = YamlLoader::load_from_str(front_matter)?[0].clone(); - Ok(Value::from(yaml_to_serde(&parsed_f_matter))) + let parsed_f_matter = YamlLoader::load_from_str(front_matter)?; + if parsed_f_matter.is_empty() { + Ok(Value::String(String::new())) + } else { + Ok(yaml_to_serde(&parsed_f_matter[0])) + } } #[cfg(test)] From cbc831bb70c71d5ed6dbc8f1d2875ac98d9ceb09 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 27 Nov 2024 04:00:38 -0800 Subject: [PATCH 12/13] fix(cron::update) fix several errors --- .../src/blog/cron/check_update/index.rs | 26 ++++++++++++++++--- .../src/blog/cron/check_update/mod.rs | 6 +++-- backend/api_server/src/blog/cron/mod.rs | 2 +- backend/markdown/Cargo.toml | 1 - 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/backend/api_server/src/blog/cron/check_update/index.rs b/backend/api_server/src/blog/cron/check_update/index.rs index 8bfe006..f391639 100644 --- a/backend/api_server/src/blog/cron/check_update/index.rs +++ b/backend/api_server/src/blog/cron/check_update/index.rs @@ -295,10 +295,28 @@ $$ LANGUAGE plpgsql; pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { let md_file = MdFile::from_file(path)?; - base(db_conn, &md_file).await?; - tag(db_conn, &md_file).await?; - meta(db_conn, &md_file).await?; - hash(db_conn, &md_file).await?; + async fn trail(db_conn: &Pool, md_file: &MdFile) -> Result<(), PostIdxError> { + base(db_conn, md_file).await?; + tag(db_conn, md_file).await?; + meta(db_conn, md_file).await?; + hash(db_conn, md_file).await.map(|_| ()) + } + let mut cnt = 5; + + while let Err(e) = trail(db_conn, &md_file).await { + match e { + PostIdxError::DBWriteFailure { .. } => { + cnt -= 1; + } + PostIdxError::DBReadFailure { .. } => { + cnt -= 1; + } + _ => return Err(e.into()), + } + if cnt <= 0 { + return Err(e.into()); + } + } info!("indexed markdown file: {}", path.display()); Ok(()) diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs index 63ca30e..9568ae2 100644 --- a/backend/api_server/src/blog/cron/check_update/mod.rs +++ b/backend/api_server/src/blog/cron/check_update/mod.rs @@ -111,7 +111,9 @@ pub fn task(db_connection: &Pool) { } } } else if let Some(path) = &update.new_path { - if path.extension().is_some_and(|ext| ext == "md") { + if path.components().any(|comp| comp.as_os_str() == "posts") + && path.extension().is_some_and(|ext| ext == "md") + { update_md.push(path.to_path_buf()); } } @@ -132,7 +134,7 @@ pub fn task(db_connection: &Pool) { match Repository::clone(&git_url, &git_work_dir) { Ok(_repo) => { // full repo index - let files = search_md(git_work_dir); + let files = search_md(git_work_dir.join("posts")); index_files(&files, db_connection); } Err(e) => { diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs index 71581de..e8fd785 100644 --- a/backend/api_server/src/blog/cron/mod.rs +++ b/backend/api_server/src/blog/cron/mod.rs @@ -7,7 +7,7 @@ pub mod check_update; pub fn get_tasks(db_connection: Pool) -> Vec> { let db_conn_clone = db_connection.clone(); vec![ - BasicTask::new(move || check_update::task(&db_conn_clone), "* * * * * *") + BasicTask::new(move || check_update::task(&db_conn_clone), "0 * * * * *") .unwrap() .to_task(), ] diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index a7d8aef..e0ca45e 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -5,7 +5,6 @@ edition = "2021" [dependencies] uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } serde = { version = "1.0.209", features = ["derive"] } serde_json = "1.0.128" regex = "1.10.4" From a0dc40d2ca7fc42dab0fa263584d86325182f697 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Sun, 1 Dec 2024 02:35:13 -0800 Subject: [PATCH 13/13] feat(blog::api) fix apis, frontend --- backend/api_server/Cargo.toml | 2 +- .../api_server/src/blog/api/get_post_list.rs | 60 +++++- backend/api_server/src/blog/api/mod.rs | 1 - backend/api_server/src/blog/api/refresh.rs | 190 ------------------ backend/api_server/src/blog/mod.rs | 4 +- backend/api_server/src/init/database.rs | 5 +- frontend/app/blog/BlogPostCard.tsx | 6 +- frontend/app/blog/PostList.tsx | 4 +- frontend/app/blog/page.tsx | 12 +- frontend/app/blog/post/[id]/Headers.tsx | 8 +- frontend/app/blog/post/[id]/PostCard.tsx | 2 +- frontend/app/blog/post/[id]/page.tsx | 8 +- frontend/components/ThemeSwitcher.tsx | 4 +- frontend/package.json | 2 +- 14 files changed, 82 insertions(+), 226 deletions(-) delete mode 100644 backend/api_server/src/blog/api/refresh.rs diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index 6307f72..af17326 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -15,7 +15,7 @@ dotenv = "0.15.0" dirs = "5.0.1" tokio = { version = "1.40.0", features = ["full"] } sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -axum = "0.7.5" +axum = { version = "0.7.5", features = ["macros"] } serde = { version = "1.0.199", features = ["derive"] } serde_json = "1.0.128" uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } diff --git a/backend/api_server/src/blog/api/get_post_list.rs b/backend/api_server/src/blog/api/get_post_list.rs index 854f495..9db7c61 100644 --- a/backend/api_server/src/blog/api/get_post_list.rs +++ b/backend/api_server/src/blog/api/get_post_list.rs @@ -1,3 +1,5 @@ +use std::error::Error; + use constants::blog::PAGE_SIZE; use axum::{extract::Query, extract::State, http::StatusCode, Json}; @@ -14,6 +16,7 @@ pub struct Params { page: Option, } +#[axum::debug_handler] pub async fn get_post_list( State(db_connection): State, Query(params): Query, @@ -47,33 +50,72 @@ pub async fn get_post_list( } }; - let mut res = Map::new(); + let mut res = Vec::new(); let posts: Vec<_> = post_list .into_iter() - .map(|post| analyze_post(&db_connection, post)) + .map(|post| async { analyze_post(&db_connection, post).await }) .collect(); - let posts = join_all(posts).await; + for post in posts { + match post { + Ok(p) => res.push(p), + Err(e) => { + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!(("error", e.to_string()))), + )) + } + } + } - res.insert("posts".to_string(), Value::from(posts)); + // res.insert("posts".to_string(), Value::from(posts)); Ok(Json::from(Value::from(res))) } -async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Value { +async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result> { let post_id: Uuid = post.get("post_id"); let tags = fetch_post_tags(db_connection, post_id).await; + let meta = fetch_post_meta(db_connection, post_id).await.unwrap(); + let header_img = meta.get("header_img"); + let last_update = meta.get("last_update"); - json!({ + Ok(json!({ "post_id": post_id.to_string(), "title": post.get::("title"), "category": post.get::("category"), - "header_img": post.get::("header_img"), + "header_img": header_img, "sub_title": post.get::("sub_title"), "summary": post.get::("summary"), - "last_update": post.get::, _>("last_update"), + "last_update": last_update, "first_update": post.get::, _>("first_update"), "tags": tags, - }) + })) +} + +async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option { + match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta +FROM meta t +WHERE post_id = $1; + "#, + ) + .bind(post_id) + .fetch_one(db_connection) + .await + { + Ok(value) => Some(value.get::("meta")), + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "post meta data", + "meta", + post_id.to_string(), + e.to_string() + ); + None + } + } } async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec { diff --git a/backend/api_server/src/blog/api/mod.rs b/backend/api_server/src/blog/api/mod.rs index e946d12..5e23c3d 100644 --- a/backend/api_server/src/blog/api/mod.rs +++ b/backend/api_server/src/blog/api/mod.rs @@ -1,4 +1,3 @@ pub mod get_page_count; pub mod get_post; pub mod get_post_list; -pub mod refresh; diff --git a/backend/api_server/src/blog/api/refresh.rs b/backend/api_server/src/blog/api/refresh.rs deleted file mode 100644 index cc7b331..0000000 --- a/backend/api_server/src/blog/api/refresh.rs +++ /dev/null @@ -1,190 +0,0 @@ -use std::path::Path; -use std::str::FromStr; - -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::Json; -use serde::Deserialize; -use serde_json::{Map, Value}; -use sqlx::PgPool; - -// use crate::api::post::errors::refresh::PostIdxError; -// use crate::api::post::utils::{build_post_idx, post}; -// use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL}; - -#[derive(Deserialize)] -pub struct Params { - /// refresh level - level: Option, -} - -pub async fn refresh( - State(db_connection): State, - Query(params): Query, -) -> Result)> { - // let is_force_refresh = params.level.unwrap_or("".to_string()) == "force"; - // - // // search post - // let data_repo_dir = match env::var("GIT_WORK_DIR") { - // Ok(value) => value, - // Err(_) => { - // return Err(( - // StatusCode::INTERNAL_SERVER_ERROR, - // Json::from(json!(( - // "error", - // "error at parsing environment variable: GIT_WORK_DIR" - // ))), - // )); - // } - // }; - // let post_dir = Path::new(&data_repo_dir).join("posts"); - // - // if is_force_refresh { - // if let Err(e) = query("DROP TABLE IF EXISTS hash, meta, post, tag;") - // .execute(&db_connection) - // .await - // { - // error!( - // "DB query error when deleting table: `hash`, `meta`, `post`, `tag`: {}", - // e.to_string() - // ); - // return Err(( - // StatusCode::INTERNAL_SERVER_ERROR, - // Json::from(json!(("error", e.to_string()))), - // )); - // } - // } - // - // // generate post idx - // let post_path = search_md(&post_dir); - // - // let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = post_path - // .into_iter() - // .filter_map(|post_path| match post::from_path(&post_path) { - // Ok(post) => match fs::read(&post_path) { - // Ok(bytes) => { - // let mut hasher = Sha256::new(); - // hasher.update(bytes); - // let hash = hasher.finalize().to_vec(); - // Some((post, hash)) - // } - // Err(e) => { - // error!( - // "Failed to read file while hashing the post <{}>, skipping: {}", - // post_path, - // e.to_string() - // ); - // None - // } - // }, - // Err(e) => { - // error!( - // "Failed to parse post <{}>, skipping: {}", - // post_path, - // e.to_string() - // ); - // None - // } - // }) - // .collect(); - // - // // db create - // query(POST_BASE_MODEL) - // .execute(&db_connection) - // .await - // .unwrap(); - // query(POST_META_MODEL) - // .execute(&db_connection) - // .await - // .unwrap(); - // query(TAG_MODEL).execute(&db_connection).await.unwrap(); - // query(POST_HASH_MODEL) - // .execute(&db_connection) - // .await - // .unwrap(); - // - // let res: Vec<_> = stream::iter(post_list_with_hash) - // .map(|(post, hash)| { - // let db_clone = db_connection.clone(); - // async move { build_post_idx(&db_clone, post, hash).await } - // }) - // .buffer_unordered(usize::MAX) - // .collect() - // .await; - // - // let errs: Vec<_> = res - // .into_iter() - // .filter_map(|res| res.err()) - // .map(|e| e.to_string()) - // .collect(); - // - // if !errs.is_empty() { - // Err(( - // StatusCode::INTERNAL_SERVER_ERROR, - // Json::from(json!({"error": errs})), - // )) - // } else { - // Ok(StatusCode::OK) - // } - todo!() -} - -/// Search a dir, return a list of markdown files -/// -/// # Arguments -/// * `entry`: dir path -/// -/// # Returns -/// Vec -/// * `String`: path of a markdown file -/// -fn search_md(entry: &Path) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path.as_path()) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry.to_str().unwrap().to_string()] - } else { - vec![] - } -} - -async fn build_post_idx( - db_connection: &PgPool, - post: Map, - hash: Vec, - // ) -> Result<(), PostIdxError> { -) -> Result<(), ()> { - todo!() - // let meta = &post["meta"]; - // - // // extract post id - // let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { - // Ok(id) => id, - // Err(e) => { - // return Err(PostIdxError::InvalidUUID { - // id: post["post_id"].as_str().unwrap().to_string(), - // err_msg: e.to_string(), - // }) - // } - // }; - // - // if build_post_idx::hash(db_connection, post_id, hash).await? { - // build_post_idx::base(db_connection, &post, post_id, meta).await?; - // build_post_idx::tag(db_connection, post_id, meta).await?; - // build_post_idx::meta(db_connection, post_id, meta).await?; - // } - // - // Ok(()) -} diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs index 8b61780..6d14b15 100644 --- a/backend/api_server/src/blog/mod.rs +++ b/backend/api_server/src/blog/mod.rs @@ -5,14 +5,12 @@ use crate::blog::api::{ get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, }; -use crate::blog::api::refresh::refresh; -use axum::routing::{get, post}; +use axum::routing::get; use axum::Router; use sqlx::{Pool, Postgres}; pub fn get_router() -> Router> { Router::new() - .route("/refresh_posts", post(refresh)) .route("/get_post_list", get(get_post_list)) .route("/get_page_count", get(get_page_count)) .route("/get_post/:post_id", get(get_post)) diff --git a/backend/api_server/src/init/database.rs b/backend/api_server/src/init/database.rs index e403269..c2a8aad 100644 --- a/backend/api_server/src/init/database.rs +++ b/backend/api_server/src/init/database.rs @@ -11,7 +11,10 @@ pub async fn init() -> Result, Box> { let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) .unwrap() .disable_statement_logging(); - Ok(PgPoolOptions::new().connect_with(db_connect_option).await?) + Ok(PgPoolOptions::new() + .max_connections(1) // todo: temporal solution, to be fixed + .connect_with(db_connect_option) + .await?) } fn parse_db_uri() -> String { diff --git a/frontend/app/blog/BlogPostCard.tsx b/frontend/app/blog/BlogPostCard.tsx index 3829fd2..0ca61a8 100644 --- a/frontend/app/blog/BlogPostCard.tsx +++ b/frontend/app/blog/BlogPostCard.tsx @@ -18,7 +18,7 @@ function BlogPostCard({ post }: Props) { // let last_update = new Date((post as any)['last_update']); let first_update = new Date((post as any)['first_update']); let post_id = (post as any)['post_id']; - let header_img = (post as any)['header_img']; + let header_img = ((post as any)['header_img'] as string | null); return ( @@ -29,7 +29,7 @@ function BlogPostCard({ post }: Props) { isHoverable > - { header_img !== '' &&
+ { header_img &&
{
-
+
{ category }

{ title }

diff --git a/frontend/app/blog/PostList.tsx b/frontend/app/blog/PostList.tsx index 4eff3a8..591c6ea 100644 --- a/frontend/app/blog/PostList.tsx +++ b/frontend/app/blog/PostList.tsx @@ -8,8 +8,8 @@ import { Card, CardBody } from '@nextui-org/card'; const PostList = async ({ page }: { page: number }) => { try { - const res = await fetchNocturneApi(`/get_post_list?page=${ page }`); - let posts = (await res.json() as any)['posts']; + const res = await fetchNocturneApi(`/blog/get_post_list?page=${ page }`); + const posts = await res.json(); return ( // blog list
diff --git a/frontend/app/blog/page.tsx b/frontend/app/blog/page.tsx index fb6d6cc..6690937 100644 --- a/frontend/app/blog/page.tsx +++ b/frontend/app/blog/page.tsx @@ -2,15 +2,15 @@ import PostList from '@/app/blog/PostList'; import Pagination from '@/app/blog/Pagination'; import { fetchNocturneApi } from '@/app/blog/api'; -async function Blog({ searchParams }: { - searchParams?: { - page?: number - }; +async function Blog({ + searchParams +}: { + searchParams: Promise<{ [key: string]: string | string[] | undefined }> }) { - const parsePage = Number(searchParams?.page); + const parsePage = Number((await searchParams)?.page); const currPage = parsePage > 0 ? parsePage : 1; - const ret = await fetchNocturneApi(`/get_page_count`); + const ret = await fetchNocturneApi(`/blog/get_page_count`); const pageCount = (await ret.json() as any)['res'] as number; return ( diff --git a/frontend/app/blog/post/[id]/Headers.tsx b/frontend/app/blog/post/[id]/Headers.tsx index cb44322..62e9090 100644 --- a/frontend/app/blog/post/[id]/Headers.tsx +++ b/frontend/app/blog/post/[id]/Headers.tsx @@ -18,11 +18,11 @@ const Headers = (props: Props) => { // let last_update = new Date((post as any)['last_update']); let first_update = new Date(post['first_update']); // let post_id = (post as any)['post_id']; - let header_img = post['header_img']; + let header_img = post['meta']['header_img'] as string | null; return ( <> - { header_img !== '' &&
+ { header_img &&
{ {
}
-
+
{ category }

{ title }

diff --git a/frontend/app/blog/post/[id]/PostCard.tsx b/frontend/app/blog/post/[id]/PostCard.tsx index 056f3d1..958979f 100644 --- a/frontend/app/blog/post/[id]/PostCard.tsx +++ b/frontend/app/blog/post/[id]/PostCard.tsx @@ -13,7 +13,7 @@ async function PostCard(props: Props) { try { - const res = await fetchNocturneApi(`/get_post/${ props.id }`); + const res = await fetchNocturneApi(`/blog/get_post/${ props.id }`); const ret: any = await res.json(); return (
diff --git a/frontend/app/blog/post/[id]/page.tsx b/frontend/app/blog/post/[id]/page.tsx index 13ea3b2..c29364d 100644 --- a/frontend/app/blog/post/[id]/page.tsx +++ b/frontend/app/blog/post/[id]/page.tsx @@ -3,7 +3,11 @@ import PostCard from '@/app/blog/post/[id]/PostCard'; import { MotionDiv } from '@/app/public/MotionDiv'; import ButtonGoBack from '@/app/blog/post/[id]/ButtonGoBack'; -function BlogReader({ params }: { params: { id: string } }) { +async function BlogReader({ + params +}: { + params: Promise<{ id: string }> +}) { return (
- +
diff --git a/frontend/components/ThemeSwitcher.tsx b/frontend/components/ThemeSwitcher.tsx index 07f6f15..0dbc95f 100644 --- a/frontend/components/ThemeSwitcher.tsx +++ b/frontend/components/ThemeSwitcher.tsx @@ -49,9 +49,9 @@ function ThemeSwitcher() { fullWidth isIconOnly={ isMobile } > - { switcherElements.get(currTheme.values().next().value)?.icon } + { switcherElements.get(currTheme.values().next().value as string)?.icon }

- { switcherElements.get(currTheme.values().next().value)?.display_name } + { switcherElements.get(currTheme.values().next().value as string)?.display_name }

diff --git a/frontend/package.json b/frontend/package.json index ed666ec..92be224 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,7 +13,7 @@ "@types/react-syntax-highlighter": "^15.5.13", "framer-motion": "^11.3.24", "lxgw-wenkai-webfont": "^1.7.0", - "next": "14.2.3", + "next": "^15.0.3", "next-themes": "^0.3.0", "react": "^18.3.1", "react-dom": "^18.3.1",