diff --git a/.env b/.env index daaad72..3708ad5 100644 --- a/.env +++ b/.env @@ -17,5 +17,3 @@ BACKEND_PORT=3001 # frontend settings API_LOCAL_URL=http://localhost:3001 API_REMOTE_URL=https://lumither.com/api - - diff --git a/.gitignore b/.gitignore index 6299755..3694827 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ .env.* /container/*/.env -# idea related +# IDE related .idea +.vscode # exclude post repo /blog_posts_git diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 1ae871b..6c75b0a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -3,5 +3,6 @@ resolver = "2" members = [ "api_server", "constants", - "markdown" + "macros", + "markdown", ] diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml index 73014cf..bb2012c 100644 --- a/backend/api_server/Cargo.toml +++ b/backend/api_server/Cargo.toml @@ -5,9 +5,18 @@ edition = "2021" [dependencies] constants = { path = "../constants" } -chrono = "0.4.38" +chrono = { version = "0.4.38", features = ["serde", "alloc"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } tracing-appender = "0.2.3" dotenv = "0.15.0" dirs = "5.0.1" +tokio = { version = "1.40.0", features = ["full"] } +sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } +axum = "0.7.5" +serde = { version = "1.0.199", features = ["derive"] } +serde_json = "1.0.128" +uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } +futures = { version = "0.3.30", features = ["executor"] } +sha2 = "0.10.8" +cron = "0.12.1" \ No newline at end of file diff --git a/backend/legacy/src/api/get/get_page_count.rs b/backend/api_server/src/blog/api/get_page_count.rs similarity index 95% rename from backend/legacy/src/api/get/get_page_count.rs rename to backend/api_server/src/blog/api/get_page_count.rs index 887e360..db540af 100644 --- a/backend/legacy/src/api/get/get_page_count.rs +++ b/backend/api_server/src/blog/api/get_page_count.rs @@ -1,4 +1,5 @@ -use crate::constants::PAGE_SIZE; +use constants::blog::PAGE_SIZE; + use axum::extract::State; use axum::Json; use serde_json::{json, Value}; diff --git a/backend/legacy/src/api/get/get_post.rs b/backend/api_server/src/blog/api/get_post.rs similarity index 99% rename from backend/legacy/src/api/get/get_post.rs rename to backend/api_server/src/blog/api/get_post.rs index a38eb89..2374dec 100644 --- a/backend/legacy/src/api/get/get_post.rs +++ b/backend/api_server/src/blog/api/get_post.rs @@ -1,12 +1,11 @@ -use std::str::FromStr; - use axum::{ extract::{Path, State}, http::StatusCode, Json, }; use serde_json::{json, Value}; -use sqlx::{Error, PgPool, query, Row}; +use sqlx::{query, Error, PgPool, Row}; +use std::str::FromStr; use tracing::error; use uuid::Uuid; diff --git a/backend/legacy/src/api/get/get_post_list.rs b/backend/api_server/src/blog/api/get_post_list.rs similarity index 98% rename from backend/legacy/src/api/get/get_post_list.rs rename to backend/api_server/src/blog/api/get_post_list.rs index 1b55610..854f495 100644 --- a/backend/legacy/src/api/get/get_post_list.rs +++ b/backend/api_server/src/blog/api/get_post_list.rs @@ -1,4 +1,5 @@ -use crate::constants::PAGE_SIZE; +use constants::blog::PAGE_SIZE; + use axum::{extract::Query, extract::State, http::StatusCode, Json}; use chrono::{DateTime, Utc}; use futures::future::join_all; diff --git a/backend/legacy/src/api/get/mod.rs b/backend/api_server/src/blog/api/mod.rs similarity index 79% rename from backend/legacy/src/api/get/mod.rs rename to backend/api_server/src/blog/api/mod.rs index 5e23c3d..e946d12 100644 --- a/backend/legacy/src/api/get/mod.rs +++ b/backend/api_server/src/blog/api/mod.rs @@ -1,3 +1,4 @@ pub mod get_page_count; pub mod get_post; pub mod get_post_list; +pub mod refresh; diff --git a/backend/api_server/src/blog/api/refresh.rs b/backend/api_server/src/blog/api/refresh.rs new file mode 100644 index 0000000..cc7b331 --- /dev/null +++ b/backend/api_server/src/blog/api/refresh.rs @@ -0,0 +1,190 @@ +use std::path::Path; +use std::str::FromStr; + +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::Json; +use serde::Deserialize; +use serde_json::{Map, Value}; +use sqlx::PgPool; + +// use crate::api::post::errors::refresh::PostIdxError; +// use crate::api::post::utils::{build_post_idx, post}; +// use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL}; + +#[derive(Deserialize)] +pub struct Params { + /// refresh level + level: Option, +} + +pub async fn refresh( + State(db_connection): State, + Query(params): Query, +) -> Result)> { + // let is_force_refresh = params.level.unwrap_or("".to_string()) == "force"; + // + // // search post + // let data_repo_dir = match env::var("GIT_WORK_DIR") { + // Ok(value) => value, + // Err(_) => { + // return Err(( + // StatusCode::INTERNAL_SERVER_ERROR, + // Json::from(json!(( + // "error", + // "error at parsing environment variable: GIT_WORK_DIR" + // ))), + // )); + // } + // }; + // let post_dir = Path::new(&data_repo_dir).join("posts"); + // + // if is_force_refresh { + // if let Err(e) = query("DROP TABLE IF EXISTS hash, meta, post, tag;") + // .execute(&db_connection) + // .await + // { + // error!( + // "DB query error when deleting table: `hash`, `meta`, `post`, `tag`: {}", + // e.to_string() + // ); + // return Err(( + // StatusCode::INTERNAL_SERVER_ERROR, + // Json::from(json!(("error", e.to_string()))), + // )); + // } + // } + // + // // generate post idx + // let post_path = search_md(&post_dir); + // + // let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = post_path + // .into_iter() + // .filter_map(|post_path| match post::from_path(&post_path) { + // Ok(post) => match fs::read(&post_path) { + // Ok(bytes) => { + // let mut hasher = Sha256::new(); + // hasher.update(bytes); + // let hash = hasher.finalize().to_vec(); + // Some((post, hash)) + // } + // Err(e) => { + // error!( + // "Failed to read file while hashing the post <{}>, skipping: {}", + // post_path, + // e.to_string() + // ); + // None + // } + // }, + // Err(e) => { + // error!( + // "Failed to parse post <{}>, skipping: {}", + // post_path, + // e.to_string() + // ); + // None + // } + // }) + // .collect(); + // + // // db create + // query(POST_BASE_MODEL) + // .execute(&db_connection) + // .await + // .unwrap(); + // query(POST_META_MODEL) + // .execute(&db_connection) + // .await + // .unwrap(); + // query(TAG_MODEL).execute(&db_connection).await.unwrap(); + // query(POST_HASH_MODEL) + // .execute(&db_connection) + // .await + // .unwrap(); + // + // let res: Vec<_> = stream::iter(post_list_with_hash) + // .map(|(post, hash)| { + // let db_clone = db_connection.clone(); + // async move { build_post_idx(&db_clone, post, hash).await } + // }) + // .buffer_unordered(usize::MAX) + // .collect() + // .await; + // + // let errs: Vec<_> = res + // .into_iter() + // .filter_map(|res| res.err()) + // .map(|e| e.to_string()) + // .collect(); + // + // if !errs.is_empty() { + // Err(( + // StatusCode::INTERNAL_SERVER_ERROR, + // Json::from(json!({"error": errs})), + // )) + // } else { + // Ok(StatusCode::OK) + // } + todo!() +} + +/// Search a dir, return a list of markdown files +/// +/// # Arguments +/// * `entry`: dir path +/// +/// # Returns +/// Vec +/// * `String`: path of a markdown file +/// +fn search_md(entry: &Path) -> Vec { + if entry.ends_with(".git") { + return vec![]; + } + if entry.is_dir() { + if let Ok(files) = entry.read_dir() { + files + .flat_map(|file| { + let file_path = file.unwrap().path(); + search_md(file_path.as_path()) + }) + .collect() + } else { + vec![] + } + } else if entry.extension().is_some_and(|ext| ext == "md") { + vec![entry.to_str().unwrap().to_string()] + } else { + vec![] + } +} + +async fn build_post_idx( + db_connection: &PgPool, + post: Map, + hash: Vec, + // ) -> Result<(), PostIdxError> { +) -> Result<(), ()> { + todo!() + // let meta = &post["meta"]; + // + // // extract post id + // let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { + // Ok(id) => id, + // Err(e) => { + // return Err(PostIdxError::InvalidUUID { + // id: post["post_id"].as_str().unwrap().to_string(), + // err_msg: e.to_string(), + // }) + // } + // }; + // + // if build_post_idx::hash(db_connection, post_id, hash).await? { + // build_post_idx::base(db_connection, &post, post_id, meta).await?; + // build_post_idx::tag(db_connection, post_id, meta).await?; + // build_post_idx::meta(db_connection, post_id, meta).await?; + // } + // + // Ok(()) +} diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs new file mode 100644 index 0000000..abfb8e6 --- /dev/null +++ b/backend/api_server/src/blog/mod.rs @@ -0,0 +1,19 @@ +mod api; +mod scheduler; + +use crate::blog::api::{ + get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, +}; + +use crate::blog::api::refresh::refresh; +use axum::routing::{get, post}; +use axum::Router; +use sqlx::{Pool, Postgres}; + +pub fn get_router() -> Router> { + Router::new() + .route("/refresh_posts", post(refresh)) + .route("/get_post_list", get(get_post_list)) + .route("/get_page_count", get(get_page_count)) + .route("/get_post/:post_id", get(get_post)) +} diff --git a/backend/api_server/src/blog/scheduler/mod.rs b/backend/api_server/src/blog/scheduler/mod.rs new file mode 100644 index 0000000..ef73764 --- /dev/null +++ b/backend/api_server/src/blog/scheduler/mod.rs @@ -0,0 +1,184 @@ +use crate::blog::{ + scheduler::scheduler_error::SchedulerError, + scheduler::scheduler_error::SchedulerError::IdAlreadyRunning, + scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper, + scheduler::tasks::Task, +}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::RwLock; +use std::{collections::HashMap, sync::Arc, thread}; +use uuid::Uuid; + +pub mod scheduler_error; +mod sleeper; +mod task_func; +pub mod tasks; + +struct TaskStatus { + sleeper: Option>, + is_alive: AtomicBool, +} + +#[derive(Default)] +pub struct Scheduler { + tasks: Arc>>>>, + running: Arc>>>, + stopped: Arc>>, +} + +impl Scheduler { + /// Start all tasks in the scheduler + pub fn start(&self) -> Result<(), SchedulerError> { + for (k, _v) in self.tasks.read()?.iter() { + self.run_non_blocking(k)? + } + Ok(()) + } + + /// Terminate all tasks in the scheduler + pub fn halt() {} + + fn run_task( + running: Arc>>>, + stopped: Arc>>, + id: Uuid, + task: Arc>, + ) -> Result<(), SchedulerError> { + if running.read()?.get(&id).is_some() { + return Err(IdAlreadyRunning); + } else { + running.write()?.insert( + id, + TaskStatus { + sleeper: None, + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + loop { + if let Some(next_execution) = task.get_next_execution() { + // guarantee is alive + let task_status = { + let running_reading_guard = running.read()?; + running_reading_guard.get(&id).cloned().clone() + }; + if let Some(task_status) = task_status { + if !&task_status.is_alive.load(Ordering::SeqCst) { + { + let mut running_lock_guard = running.write()?; + running_lock_guard.remove(&id); + } + stopped.write()?.push(id); + return Ok(()); + } else { + let sleeper = Arc::from(SkippableSleeper::from(next_execution)); + { + let mut running_lock_guard = running.write()?; + running_lock_guard.insert( + id, + TaskStatus { + sleeper: Some(sleeper.clone()), + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + sleeper.start(); + task.call(); + } + } + } else { + stopped.write()?.push(id); + break; + } + } + Ok(()) + } + + /// Run a task with `id` + pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + Self::run_task(self.running.clone(), self.stopped.clone(), id, task.clone()) + } else { + Err(IdNotFound) + } + } + + pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + let running_clone = self.running.clone(); + let stopped_clone = self.stopped.clone(); + let task_clone = task.clone(); + thread::spawn(move || Self::run_task(running_clone, stopped_clone, id, task_clone)); + Ok(()) + } else { + Err(IdNotFound) + } + } + + fn get_running(&self, id: &Uuid) -> Option> { + let ret = self.running.read().unwrap().get(id).cloned(); + ret + } + + /// Stop a task with `id` + pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + status.is_alive.store(false, Ordering::SeqCst); + if let Some(s) = &status.sleeper { + s.skip(); + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + if let Some(s) = &status.sleeper { + s.skip() + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn insert(&self, task: Box) -> Result { + let id = Uuid::new_v4(); + self.tasks.write()?.insert(id, Arc::new(task)); + Ok(id) + } +} + +#[cfg(test)] +mod tests { + use crate::blog::scheduler::tasks::basic::BasicTask; + use crate::blog::scheduler::Scheduler; + use std::error::Error; + use std::thread; + use std::time::Duration; + + #[test] + fn test_run() -> Result<(), Box> { + let scheduler = Scheduler::default(); + let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; + let id = scheduler.insert(Box::new(task))?; + scheduler.run_non_blocking(&id)?; + thread::sleep(Duration::from_secs(4)); + scheduler.stop(&id)?; + // test termination + thread::sleep(Duration::from_secs(5)); + Ok(()) + } +} diff --git a/backend/api_server/src/blog/scheduler/scheduler_error.rs b/backend/api_server/src/blog/scheduler/scheduler_error.rs new file mode 100644 index 0000000..54615be --- /dev/null +++ b/backend/api_server/src/blog/scheduler/scheduler_error.rs @@ -0,0 +1,44 @@ +use crate::blog::scheduler::scheduler_error::SchedulerError::MutexPoison; +use std::error::Error; +use std::fmt::{Debug, Display, Formatter}; +use std::sync::PoisonError; + +pub enum SchedulerError { + IdNotFound, + IdAlreadyRunning, + MutexPoison(String), +} + +impl From> for SchedulerError { + fn from(value: PoisonError) -> Self { + MutexPoison(value.to_string()) + } +} + +impl Debug for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) + } +} + +impl Display for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + SchedulerError::IdNotFound => { + "SchedulerError::IdNotFound".to_string() + } + SchedulerError::IdAlreadyRunning => { + "SchedulerError::IdAlreadyRunning".to_string() + } + MutexPoison(str) => { + format!("SchedulerError::MutexPoison: {}", str) + } + } + ) + } +} + +impl Error for SchedulerError {} diff --git a/backend/api_server/src/blog/scheduler/sleeper.rs b/backend/api_server/src/blog/scheduler/sleeper.rs new file mode 100644 index 0000000..dd84bd6 --- /dev/null +++ b/backend/api_server/src/blog/scheduler/sleeper.rs @@ -0,0 +1,77 @@ +use chrono::{DateTime, Utc}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; +use std::time::Duration; + +#[derive(Clone, Debug, Default)] +pub struct SkippableSleeper { + pub duration: Duration, + state: Arc<(Mutex, Condvar)>, +} + +impl SkippableSleeper { + pub fn new(duration: Duration) -> Self { + SkippableSleeper { + duration, + state: Arc::new((Mutex::new(false), Condvar::new())), + } + } + + // Blocks until the duration has passed or skip() is called + pub fn start(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + let state_clone = Arc::clone(&self.state); + let duration_clone = self.duration; + thread::spawn(move || { + thread::sleep(duration_clone); + let (lock, cvar) = &*state_clone; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + }); + while !*received { + received = cvar.wait(received).unwrap(); + } + } + + // Skip the blocking delay + pub fn skip(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + } +} + +impl From> for SkippableSleeper { + fn from(value: DateTime) -> Self { + let from_utc_timestamp = value.timestamp() as u64; + let curr_utc_timestamp = Utc::now().timestamp() as u64; + SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) + } +} + +#[cfg(test)] +mod test { + use crate::blog::scheduler::sleeper::SkippableSleeper; + use std::thread; + use std::time::Duration; + + #[test] + fn usage() { + let sleeper = SkippableSleeper::new(Duration::new(10, 0)); + thread::spawn({ + let sleeper = sleeper.clone(); + move || { + println!("Sleeping..."); + sleeper.start(); + println!("Woke up!"); + } + }); + println!("Start Sleeping..."); + thread::sleep(Duration::from_secs(4)); + println!("Skipping sleep..."); + sleeper.skip(); + } +} diff --git a/backend/api_server/src/blog/scheduler/task_func.rs b/backend/api_server/src/blog/scheduler/task_func.rs new file mode 100644 index 0000000..af27ded --- /dev/null +++ b/backend/api_server/src/blog/scheduler/task_func.rs @@ -0,0 +1,3 @@ +pub trait TaskFunc: Fn() + Send + Sync + 'static {} + +impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} diff --git a/backend/api_server/src/blog/scheduler/tasks/basic.rs b/backend/api_server/src/blog/scheduler/tasks/basic.rs new file mode 100644 index 0000000..d18e60c --- /dev/null +++ b/backend/api_server/src/blog/scheduler/tasks/basic.rs @@ -0,0 +1,30 @@ +use crate::blog::scheduler::task_func::TaskFunc; +use crate::blog::scheduler::tasks::Task; +use chrono::{DateTime, Utc}; +use cron::Schedule; +use std::error::Error; +use std::str::FromStr; + +pub struct BasicTask { + schedule: Schedule, + func: Box, +} + +impl BasicTask { + pub fn new(func: fn(), cron_expr: &str) -> Result> { + Ok(BasicTask { + schedule: Schedule::from_str(cron_expr)?, + func: Box::new(func), + }) + } +} + +impl Task for BasicTask { + fn get_next_execution(&self) -> Option> { + self.schedule.upcoming(Utc).next() + } + + fn call(&self) { + (self.func)() + } +} diff --git a/backend/api_server/src/blog/scheduler/tasks/mod.rs b/backend/api_server/src/blog/scheduler/tasks/mod.rs new file mode 100644 index 0000000..898bc9e --- /dev/null +++ b/backend/api_server/src/blog/scheduler/tasks/mod.rs @@ -0,0 +1,9 @@ +use crate::blog::scheduler::task_func::TaskFunc; +use chrono::{DateTime, Utc}; + +pub mod basic; + +pub trait Task: Send + Sync { + fn get_next_execution(&self) -> Option>; + fn call(&self); +} diff --git a/backend/api_server/src/init.rs b/backend/api_server/src/init.rs index 8785944..809b977 100644 --- a/backend/api_server/src/init.rs +++ b/backend/api_server/src/init.rs @@ -1,5 +1,7 @@ -use crate::logger; use std::env; + +use crate::logger; + use tracing_appender::non_blocking::WorkerGuard; pub fn logger_init() -> (WorkerGuard, WorkerGuard) { diff --git a/backend/api_server/src/logger.rs b/backend/api_server/src/logger.rs index 7721323..7dc26e2 100644 --- a/backend/api_server/src/logger.rs +++ b/backend/api_server/src/logger.rs @@ -1,16 +1,14 @@ -use chrono::Local; -use dirs::home_dir; use std::path::Path; use std::{env, io}; + +use chrono::Local; +use dirs::home_dir; use tracing::info; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_appender::{non_blocking, rolling}; -use tracing_subscriber::filter::LevelFilter; -use tracing_subscriber::fmt::format::Writer; -use tracing_subscriber::fmt::time::FormatTime; -use tracing_subscriber::fmt::MakeWriter; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::{fmt, Layer, Registry}; +use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; +use tracing_subscriber::{ + filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, + layer::SubscriberExt, Layer, Registry, +}; struct LocalTimer; @@ -95,10 +93,10 @@ pub fn init() -> (WorkerGuard, WorkerGuard) { ); tracing::subscriber::set_global_default(subscribers) - .expect("[fatal] Failed to setup logging system"); + .expect("[fatal] failed to setup logging system"); - info!("Module Started: Logger"); - info!("Logging to file {}", log_path); + info!("module started: logger"); + info!("logging to dir: {}", log_path); (info_guard, error_guard) } diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs index 1db4480..7187196 100644 --- a/backend/api_server/src/main.rs +++ b/backend/api_server/src/main.rs @@ -1,7 +1,12 @@ +mod blog; mod init; mod logger; +mod server; -fn main() { +#[tokio::main] +async fn main() { init::load_env(); let _guards = init::logger_init(); + + server::start().await; } diff --git a/backend/legacy/src/main.rs b/backend/api_server/src/server.rs similarity index 52% rename from backend/legacy/src/main.rs rename to backend/api_server/src/server.rs index 7fef8f3..e23e63b 100644 --- a/backend/legacy/src/main.rs +++ b/backend/api_server/src/server.rs @@ -1,45 +1,20 @@ use std::env; -use std::error::Error; use std::num::ParseIntError; use std::process::exit; use std::str::FromStr; -use crate::api::{ - get::{get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list}, - post::refresh::refresh, -}; -use axum::routing::{get, post}; +use crate::blog; use axum::Router; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; -use sqlx::ConnectOptions; +use sqlx::{ + postgres::{PgConnectOptions, PgPoolOptions}, + ConnectOptions, +}; use tracing::{error, info, warn}; -mod api; -mod constants; -mod logger; -mod markdown; -mod model; - -#[tokio::main] -async fn main() -> Result<(), Box> { - if let Some(env_file) = env::args().nth(1) { - match dotenv::from_filename(&env_file) { - Ok(_) => { - println!("[Info] env loaded from {}, starting up", env_file); - } - Err(e) => { - panic!("[Fatal] failed to read {}: {}", env_file, e); - } - } - } else { - println!("[info] no .env file referred, starting up") - } - - let _guards = logger::init(); - +pub async fn start() { let port: u32 = match env::var("BACKEND_PORT") { Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { - warn!("Failed to parse BACKEND_PORT: {}", e.to_string()); + warn!("failed to parse BACKEND_PORT: {}", e.to_string()); 3001 }), Err(_) => 3001, @@ -51,35 +26,30 @@ async fn main() -> Result<(), Box> { let db_pool = match PgPoolOptions::new().connect_with(db_connect_option).await { Ok(pool) => { - info!("Database connected"); + info!("database connected"); pool } Err(e) => { - error!("Failed to load database: {}", e.to_string()); + error!("failed to load database: {}", e.to_string()); exit(1); } }; let app = Router::new() - .route("/refresh_posts", post(refresh)) - .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count)) - .route("/get_post/:post_id", get(get_post)) + .nest("blog", blog::get_router()) .with_state(db_pool); let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)) .await .unwrap_or_else(|e| { - error!("Failed to build TCP listener: {}", e.to_string()); + error!("failed to build TCP listener: {}", e.to_string()); exit(1); }); axum::serve(listener, app).await.unwrap_or_else(|e| { - error!("Failed to start axum: {}", e.to_string()); + error!("failed to start axum: {}", e.to_string()); exit(1); }); - - Ok(()) } fn parse_db_uri() -> String { diff --git a/backend/legacy/Cargo.toml b/backend/legacy/Cargo.toml deleted file mode 100644 index 5d3cbd3..0000000 --- a/backend/legacy/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "backend" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -axum = "0.7.5" -dotenv = "0.15.0" -serde = { version = "1.0.199", features = ["derive"] } -serde_json = "1.0.116" -tokio = { version = "1.37.0", features = ["full"] } -uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } -sqlx = { version = "0.7.4", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } -regex = "1.10.4" -futures = { version = "0.3.30", features = ["executor"] } -tracing = "0.1.40" -tracing-subscriber = "0.3.18" -tracing-appender = "0.2.3" -log = "0.4.21" -sha2 = "0.10.8" -thiserror = "1.0.63" -base64 = "0.22.1" diff --git a/backend/legacy/src/api/mod.rs b/backend/legacy/src/api/mod.rs deleted file mode 100644 index 69e599d..0000000 --- a/backend/legacy/src/api/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod get; -pub mod post; diff --git a/backend/legacy/src/api/post/errors/mod.rs b/backend/legacy/src/api/post/errors/mod.rs deleted file mode 100644 index 1d2e0f5..0000000 --- a/backend/legacy/src/api/post/errors/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod refresh; diff --git a/backend/legacy/src/api/post/errors/refresh.rs b/backend/legacy/src/api/post/errors/refresh.rs deleted file mode 100644 index 1c4224a..0000000 --- a/backend/legacy/src/api/post/errors/refresh.rs +++ /dev/null @@ -1,61 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum PostIdxError { - #[error("Failed to parse UUID <{}>: {}", id, err_msg)] - InvalidUUID { id: String, err_msg: String }, - - #[error( - "Failed to parse `last update time` for <{}> when creating post index: {}", - id, - err_msg - )] - InvalidLastUpdate { id: String, err_msg: String }, - - #[error( - "Failed to parse `first update time` for <{}> when creating post index: {}", - id, - err_msg - )] - InvalidFirstUpdate { id: String, err_msg: String }, - - #[error( - "Failed to parse `tag` for <{}> when creating post index: <{}>: {}", - id, - invalid_string, - err_msg - )] - InvalidTagString { - id: String, - invalid_string: String, - err_msg: String, - }, - - #[error( - "Database error on writing `{}` to table `{}`: <{}>: {}", - data_desc, - db_table, - id, - err_msg - )] - DBWriteFailure { - data_desc: String, - db_table: String, - id: String, - err_msg: String, - }, - - #[error( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - data_desc, - db_table, - id, - err_msg - )] - DBReadFailure { - data_desc: String, - db_table: String, - id: String, - err_msg: String, - }, -} diff --git a/backend/legacy/src/api/post/mod.rs b/backend/legacy/src/api/post/mod.rs deleted file mode 100644 index 1bba1ec..0000000 --- a/backend/legacy/src/api/post/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod errors; -pub mod refresh; -mod utils; diff --git a/backend/legacy/src/api/post/refresh.rs b/backend/legacy/src/api/post/refresh.rs deleted file mode 100644 index 65d7be3..0000000 --- a/backend/legacy/src/api/post/refresh.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::path::Path; -use std::str::FromStr; -use std::{env, fs}; - -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::Json; -use futures::{stream, StreamExt}; -use serde::Deserialize; -use serde_json::{json, Map, Value}; -use sha2::{Digest, Sha256}; -use sqlx::{query, PgPool}; -use tracing::error; -use uuid::Uuid; - -use crate::api::post::errors::refresh::PostIdxError; -use crate::api::post::utils::{build_post_idx, post}; -use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL}; - -#[derive(Deserialize)] -pub struct Params { - /// refresh level - level: Option, -} - -pub async fn refresh( - State(db_connection): State, - Query(params): Query, -) -> Result)> { - let is_force_refresh = params.level.unwrap_or("".to_string()) == "force"; - - // search post - let data_repo_dir = match env::var("GIT_WORK_DIR") { - Ok(value) => value, - Err(_) => { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(( - "error", - "error at parsing environment variable: GIT_WORK_DIR" - ))), - )); - } - }; - let post_dir = Path::new(&data_repo_dir).join("posts"); - - if is_force_refresh { - if let Err(e) = query("DROP TABLE IF EXISTS hash, meta, post, tag;") - .execute(&db_connection) - .await - { - error!( - "DB query error when deleting table: `hash`, `meta`, `post`, `tag`: {}", - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(("error", e.to_string()))), - )); - } - } - - // generate post idx - let post_path = search_md(&post_dir); - - let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = post_path - .into_iter() - .filter_map(|post_path| match post::from_path(&post_path) { - Ok(post) => match fs::read(&post_path) { - Ok(bytes) => { - let mut hasher = Sha256::new(); - hasher.update(bytes); - let hash = hasher.finalize().to_vec(); - Some((post, hash)) - } - Err(e) => { - error!( - "Failed to read file while hashing the post <{}>, skipping: {}", - post_path, - e.to_string() - ); - None - } - }, - Err(e) => { - error!( - "Failed to parse post <{}>, skipping: {}", - post_path, - e.to_string() - ); - None - } - }) - .collect(); - - // db create - query(POST_BASE_MODEL) - .execute(&db_connection) - .await - .unwrap(); - query(POST_META_MODEL) - .execute(&db_connection) - .await - .unwrap(); - query(TAG_MODEL).execute(&db_connection).await.unwrap(); - query(POST_HASH_MODEL) - .execute(&db_connection) - .await - .unwrap(); - - let res: Vec<_> = stream::iter(post_list_with_hash) - .map(|(post, hash)| { - let db_clone = db_connection.clone(); - async move { build_post_idx(&db_clone, post, hash).await } - }) - .buffer_unordered(usize::MAX) - .collect() - .await; - - let errs: Vec<_> = res - .into_iter() - .filter_map(|res| res.err()) - .map(|e| e.to_string()) - .collect(); - - if !errs.is_empty() { - Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": errs})), - )) - } else { - Ok(StatusCode::OK) - } -} - -/// Search a dir, return a list of markdown files -/// -/// # Arguments -/// * `entry`: dir path -/// -/// # Returns -/// Vec -/// * `String`: path of a markdown file -/// -fn search_md(entry: &Path) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path.as_path()) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry.to_str().unwrap().to_string()] - } else { - vec![] - } -} - -async fn build_post_idx( - db_connection: &PgPool, - post: Map, - hash: Vec, -) -> Result<(), PostIdxError> { - let meta = &post["meta"]; - - // extract post id - let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { - Ok(id) => id, - Err(e) => { - return Err(PostIdxError::InvalidUUID { - id: post["post_id"].as_str().unwrap().to_string(), - err_msg: e.to_string(), - }) - } - }; - - if build_post_idx::hash(db_connection, post_id, hash).await? { - build_post_idx::base(db_connection, &post, post_id, meta).await?; - build_post_idx::tag(db_connection, post_id, meta).await?; - build_post_idx::meta(db_connection, post_id, meta).await?; - } - - Ok(()) -} diff --git a/backend/legacy/src/api/post/utils/build_post_idx.rs b/backend/legacy/src/api/post/utils/build_post_idx.rs deleted file mode 100644 index 47b2692..0000000 --- a/backend/legacy/src/api/post/utils/build_post_idx.rs +++ /dev/null @@ -1,288 +0,0 @@ -use base64::{Engine, engine::general_purpose}; -use chrono::DateTime; -use serde_json::{Map, Value}; -use sqlx::{PgPool, query, Row}; -use uuid::Uuid; - -use crate::api::post::errors::refresh::PostIdxError; -use crate::constants::GLOBAL_TIME_FORMAT; - -pub async fn base( - db_connection: &PgPool, - post: &Map, - post_id: Uuid, - meta: &Value, -) -> Result<(), PostIdxError> { - let title = post["title"].as_str(); - let summary = post - .get("summary") - .and_then(|summary| summary.as_str()) - .unwrap_or(""); - let content = post["content"].as_str(); - let last_update = - match DateTime::parse_from_str(post["last_update"].as_str().unwrap(), GLOBAL_TIME_FORMAT) { - Ok(time) => time, - Err(e) => { - return Err(PostIdxError::InvalidLastUpdate { - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - }; - let first_update = match DateTime::parse_from_str( - post["first_update"].as_str().unwrap(), - GLOBAL_TIME_FORMAT, - ) { - Ok(time) => time, - Err(e) => { - return Err(PostIdxError::InvalidFirstUpdate { - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - }; - let sub_title = meta - .get("sub_title") - .and_then(|sub_title| sub_title.as_str()) - .unwrap_or(""); - let category = meta - .get("category") - .and_then(|category| category.as_str()) - .unwrap_or("N/A"); - let header_img = meta - .get("header_img") - .and_then(|header_img| header_img.as_str()) - .unwrap_or(""); - - match query( - r##" -INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) -ON CONFLICT (post_id) DO -UPDATE SET - title = EXCLUDED.title, - summary = EXCLUDED.summary, - content = EXCLUDED.content, - last_update = EXCLUDED.last_update, - first_update = EXCLUDED.first_update, - sub_title = EXCLUDED.sub_title, - category = EXCLUDED.category, - header_img = EXCLUDED.header_img; - "##, - ) - .bind(post_id) - .bind(title) - .bind(summary) - .bind(content) - .bind(last_update) - .bind(first_update) - .bind(sub_title) - .bind(category) - .bind(header_img) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err( - PostIdxError::DBWriteFailure { - data_desc: "post base information".to_string(), - db_table: "Post".to_string(), - id: post_id.to_string(), - err_msg: e.to_string() - } - ) - } - } - - Ok(()) -} - -pub async fn tag(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> { - if let Some(tags_string) = meta.get("tags").to_owned() { - // parse tags string - let parsed: Value = match serde_json::from_str(tags_string.as_str().unwrap()) { - Ok(value) => value, - Err(e) => { - return Err(PostIdxError::InvalidTagString { - id: post_id.to_string(), - invalid_string: tags_string.to_string().to_string(), - err_msg: e.to_string(), - }) - } - }; - let string_array = parsed.as_array().unwrap(); - let tags: Vec = string_array - .iter() - .map(|value| value.as_str().unwrap().to_string()) - .collect(); - - // remove old record - if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") - .bind(post_id) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - - // add new record - for tag in tags { - match query("INSERT INTO Tag VALUES ($1, $2);") - .bind(post_id) - .bind(tag) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - } - } - } - - Ok(()) -} - -/// # Arguments -/// -/// * `db_connection`: database connection -/// * `post_id`: UUID of the post -/// * `hash`: Hex String -/// -/// # Returns -/// Result -/// - `bool`: if the hash is different from database, return true, else false -pub async fn hash( - db_connection: &PgPool, - post_id: Uuid, - hash: Vec, -) -> Result { - let base64_form = general_purpose::STANDARD.encode(hash); - match query("SELECT hash FROM Hash WHERE post_id = $1") - .bind(post_id) - .fetch_optional(db_connection) - .await - { - Ok(row) => { - let db_record: String = match row { - None => "".to_string(), - Some(row) => row.try_get("hash").unwrap_or("".to_string()), - }; - if base64_form == db_record { - Ok(false) - } else { - if let Err(e) = query( - r#" -INSERT INTO Hash (post_id, hash) VALUES ($1, $2) - ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, - ) - .bind(post_id) - .bind(&base64_form) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - Ok(true) - } - } - Err(e) => Err(PostIdxError::DBReadFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }), - } -} - -pub async fn meta(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> { - let mut err_trails = 5; - loop { - if let Err(e) = query( - format!( - r#" -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY['tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY(_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - "#, post_id.to_string().as_str(), meta.to_string() - ).as_str() - ) - .execute(db_connection) - .await { - if err_trails >= 0 { - err_trails -= 1; - continue - } else { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post meta data".to_string(), - db_table: "Meta".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - } else { - return Ok(()) - } - } -} diff --git a/backend/legacy/src/api/post/utils/mod.rs b/backend/legacy/src/api/post/utils/mod.rs deleted file mode 100644 index faee48e..0000000 --- a/backend/legacy/src/api/post/utils/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod build_post_idx; -pub mod post; diff --git a/backend/legacy/src/api/post/utils/post.rs b/backend/legacy/src/api/post/utils/post.rs deleted file mode 100644 index 901d799..0000000 --- a/backend/legacy/src/api/post/utils/post.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::{env, fs}; -use std::error::Error; -use std::process::Command; -use std::str; -use std::str::FromStr; - -use chrono::{DateTime, Utc}; -use serde_json::{Map, Value}; -use tracing::{error, warn}; -use uuid::Uuid; - -use crate::constants::GLOBAL_TIME_FORMAT; -use crate::markdown::meta::parse_meta; - -pub fn from_path(path: &str) -> Result, Box> { - let mut res = Map::new(); - - // field `content` - let md_content = fs::read_to_string(path)?; - res.insert("content".to_string(), Value::from(md_content.to_owned())); - - // field `meta` - let meta = parse_meta(&md_content); - let meta_map: Map = meta - .clone() - .into_iter() - .map(|(k, v)| (k, Value::from(v))) - .collect(); - res.insert("meta".to_string(), Value::from(meta_map)); - - // field `post_id` - let post_id = match meta.get("id") { - None => Uuid::new_v4(), - Some(id) => match Uuid::from_str(id.as_str()) { - Ok(id) => id, - Err(e) => { - error!("Failed to parse UUID <{}>: {}", id.as_str(), e.to_string()); - return Err(Box::new(e)); - } - }, - }; - res.insert("post_id".to_string(), Value::from(post_id.to_string())); - - // field `title` - let title = match meta.get("title") { - None => "Title Not Found".to_string(), - Some(title) => title.to_owned(), - }; - res.insert("title".to_string(), Value::from(title)); - - // field `summary` - let summary = match meta.get("summary") { - None => "".to_string(), - Some(summary) => summary.to_owned(), - }; - res.insert("summary".to_string(), Value::from(summary)); - - // field `last_update` - let git_output = match Command::new("git") - .arg("--no-pager") - .arg("log") - .arg("-1") - .arg("--format=%cI") - .arg("--") - .current_dir(env::var("GIT_WORK_DIR")?) - .arg(path) - .output() - { - Ok(res) => res, - Err(e) => { - error!("Failed to execute `git` command: {}", e.to_string()); - return Err(Box::new(e)); - } - }; - let last_update: DateTime = if git_output.status.success() { - chrono::DateTime::parse_from_str( - str::from_utf8(&git_output.stdout).unwrap().trim_end(), - GLOBAL_TIME_FORMAT, - ) - .unwrap_or_else(|_| DateTime::from(Utc::now())) - .to_utc() - } else { - Utc::now().to_utc() - }; - res.insert( - "last_update".to_string(), - Value::from(last_update.format(GLOBAL_TIME_FORMAT).to_string()), - ); - - // field `first_update` - let first_update: DateTime = match meta.get("date") { - None => { - warn!( - "Failed to parse first update time for post {}, using current time instead", - post_id.to_string() - ); - Utc::now().to_utc() - } - Some(date) => DateTime::from( - match DateTime::parse_from_str(format!("{}T000000", date).as_str(), "%Y-%m-%d%zT%H%M%S") - { - Ok(time) => time, - Err(e) => { - warn!( - "Failed to parse first update time for post {}, using current time instead: {}", - post_id.to_string(), - e.to_string() - ); - DateTime::from(Utc::now()) - } - }, - ), - }; - res.insert( - "first_update".to_string(), - Value::from(first_update.format(GLOBAL_TIME_FORMAT).to_string()), - ); - - Ok(res) -} diff --git a/backend/legacy/src/constants.rs b/backend/legacy/src/constants.rs deleted file mode 100644 index 5dd57e4..0000000 --- a/backend/legacy/src/constants.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub const GLOBAL_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; - -pub const PAGE_SIZE: u32 = 6; diff --git a/backend/legacy/src/logger.rs b/backend/legacy/src/logger.rs deleted file mode 100644 index 974a706..0000000 --- a/backend/legacy/src/logger.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::{env, io}; - -use chrono::Local; -use tracing::info; -use tracing_appender::{non_blocking, rolling}; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_subscriber::{fmt, Registry}; -use tracing_subscriber::fmt::format::Writer; -use tracing_subscriber::fmt::MakeWriter; -use tracing_subscriber::fmt::time::FormatTime; -use tracing_subscriber::layer::SubscriberExt; - -use crate::constants::GLOBAL_TIME_FORMAT; - -struct LocalTimer; - -impl FormatTime for LocalTimer { - fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { - write!(w, "{}", Local::now().format(GLOBAL_TIME_FORMAT)) - } -} - -type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; - -pub fn init() -> (WorkerGuard, WorkerGuard) { - let log_path = env::var("LOG_ROOT_DIR").unwrap_or_else(|e| { - panic!("[Fatal] failed to load LOG_ROOT_DIR: {}", e); - }); - - let log_path = format!("{}/backend", log_path); - - let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); - - let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); - - let log_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_timer(LocalTimer); - - let subscribers = Registry::default() - .with( - fmt::Layer::default() - .with_writer(info_log_writer) - .with_ansi(false), - ) - .with( - fmt::Layer::default() - .with_writer(err_log_writer) - .with_ansi(false), - ) - .with( - fmt::Layer::default() - .with_writer(io::stdout) - .with_ansi(true) - .event_format(log_format), - ); - - tracing::subscriber::set_global_default(subscribers) - .expect("[Fatal] Failed to setup logging system"); - - info!("Module Started: Logger"); - - // error!("test"); - (info_guard, error_guard) -} diff --git a/backend/legacy/src/markdown/meta.rs b/backend/legacy/src/markdown/meta.rs deleted file mode 100644 index b2d7cac..0000000 --- a/backend/legacy/src/markdown/meta.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::collections::HashMap; - -use regex::Regex; - -/// return the meta info contained in the .md file front matter -pub fn parse_meta(md_content: &str) -> HashMap { - let mut fields: HashMap = HashMap::new(); - let match_regex = Regex::new(r"(?s)---\n(.*\n)---\n").unwrap(); - if let Some(front_matter) = match_regex.captures(md_content) { - let fm_content = front_matter.get(1).unwrap().as_str(); - for line in fm_content.lines() { - if let Some((k, v)) = extract_kv(line) { - fields.insert(k, v); - } - } - } - fields -} - -fn extract_kv(line: &str) -> Option<(String, String)> { - let split: Vec<&str> = line.splitn(2, ':').map(|s| s.trim()).collect(); - if split.len() == 2 { - let key = split[0].to_string(); - let value = split[1].to_string(); - Some((key, value)) - } else { - None - } -} - -#[cfg(test)] -mod meta_test { - use std::error::Error; - - use crate::markdown::meta::parse_meta; - - #[test] - fn test_parse_meta_front_matter() -> Result<(), Box> { - let parsed_meta = parse_meta( - r##" ---- -title: Hello World! -tags: [ "test", "hello_world" ] ---- - "##, - ); - dbg!(&parsed_meta); - assert_eq!(parsed_meta.get("title"), Some(&"Hello World!".to_string())); - assert_eq!( - parsed_meta.get("tags"), - Some(&r#"[ "test", "hello_world" ]"#.to_string()) - ); - Ok(()) - } -} diff --git a/backend/legacy/src/markdown/mod.rs b/backend/legacy/src/markdown/mod.rs deleted file mode 100644 index 6b50816..0000000 --- a/backend/legacy/src/markdown/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod meta; diff --git a/backend/legacy/src/model/blog/hash.sql b/backend/legacy/src/model/blog/hash.sql deleted file mode 100644 index 7dc91de..0000000 --- a/backend/legacy/src/model/blog/hash.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); \ No newline at end of file diff --git a/backend/legacy/src/model/blog/meta.sql b/backend/legacy/src/model/blog/meta.sql deleted file mode 100644 index 6cf903a..0000000 --- a/backend/legacy/src/model/blog/meta.sql +++ /dev/null @@ -1,79 +0,0 @@ -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); - -DROP table meta; - -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY ['id', 'tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY (_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - -SELECT jsonb_strip_nulls(to_jsonb(t)) AS meta -FROM meta t -WHERE post_id = '8ca3fa2b-f040-448c-8885-81aab56db9bb'; - -WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = '8ca3fa2b-f040-448c-8885-81aab56db9bb'), - prev AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date < (SELECT date FROM target_post) - ORDER BY t.date DESC - LIMIT 1), - next AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date > (SELECT date FROM target_post) - ORDER BY t.date - LIMIT 1) -SELECT json_strip_nulls(json_build_object( - 'prev', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM prev), - 'next', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM next) - )) AS result; - diff --git a/backend/legacy/src/model/blog/mod.rs b/backend/legacy/src/model/blog/mod.rs deleted file mode 100644 index c282af7..0000000 --- a/backend/legacy/src/model/blog/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -pub const POST_BASE_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - header_img TEXT, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, - first_update TIMESTAMPTZ NOT NULL -); -"##; - -// base mode, will add more cols in runtime -pub const POST_META_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); -"##; - -pub const POST_HASH_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); -"##; - -pub const TAG_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); -"##; diff --git a/backend/legacy/src/model/blog/post.sql b/backend/legacy/src/model/blog/post.sql deleted file mode 100644 index 1afc493..0000000 --- a/backend/legacy/src/model/blog/post.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - header_img TEXT, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, - first_update TIMESTAMPTZ NOT NULL -); diff --git a/backend/legacy/src/model/blog/tag.sql b/backend/legacy/src/model/blog/tag.sql deleted file mode 100644 index 6c84b76..0000000 --- a/backend/legacy/src/model/blog/tag.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); diff --git a/backend/legacy/src/model/mod.rs b/backend/legacy/src/model/mod.rs deleted file mode 100644 index 21aa2c6..0000000 --- a/backend/legacy/src/model/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod blog; diff --git a/backend/macros/Cargo.toml b/backend/macros/Cargo.toml new file mode 100644 index 0000000..a11722a --- /dev/null +++ b/backend/macros/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "macros" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs new file mode 100644 index 0000000..b93cf3f --- /dev/null +++ b/backend/macros/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index 6f30a74..896c0c8 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -4,3 +4,11 @@ version = "0.1.0" edition = "2021" [dependencies] +uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } +chrono = { version = "0.4.38", features = ["serde", "alloc"] } +serde = { version = "1.0.209", features = ["derive"] } +serde_json = "1.0.128" +regex = "1.10.4" +yaml-rust2 = "0.8.1" +sha2 = "0.10.8" +base64 = "0.22.1" diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index b93cf3f..ce6345f 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -1,14 +1,55 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right +mod utils; + +use std::error::Error; +use std::fs; +use std::path::Path; +use std::str::FromStr; + +use base64::{engine::general_purpose, Engine}; +use serde::Serialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + +#[derive(Default, Debug, Serialize)] +pub struct MdFile { + file_id: Uuid, + file_name: String, + meta: Value, + content: String, + hash: String, } -#[cfg(test)] -mod tests { - use super::*; +impl MdFile { + pub async fn from_file(file_path: &str) -> Result> { + let raw_file = fs::read_to_string(file_path)?; + let raw_file_bytes = raw_file.as_bytes().to_vec(); + let (front_matter_str, content) = split_md_front_matter(raw_file); + let front_matter = parse_front_matter(&front_matter_str)?; + let hash = calculate_hash_base64(&raw_file_bytes); + let file_id = Uuid::from_str(front_matter["id"].to_string().as_str())?; + let file_name = Path::new(file_path) + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); + Ok(MdFile { + file_id, + file_name, + meta: front_matter, + content, + hash, + }) } } + +fn calculate_hash_base64(bytes: &Vec) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let hash = hasher.finalize().to_vec(); + general_purpose::STANDARD.encode(hash) +} diff --git a/backend/markdown/src/utils/front_matter.rs b/backend/markdown/src/utils/front_matter.rs new file mode 100644 index 0000000..5bca291 --- /dev/null +++ b/backend/markdown/src/utils/front_matter.rs @@ -0,0 +1,77 @@ +use crate::utils::yaml::yaml_to_serde; +use regex::Regex; +use serde_json::Value; +use std::error::Error; +use yaml_rust2::YamlLoader; + +pub fn split_md_front_matter(raw_file: String) -> (String, String) { + let re = Regex::new(r"(?s)^(---\n.*?\n---\n)(.*)").unwrap(); + if let Some(captures) = re.captures(raw_file.as_str()) { + let front_matter = captures.get(1).map_or("", |m| m.as_str()).to_string(); + let main_content = captures.get(2).map_or("", |m| m.as_str()).to_string(); + (front_matter, main_content) + } else { + (String::new(), raw_file) + } +} + +pub fn parse_front_matter(front_matter: &str) -> Result> { + let parsed_f_matter = YamlLoader::load_from_str(front_matter)?[0].clone(); + Ok(Value::from(yaml_to_serde(&parsed_f_matter))) +} + +#[cfg(test)] +mod tests { + use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + + #[test] + fn front_matter_to_string() { + let (front_matter, main_content) = split_md_front_matter( + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- + +test body + +"## + .to_string(), + ); + assert_eq!( + front_matter, + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- +"## + ); + assert_eq!(main_content, "\ntest body\n\n"); + } + + #[test] + fn test_front_matter_parser() { + parse_front_matter( + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- +"##, + ) + .expect("Failed to parse front matter"); + } +} diff --git a/backend/markdown/src/utils/mod.rs b/backend/markdown/src/utils/mod.rs new file mode 100644 index 0000000..27be997 --- /dev/null +++ b/backend/markdown/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod front_matter; +mod yaml; diff --git a/backend/markdown/src/utils/yaml.rs b/backend/markdown/src/utils/yaml.rs new file mode 100644 index 0000000..8c52f4a --- /dev/null +++ b/backend/markdown/src/utils/yaml.rs @@ -0,0 +1,29 @@ +use serde_json::Value; +use std::str::FromStr; +use yaml_rust2::Yaml; + +pub fn yaml_to_serde(yaml: &Yaml) -> Value { + match yaml { + Yaml::Null => Value::Null, + Yaml::Boolean(b) => Value::Bool(*b), + Yaml::Integer(i) => Value::Number(serde_json::Number::from(*i)), + Yaml::Real(s) => Value::Number(serde_json::Number::from_str(s).unwrap()), + Yaml::String(s) => Value::String(s.clone()), + Yaml::Array(arr) => Value::Array(arr.iter().map(yaml_to_serde).collect()), + Yaml::Hash(hash) => { + let obj: serde_json::map::Map = hash + .iter() + .map(|(k, v)| { + let key = match k { + Yaml::String(s) => s.clone(), + _ => "INVALID_KEY".to_string(), + }; + (key, yaml_to_serde(v)) + }) + .collect(); + Value::Object(obj) + } + Yaml::Alias(_) => Value::Null, + Yaml::BadValue => Value::Null, + } +} diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index 3e71f6c..1c4b3c9 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -8,7 +8,7 @@ import Footer from '@/components/Footer'; export const metadata: Metadata = { title: 'Lumitherの酒馆', - description: '一间位于时间夹缝中的旅馆，静候下一位旅者的到来...' + description: '一间位于时间夹缝中的酒馆，静候下一位旅者的到来...' }; export default function RootLayout({