diff --git a/.env b/.env index cf1db3d..3708ad5 100644 --- a/.env +++ b/.env @@ -6,13 +6,14 @@ POSTGRES_USER= POSTGRES_DB=nocturne # backend settings -GIT_WORK_DIR=../blog_posts_git + +#GIT_WORK_DIR=../blog_posts_git +#LOG_ROOT_DIR=/var/log/nocturne + BLOG_ENDPOINT=https://github.com/Lumither/blog-posts.git -LOG_ROOT_DIR=/var/log/nocturne +WORK_DIR=~/.nocturne BACKEND_PORT=3001 # frontend settings API_LOCAL_URL=http://localhost:3001 API_REMOTE_URL=https://lumither.com/api - - diff --git a/.gitignore b/.gitignore index 6299755..3694827 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ .env.* /container/*/.env -# idea related +# IDE related .idea +.vscode # exclude post repo /blog_posts_git diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5d3cbd3..6c75b0a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,25 +1,8 @@ -[package] -name = "backend" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -axum = "0.7.5" -dotenv = "0.15.0" -serde = { version = "1.0.199", features = ["derive"] } -serde_json = "1.0.116" -tokio = { version = "1.37.0", features = ["full"] } -uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } -sqlx = { version = "0.7.4", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } -regex = "1.10.4" -futures = { version = "0.3.30", features = ["executor"] } -tracing = "0.1.40" -tracing-subscriber = "0.3.18" -tracing-appender = "0.2.3" -log = "0.4.21" -sha2 = "0.10.8" -thiserror = "1.0.63" -base64 = "0.22.1" +[workspace] +resolver = "2" +members = [ + "api_server", + "constants", + "macros", + "markdown", +] diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml new file mode 100644 index 0000000..af17326 --- /dev/null +++ b/backend/api_server/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "api_server" +version = "0.1.0" +edition = "2021" + +[dependencies] +macros = { path = "../macros" } +constants = { path = "../constants" } +markdown = { path = "../markdown" } +chrono = { version = "0.4.38", features = ["serde", "alloc"] } +tracing = "0.1.40" +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } +tracing-appender = "0.2.3" +dotenv = "0.15.0" +dirs = "5.0.1" +tokio = { version = "1.40.0", features = ["full"] } +sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } +axum = { version = "0.7.5", features = ["macros"] } +serde = { version = "1.0.199", features = ["derive"] } +serde_json = "1.0.128" +uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } +futures = { version = "0.3.30", features = ["executor"] } +cron = "0.13.0" +git2 = "0.19.0" +thiserror = "2.0.3" diff --git a/backend/src/api/get/get_page_count.rs b/backend/api_server/src/blog/api/get_page_count.rs similarity index 95% rename from backend/src/api/get/get_page_count.rs rename to backend/api_server/src/blog/api/get_page_count.rs index 887e360..db540af 100644 --- a/backend/src/api/get/get_page_count.rs +++ b/backend/api_server/src/blog/api/get_page_count.rs @@ -1,4 +1,5 @@ -use crate::constants::PAGE_SIZE; +use constants::blog::PAGE_SIZE; + use axum::extract::State; use axum::Json; use serde_json::{json, Value}; diff --git a/backend/src/api/get/get_post.rs b/backend/api_server/src/blog/api/get_post.rs similarity index 99% rename from backend/src/api/get/get_post.rs rename to backend/api_server/src/blog/api/get_post.rs index a38eb89..2374dec 100644 --- a/backend/src/api/get/get_post.rs +++ b/backend/api_server/src/blog/api/get_post.rs @@ -1,12 +1,11 @@ -use std::str::FromStr; - use axum::{ extract::{Path, State}, http::StatusCode, Json, }; use serde_json::{json, Value}; -use sqlx::{Error, PgPool, query, Row}; +use sqlx::{query, Error, PgPool, Row}; +use std::str::FromStr; use tracing::error; use uuid::Uuid; diff --git a/backend/src/api/get/get_post_list.rs b/backend/api_server/src/blog/api/get_post_list.rs similarity index 60% rename from backend/src/api/get/get_post_list.rs rename to backend/api_server/src/blog/api/get_post_list.rs index 1b55610..9db7c61 100644 --- a/backend/src/api/get/get_post_list.rs +++ b/backend/api_server/src/blog/api/get_post_list.rs @@ -1,4 +1,7 @@ -use crate::constants::PAGE_SIZE; +use std::error::Error; + +use constants::blog::PAGE_SIZE; + use axum::{extract::Query, extract::State, http::StatusCode, Json}; use chrono::{DateTime, Utc}; use futures::future::join_all; @@ -13,6 +16,7 @@ pub struct Params { page: Option, } +#[axum::debug_handler] pub async fn get_post_list( State(db_connection): State, Query(params): Query, @@ -46,33 +50,72 @@ pub async fn get_post_list( } }; - let mut res = Map::new(); + let mut res = Vec::new(); let posts: Vec<_> = post_list .into_iter() - .map(|post| analyze_post(&db_connection, post)) + .map(|post| async { analyze_post(&db_connection, post).await }) .collect(); - let posts = join_all(posts).await; + for post in posts { + match post { + Ok(p) => res.push(p), + Err(e) => { + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!(("error", e.to_string()))), + )) + } + } + } - res.insert("posts".to_string(), Value::from(posts)); + // res.insert("posts".to_string(), Value::from(posts)); Ok(Json::from(Value::from(res))) } -async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Value { +async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result> { let post_id: Uuid = post.get("post_id"); let tags = fetch_post_tags(db_connection, post_id).await; + let meta = fetch_post_meta(db_connection, post_id).await.unwrap(); + let header_img = meta.get("header_img"); + let last_update = meta.get("last_update"); - json!({ + Ok(json!({ "post_id": post_id.to_string(), "title": post.get::("title"), "category": post.get::("category"), - "header_img": post.get::("header_img"), + "header_img": header_img, "sub_title": post.get::("sub_title"), "summary": post.get::("summary"), - "last_update": post.get::, _>("last_update"), + "last_update": last_update, "first_update": post.get::, _>("first_update"), "tags": tags, - }) + })) +} + +async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option { + match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta +FROM meta t +WHERE post_id = $1; + "#, + ) + .bind(post_id) + .fetch_one(db_connection) + .await + { + Ok(value) => Some(value.get::("meta")), + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "post meta data", + "meta", + post_id.to_string(), + e.to_string() + ); + None + } + } } async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec { diff --git a/backend/src/api/get/mod.rs b/backend/api_server/src/blog/api/mod.rs similarity index 100% rename from backend/src/api/get/mod.rs rename to backend/api_server/src/blog/api/mod.rs diff --git a/backend/api_server/src/blog/cron/check_update/error.rs b/backend/api_server/src/blog/cron/check_update/error.rs new file mode 100644 index 0000000..0d27c29 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/error.rs @@ -0,0 +1,49 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PostIdxError { + #[error("missing essential field `{}` for file `{}`", field, filename)] + MissingField { field: String, filename: String }, + + #[error( + "invalid field `{}` for file `{}`, <{}>: {}", + field_name, + filename, + err_field, + msg + )] + InvalidField { + field_name: String, + err_field: String, + filename: String, + msg: String, + }, + + #[error( + "database error on writing `{}` to table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBWriteFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, + + #[error( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBReadFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, +} diff --git a/backend/api_server/src/blog/cron/check_update/index.rs b/backend/api_server/src/blog/cron/check_update/index.rs new file mode 100644 index 0000000..f391639 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/index.rs @@ -0,0 +1,351 @@ +use std::error::Error; +use std::path::Path; + +use crate::blog::cron::check_update::utils::get_md_file_basename; +use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date}; +use markdown::MdFile; + +use sqlx::{query, PgPool, Pool, Postgres, Row}; +use tokio::runtime::Runtime; +use tracing::{info, warn}; + +async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + let post_id = post.file_id; + let title = match post.meta.get("title") { + None => { + return Err(PostIdxError::MissingField { + field: "title".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid title for `{}`", &post.filename); + "invalid title" + }), + }; + let sub_title = post + .meta + .get("sub_title") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let summary = post + .meta + .get("summary") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let content = post.content.as_str(); + let category = match post.meta.get("category") { + None => { + return Err(PostIdxError::MissingField { + field: "category".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid category for `{}`", &post.filename); + "invalid category" + }), + }; + let last_update = { + if let Some(last_update) = post.meta.get("last_update") { + match parse_date(last_update.as_str().unwrap()) { + Ok(date) => Some(date), + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "last_update".to_string(), + err_field: last_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + } + } else { + None + } + }; + let first_update = match post.meta.get("date") { + None => { + return Err(PostIdxError::MissingField { + field: "date".to_string(), + filename: post.filename.clone(), + }) + } + Some(first_update) => match parse_date(first_update.as_str().unwrap()) { + Ok(date) => date, + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "date".to_string(), + err_field: first_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + }, + }; + let filename = get_md_file_basename(&post.filename); + + if let Err(e) = query( + r##" +INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (post_id) DO +UPDATE SET + title = EXCLUDED.title, + summary = EXCLUDED.summary, + content = EXCLUDED.content, + last_update = EXCLUDED.last_update, + first_update = EXCLUDED.first_update, + sub_title = EXCLUDED.sub_title, + category = EXCLUDED.category, + filename = EXCLUDED.filename; + "##, + ) + .bind(post_id) + .bind(title) + .bind(summary) + .bind(content) + .bind(last_update) + .bind(first_update) + .bind(sub_title) + .bind(category) + .bind(filename) + .execute(db_connection) + .await + { + return Err( + PostIdxError::DBWriteFailure { + data_desc: "post base information".to_string(), + db_table: "Post".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + } + ); + } + Ok(()) +} + +async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Some(tags_string) = post.meta.get("tags").to_owned() { + let tags = match tags_string.as_array().map(|vec| { + vec.iter() + .map(|val| val.as_str().unwrap()) + .collect::>() + }) { + None => { + return Err(PostIdxError::MissingField { + field: "tags".to_string(), + filename: post.filename.clone(), + }) + } + Some(val) => val, + }; + + let post_id = post.file_id; + + // remove old record + if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") + .bind(post_id) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }); + } + + // add new record + for tag in tags { + match query("INSERT INTO Tag VALUES ($1, $2);") + .bind(post_id) + .bind(tag) + .execute(db_connection) + .await + { + Ok(_) => {} + Err(e) => { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }) + } + } + } + } + + Ok(()) +} + +async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { + let base64_form = post.hash.clone(); + match query("SELECT hash FROM Hash WHERE post_id = $1") + .bind(post.file_id) + .fetch_optional(db_connection) + .await + { + Ok(row) => { + let db_record: String = match row { + None => "".to_string(), + Some(row) => row.try_get("hash").unwrap_or("".to_string()), + }; + if base64_form == db_record { + Ok(false) + } else { + if let Err(e) = query( + r#" +INSERT INTO Hash (post_id, hash) VALUES ($1, $2) + ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, + ) + .bind(post.file_id) + .bind(&base64_form) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }); + } + Ok(true) + } + } + Err(e) => Err(PostIdxError::DBReadFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }), + } +} + +async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Err(e) = query( + // todo: sanitization + format!( + r#" +DO +$$ + DECLARE + _post_id UUID := '{}'; + _meta_cols JSONB := '{}'; + _exists BOOLEAN; + _col_name TEXT; + _col_value TEXT; + _sql TEXT; + _insert_columns TEXT := 'post_id'; + _insert_values TEXT := quote_literal(_post_id); + _update_set TEXT := ''; + _blacklist TEXT[] := ARRAY['tags']; + BEGIN + SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; + + FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) + LOOP + IF _col_name = ANY(_blacklist) THEN + CONTINUE; + END IF; + + IF NOT EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'meta' + AND column_name = _col_name + -- may change in the future + AND table_schema = 'public') THEN + _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; + EXECUTE _sql; + END IF; + _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); + _insert_values := _insert_values || ', ' || quote_literal(_col_value); + _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); + END LOOP; + + _update_set := substring(_update_set FROM 3); + + IF _exists THEN + _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); + EXECUTE _sql; + ELSE + _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; + EXECUTE _sql; + END IF; + END; +$$ LANGUAGE plpgsql; + "#, post.file_id.to_string().as_str(), post.meta + ).as_str() + ) + .execute(db_connection) + .await { + Err(PostIdxError::DBWriteFailure { + data_desc: "post meta data".to_string(), + db_table: "Meta".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }) + } else { + Ok(()) + } +} + +pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { + let md_file = MdFile::from_file(path)?; + + async fn trail(db_conn: &Pool, md_file: &MdFile) -> Result<(), PostIdxError> { + base(db_conn, md_file).await?; + tag(db_conn, md_file).await?; + meta(db_conn, md_file).await?; + hash(db_conn, md_file).await.map(|_| ()) + } + let mut cnt = 5; + + while let Err(e) = trail(db_conn, &md_file).await { + match e { + PostIdxError::DBWriteFailure { .. } => { + cnt -= 1; + } + PostIdxError::DBReadFailure { .. } => { + cnt -= 1; + } + _ => return Err(e.into()), + } + if cnt <= 0 { + return Err(e.into()); + } + } + + info!("indexed markdown file: {}", path.display()); + Ok(()) +} + +pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + if let Err(e) = query( + r#" +WITH deleted_post AS ( + DELETE FROM post + WHERE id = (SELECT id FROM post WHERE filename = $1) + RETURNING id +) +DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); + "#, + ) + .bind(file_basename) + .execute(db_conn) + .await + { + Err(Box::from( + format!("failed to drop index: {}", e).to_string(), + )) + } else { + Ok(()) + } + }) +} diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs new file mode 100644 index 0000000..9568ae2 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/mod.rs @@ -0,0 +1,146 @@ +use std::env; +use std::fs::remove_dir_all; +use std::path::{Path, PathBuf}; + +use crate::{ + blog::cron::check_update::{ + index::drop_index, + utils::{expand_path, get_md_file_basename, index_files, search_md}, + }, + utils::git, +}; +use constants::config::general::{default_value, var_name}; +use macros::panic_with_log; + +use git2::{Delta, Repository}; +use sqlx::{Pool, Postgres}; +use tracing::{error, info, warn, Level}; + +mod error; +mod index; +mod utils; + +pub fn task(db_connection: &Pool) { + // init + let git_work_dir = { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_path(default_log_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") + }; + + // clone & check update + + let git_url = match env::var(var_name::BLOG_GIT_URL) { + Ok(url) => url, + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to read `{}`: {}", + var_name::BLOG_GIT_URL, + e + ) + } + }; + let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) + .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); + let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) + .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); + + match Repository::open(&git_work_dir) { + Ok(repo) => { + // prevent git remote url mismatch (clean cache) + match repo.find_remote(&git_remote_name) { + Ok(remote) => { + if remote.url().is_some_and(|url| url != git_url) { + remove_dir_all(&git_work_dir).unwrap(); + warn!("blog content git({}) url mismatch, reinitialized", &git_url); + return; + } + } + Err(e) => { + remove_dir_all(&git_work_dir).unwrap(); + error!( + "unexpected error on indexing blog content git({}), reinitialized: {}", + &git_url, e + ); + } + } + + match git::sync(&git_remote_name, &git_remote_branch, &repo) { + Ok(updates) => { + if !updates.is_empty() { + // incremental update + info!( + "blog content git({}) update detected: {:?}", + &git_url, updates + ); + + let mut update_md: Vec = vec![]; + + for update in updates { + if [Delta::Deleted, Delta::Untracked].contains(&update.status) { + if let Some(path) = &update.old_path { + if drop_index( + &get_md_file_basename( + path.file_name().unwrap().to_str().unwrap(), + ), + db_connection, + ) + .is_ok() + { + info!( + "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", + &git_url, + &path.display() + ); + } else { + error!( + "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", + &git_url, + &path.display() + ) + } + } + } else if let Some(path) = &update.new_path { + if path.components().any(|comp| comp.as_os_str() == "posts") + && path.extension().is_some_and(|ext| ext == "md") + { + update_md.push(path.to_path_buf()); + } + } + } + + index_files(&update_md, db_connection); + } + } + Err(e) => { + error!( + "failed to sync blog content git({}) updates: {}", + &git_url, e + ); + } + } + } + Err(_) => { + match Repository::clone(&git_url, &git_work_dir) { + Ok(_repo) => { + // full repo index + let files = search_md(git_work_dir.join("posts")); + index_files(&files, db_connection); + } + Err(e) => { + error!("failed to clone blog content git({}): {}", &git_url, e); + } + } + } + }; +} diff --git a/backend/api_server/src/blog/cron/check_update/utils.rs b/backend/api_server/src/blog/cron/check_update/utils.rs new file mode 100644 index 0000000..d0da864 --- /dev/null +++ b/backend/api_server/src/blog/cron/check_update/utils.rs @@ -0,0 +1,86 @@ +use dirs::home_dir; +use std::path::PathBuf; + +use crate::blog::cron::check_update::index::index_file; +use chrono::{DateTime, FixedOffset, ParseResult}; +use constants::config::general::{default_value, var_name}; +use constants::time::DEFAULT_DATE_FORMAT; + +use futures::executor::block_on; +use futures::future::join_all; +use sqlx::{Pool, Postgres}; +use tokio::runtime::Runtime; +use tokio::spawn; +use tracing::{error, warn}; + +pub fn search_md(entry: PathBuf) -> Vec { + if entry.ends_with(".git") { + return vec![]; + } + if entry.is_dir() { + if let Ok(files) = entry.read_dir() { + files + .flat_map(|file| { + let file_path = file.unwrap().path(); + search_md(file_path) + }) + .collect() + } else { + vec![] + } + } else if entry.extension().is_some_and(|ext| ext == "md") { + vec![entry] + } else { + vec![] + } +} + +pub fn index_files(files: &[PathBuf], db_conn: &Pool) { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let files_clone = files.to_owned(); + let handlers = files_clone + .into_iter() + .map(|file: PathBuf| { + let conn_clone = db_conn.clone(); + spawn(async move { + if let Err(e) = index_file(&file, &conn_clone).await { + error!("failed to index markdown file `{}`: {}", file.display(), e); + } + }) + }) + .collect::>(); + + block_on(join_all(handlers)); + }); +} + +pub fn expand_path(path: String) -> String { + if path.starts_with('~') { + home_dir() + .map(|dir| { + dir.join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + }) + .unwrap_or_else(|| { + warn!( + "failed to parse env var `{}`, using default value `{}`", + var_name::WORK_DIR, + default_value::WORK_DIR + ); + default_value::WORK_DIR.to_string() + }) + } else { + path + } +} + +pub fn parse_date(date_str: &str) -> ParseResult> { + DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) +} + +pub fn get_md_file_basename(f_name: &str) -> String { + f_name.replace(".md", "") +} diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs new file mode 100644 index 0000000..e8fd785 --- /dev/null +++ b/backend/api_server/src/blog/cron/mod.rs @@ -0,0 +1,14 @@ +use crate::scheduler::tasks::basic::BasicTask; +use crate::scheduler::tasks::Task; +use sqlx::{Pool, Postgres}; + +pub mod check_update; + +pub fn get_tasks(db_connection: Pool) -> Vec> { + let db_conn_clone = db_connection.clone(); + vec![ + BasicTask::new(move || check_update::task(&db_conn_clone), "0 * * * * *") + .unwrap() + .to_task(), + ] +} diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs new file mode 100644 index 0000000..6d14b15 --- /dev/null +++ b/backend/api_server/src/blog/mod.rs @@ -0,0 +1,17 @@ +mod api; +pub mod cron; + +use crate::blog::api::{ + get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, +}; + +use axum::routing::get; +use axum::Router; +use sqlx::{Pool, Postgres}; + +pub fn get_router() -> Router> { + Router::new() + .route("/get_post_list", get(get_post_list)) + .route("/get_page_count", get(get_page_count)) + .route("/get_post/:post_id", get(get_post)) +} diff --git a/backend/api_server/src/init/cron.rs b/backend/api_server/src/init/cron.rs new file mode 100644 index 0000000..429537a --- /dev/null +++ b/backend/api_server/src/init/cron.rs @@ -0,0 +1,12 @@ +use crate::blog; +use crate::scheduler::Scheduler; +use sqlx::{Pool, Postgres}; + +pub fn start(db_pool: Pool) { + let scheduler = Scheduler::new(); + let _ = vec![blog::cron::get_tasks(db_pool.clone())] + .into_iter() + .map(|list| scheduler.insert_list(list)) + .collect::>(); + scheduler.start().unwrap() +} diff --git a/backend/api_server/src/init/database.rs b/backend/api_server/src/init/database.rs new file mode 100644 index 0000000..c2a8aad --- /dev/null +++ b/backend/api_server/src/init/database.rs @@ -0,0 +1,41 @@ +use std::env; +use std::error::Error; +use std::str::FromStr; + +use constants::config::database::{default_value, var_name}; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, Pool, Postgres}; +use tracing::info; + +pub async fn init() -> Result, Box> { + let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) + .unwrap() + .disable_statement_logging(); + Ok(PgPoolOptions::new() + .max_connections(1) // todo: temporal solution, to be fixed + .connect_with(db_connect_option) + .await?) +} + +fn parse_db_uri() -> String { + env::var(var_name::URI).unwrap_or_else(|_| { + info!( + "`{}` env var is not set, \ + using `{}`, `{}`, `{}`, `{}` and `{}`", + var_name::URI, + var_name::USERNAME, + var_name::PASSWORD, + var_name::HOST, + var_name::PORT, + var_name::DBNAME + ); + let postgres_user = env::var(var_name::USERNAME).unwrap_or(default_value::USERNAME.into()); + let postgres_password = + env::var(var_name::PASSWORD).unwrap_or(default_value::PASSWORD.into()); + let db_host = env::var(var_name::HOST).unwrap_or(default_value::HOST.into()); + let db_port = env::var(var_name::PORT).unwrap_or(default_value::PORT.into()); + let postgres_db = env::var(var_name::DBNAME).unwrap_or(default_value::DBNAME.into()); + let db_type = default_value::DATABASE_TYPE; + format!("{db_type}://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") + }) +} diff --git a/backend/api_server/src/init/logger.rs b/backend/api_server/src/init/logger.rs new file mode 100644 index 0000000..5fd56b4 --- /dev/null +++ b/backend/api_server/src/init/logger.rs @@ -0,0 +1,106 @@ +use std::path::Path; +use std::{env, io}; + +use chrono::Local; +use constants::config::general::{default_value, var_name}; +use dirs::home_dir; +use tracing::info; +use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; +use tracing_subscriber::{ + filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, + layer::SubscriberExt, Layer, Registry, +}; + +struct LocalTimer; + +impl FormatTime for LocalTimer { + fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { + write!( + w, + "{}", + Local::now().format(constants::time::DEFAULT_TIME_FORMAT) + ) + } +} + +type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; + +pub fn init() -> (WorkerGuard, WorkerGuard) { + let log_path = match env::var(var_name::LOG_DIR) { + Ok(path) => Path::new(&path).to_path_buf(), + Err(_) => { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_home_path(default_log_path) + } else { + let fallback_path = expand_home_path(default_value::WORK_DIR.to_string()); + println!( + "[warn] unset env var `{}` and `{}`, trying to log on fallback path: {}/log", + var_name::LOG_DIR, + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("log") + } + }; + + let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); + let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); + + let info_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let err_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let stdout_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_timer(LocalTimer); + + let subscribers = Registry::default() + .with( + fmt::Layer::default() + .with_writer(info_log_writer) + .event_format(info_format), + ) + .with( + fmt::Layer::default() + .with_writer(err_log_writer) + .event_format(err_format) + .with_filter(LevelFilter::ERROR), + ) + .with( + fmt::Layer::default() + .with_writer(io::stdout) + .with_ansi(true) + .event_format(stdout_format), + ); + + tracing::subscriber::set_global_default(subscribers) + .expect("[fatal] failed to setup logging system"); + + info!("logging to dir: {}", log_path.to_str().unwrap()); + + (info_guard, error_guard) +} + +fn expand_home_path(path: String) -> String { + if path.starts_with('~') { + let home_path = home_dir() + .unwrap_or_else(|| panic!("[fatal] unable to locate home dir when parsing env `{}`, try to use absolute path instead", var_name::WORK_DIR)); + home_path + .join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + } else { + path + } +} diff --git a/backend/api_server/src/init/misc.rs b/backend/api_server/src/init/misc.rs new file mode 100644 index 0000000..6a08bc2 --- /dev/null +++ b/backend/api_server/src/init/misc.rs @@ -0,0 +1,14 @@ +use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds}; +use tracing::warn; + +pub fn init() { + // git2 option + unsafe { + if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) { + warn!("failed to set git2 server connect timeout: {}", e); + } + if let Err(e) = set_server_timeout_in_milliseconds(60_000) { + warn!("failed to set git2 server timeout: {}", e); + } + } +} diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs new file mode 100644 index 0000000..507d52a --- /dev/null +++ b/backend/api_server/src/init/mod.rs @@ -0,0 +1,58 @@ +mod cron; +mod database; +pub mod logger; +mod misc; +mod schema; +pub mod server; + +use macros::panic_with_log; +use std::{env, panic}; +use tracing::{info, Level}; + +pub fn load_env() { + if let Some(env_file) = env::args().nth(1) { + match dotenv::from_filename(&env_file) { + Ok(_) => { + println!("[info] env loaded from {}, starting up", env_file); + } + Err(e) => { + panic!("[fatal] failed to read {}: {}", env_file, e); + } + } + } else { + println!("[info] no .env file referred, starting up") + } +} + +pub async fn start() { + misc::init(); + + // connect database + let db_pool = match database::init().await { + Ok(pool) => { + info!("database connection pool initialized"); + pool + } + Err(e) => { + panic_with_log!( + Level::ERROR, + "database connection pool failed to initialize: {}", + e + ); + } + }; + + // schema init + schema::init(&db_pool).await; + + // load cron task + cron::start(db_pool.clone()); + + // api serve + match server::start(db_pool).await { + Ok(_) => {} + Err(e) => { + panic_with_log!(Level::ERROR, "server panicked: {}", e) + } + } +} diff --git a/backend/api_server/src/init/schema.rs b/backend/api_server/src/init/schema.rs new file mode 100644 index 0000000..ef9def0 --- /dev/null +++ b/backend/api_server/src/init/schema.rs @@ -0,0 +1,49 @@ +use sqlx::{query, Pool, Postgres}; +use tracing::info; + +const POST_BASE_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Post +( + post_id UUID PRIMARY KEY, + title VARCHAR(255) NOT NULL, + category VARCHAR(255) NOT NULL, + filename TEXT UNIQUE NOT NULL, + summary TEXT, + sub_title TEXT, + content TEXT NOT NULL, + last_update TIMESTAMPTZ, + first_update TIMESTAMPTZ NOT NULL +); +"##; + +// base model, will add more cols at runtime +const POST_META_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Meta +( + post_id UUID PRIMARY KEY +); +"##; + +const POST_HASH_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Hash +( + post_id UUID PRIMARY KEY, + hash VARCHAR(44) +); +"##; + +const TAG_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Tag +( + post_id UUID NOT NULL, + tag VARCHAR(255) NOT NULL +); +"##; + +pub async fn init(db_connection: &Pool) { + query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); + query(POST_META_MODEL).execute(db_connection).await.unwrap(); + query(TAG_MODEL).execute(db_connection).await.unwrap(); + query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); + info!("database schema initialized"); +} diff --git a/backend/api_server/src/init/server.rs b/backend/api_server/src/init/server.rs new file mode 100644 index 0000000..703a043 --- /dev/null +++ b/backend/api_server/src/init/server.rs @@ -0,0 +1,52 @@ +use std::env; +use std::error::Error; +use std::num::ParseIntError; +use std::process::exit; + +use crate::blog; +use axum::Router; +use constants::config::server::{default_value, var_name}; +use macros::panic_with_log; +use sqlx::{Pool, Postgres}; +use tracing::{error, info, warn, Level}; + +pub async fn start(db_pool: Pool) -> Result<(), Box> { + let app = Router::new() + .nest("/blog", blog::get_router()) + .with_state(db_pool); + + let port: u32 = match env::var(var_name::PORT) { + Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { + warn!( + "failed to parse `{}`, using default port: {}", + var_name::PORT, + e.to_string() + ); + default_value::PORT + }), + Err(_) => default_value::PORT, + }; + + let listener = + match tokio::net::TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await { + Ok(listener) => { + info!("server started on {}:{}", default_value::HOST, port); + listener + } + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to start server on {}:{}: {}", + default_value::HOST, + port, + e.to_string() + ); + } + }; + + axum::serve(listener, app).await.unwrap_or_else(|e| { + error!("failed to start axum: {}", e.to_string()); + exit(1); + }); + Ok(()) +} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs new file mode 100644 index 0000000..8aab2c7 --- /dev/null +++ b/backend/api_server/src/main.rs @@ -0,0 +1,13 @@ +mod blog; +mod init; +mod scheduler; +mod utils; + +#[tokio::main] +async fn main() { + init::load_env(); + + let _guards = init::logger::init(); + + init::start().await; +} diff --git a/backend/api_server/src/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs new file mode 100644 index 0000000..ce86797 --- /dev/null +++ b/backend/api_server/src/scheduler/mod.rs @@ -0,0 +1,194 @@ +use crate::{ + scheduler::scheduler_error::SchedulerError, + scheduler::scheduler_error::SchedulerError::IdAlreadyRunning, + scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper, + scheduler::tasks::Task, +}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::RwLock; +use std::{collections::HashMap, sync::Arc, thread}; +use tracing::log::error; +use uuid::Uuid; + +pub mod scheduler_error; +mod sleeper; +mod task_func; +pub mod tasks; + +struct TaskStatus { + sleeper: Option>, + is_alive: AtomicBool, +} + +#[derive(Default)] +pub struct Scheduler { + tasks: Arc>>>>, + running: Arc>>>, +} + +impl Scheduler { + /// Start all tasks in the scheduler + pub fn start(&self) -> Result<(), SchedulerError> { + for (k, _v) in self.tasks.read()?.iter() { + self.run_non_blocking(k)? + } + Ok(()) + } + + /// Terminate all tasks in the scheduler + pub fn halt() {} + + fn run_task( + running: Arc>>>, + id: Uuid, + task: Arc>, + ) -> Result<(), SchedulerError> { + if running.read()?.get(&id).is_some() { + return Err(IdAlreadyRunning); + } else { + running.write()?.insert( + id, + TaskStatus { + sleeper: None, + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + loop { + if let Some(next_execution) = task.get_next_execution() { + // guarantee is alive + let task_status = { + let running_reading_guard = running.read()?; + running_reading_guard.get(&id).cloned().clone() + }; + if let Some(task_status) = task_status { + if !&task_status.is_alive.load(Ordering::SeqCst) { + { + let mut running_lock_guard = running.write()?; + running_lock_guard.remove(&id); + } + return Ok(()); + } else { + let sleeper = Arc::from(SkippableSleeper::from(next_execution)); + { + let mut running_lock_guard = running.write()?; + running_lock_guard.insert( + id, + TaskStatus { + sleeper: Some(sleeper.clone()), + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + sleeper.start(); + task.call(); + } + } + } else { + error!("unexpected error "); + break; + } + } + Ok(()) + } + + /// Run a task with `id` + pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + Self::run_task(self.running.clone(), id, task.clone()) + } else { + Err(IdNotFound) + } + } + + pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + let running_clone = self.running.clone(); + let task_clone = task.clone(); + thread::spawn(move || Self::run_task(running_clone, id, task_clone)); + Ok(()) + } else { + Err(IdNotFound) + } + } + + fn get_running(&self, id: &Uuid) -> Option> { + let ret = self.running.read().unwrap().get(id).cloned(); + ret + } + + /// Stop a task with `id` + pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + status.is_alive.store(false, Ordering::SeqCst); + if let Some(s) = &status.sleeper { + s.skip(); + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + if let Some(s) = &status.sleeper { + s.skip() + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn insert(&self, task: Box) -> Result { + let id = Uuid::new_v4(); + self.tasks.write()?.insert(id, Arc::new(task)); + Ok(id) + } + + pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { + tasks.into_iter().map(|t| self.insert(t)).collect() + } + + pub fn new() -> Self { + Self::default() + } +} + +#[cfg(test)] +mod tests { + use crate::scheduler::tasks::basic::BasicTask; + use crate::scheduler::Scheduler; + use std::error::Error; + use std::thread; + use std::time::Duration; + + #[test] + fn test_run() -> Result<(), Box> { + let scheduler = Scheduler::default(); + let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; + let id = scheduler.insert(Box::new(task))?; + for i in 1..=2 { + println!("Loop {} start", i); + println!("Run {}", i); + scheduler.run_non_blocking(&id)?; + thread::sleep(Duration::from_secs(4)); + scheduler.stop(&id)?; + println!("Stop {}", i); + thread::sleep(Duration::from_secs(2)); + println!("Loop {} finished", i); + } + Ok(()) + } +} diff --git a/backend/api_server/src/scheduler/scheduler_error.rs b/backend/api_server/src/scheduler/scheduler_error.rs new file mode 100644 index 0000000..43f30dd --- /dev/null +++ b/backend/api_server/src/scheduler/scheduler_error.rs @@ -0,0 +1,44 @@ +use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; +use std::error::Error; +use std::fmt::{Debug, Display, Formatter}; +use std::sync::PoisonError; + +pub enum SchedulerError { + IdNotFound, + IdAlreadyRunning, + MutexPoison(String), +} + +impl From> for SchedulerError { + fn from(value: PoisonError) -> Self { + MutexPoison(value.to_string()) + } +} + +impl Debug for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) + } +} + +impl Display for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + SchedulerError::IdNotFound => { + "SchedulerError::IdNotFound".to_string() + } + SchedulerError::IdAlreadyRunning => { + "SchedulerError::IdAlreadyRunning".to_string() + } + MutexPoison(str) => { + format!("SchedulerError::MutexPoison: {}", str) + } + } + ) + } +} + +impl Error for SchedulerError {} diff --git a/backend/api_server/src/scheduler/sleeper.rs b/backend/api_server/src/scheduler/sleeper.rs new file mode 100644 index 0000000..4155e57 --- /dev/null +++ b/backend/api_server/src/scheduler/sleeper.rs @@ -0,0 +1,77 @@ +use chrono::{DateTime, Utc}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; +use std::time::Duration; + +#[derive(Clone, Debug, Default)] +pub struct SkippableSleeper { + pub duration: Duration, + state: Arc<(Mutex, Condvar)>, +} + +impl SkippableSleeper { + pub fn new(duration: Duration) -> Self { + SkippableSleeper { + duration, + state: Arc::new((Mutex::new(false), Condvar::new())), + } + } + + // Blocks until the duration has passed or skip() is called + pub fn start(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + let state_clone = Arc::clone(&self.state); + let duration_clone = self.duration; + thread::spawn(move || { + thread::sleep(duration_clone); + let (lock, cvar) = &*state_clone; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + }); + while !*received { + received = cvar.wait(received).unwrap(); + } + } + + // Skip the blocking delay + pub fn skip(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + } +} + +impl From> for SkippableSleeper { + fn from(value: DateTime) -> Self { + let from_utc_timestamp = value.timestamp() as u64; + let curr_utc_timestamp = Utc::now().timestamp() as u64; + SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) + } +} + +#[cfg(test)] +mod test { + use crate::scheduler::sleeper::SkippableSleeper; + use std::thread; + use std::time::Duration; + + #[test] + fn usage() { + let sleeper = SkippableSleeper::new(Duration::new(10, 0)); + thread::spawn({ + let sleeper = sleeper.clone(); + move || { + println!("Sleeping..."); + sleeper.start(); + println!("Woke up!"); + } + }); + println!("Start Sleeping..."); + thread::sleep(Duration::from_secs(4)); + println!("Skipping sleep..."); + sleeper.skip(); + } +} diff --git a/backend/api_server/src/scheduler/task_func.rs b/backend/api_server/src/scheduler/task_func.rs new file mode 100644 index 0000000..af27ded --- /dev/null +++ b/backend/api_server/src/scheduler/task_func.rs @@ -0,0 +1,3 @@ +pub trait TaskFunc: Fn() + Send + Sync + 'static {} + +impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} diff --git a/backend/api_server/src/scheduler/tasks/basic.rs b/backend/api_server/src/scheduler/tasks/basic.rs new file mode 100644 index 0000000..71fc3a6 --- /dev/null +++ b/backend/api_server/src/scheduler/tasks/basic.rs @@ -0,0 +1,37 @@ +use crate::scheduler::task_func::TaskFunc; +use crate::scheduler::tasks::Task; +use chrono::{DateTime, Utc}; +use cron::Schedule; +use std::error::Error; +use std::str::FromStr; + +pub struct BasicTask { + schedule: Schedule, + func: Box, +} + +impl BasicTask { + pub fn new(func: F, cron_expr: &str) -> Result> + where + F: TaskFunc, + { + Ok(BasicTask { + schedule: Schedule::from_str(cron_expr)?, + func: Box::new(func), + }) + } + + pub fn to_task(self) -> Box { + Box::new(self) + } +} + +impl Task for BasicTask { + fn get_next_execution(&self) -> Option> { + self.schedule.upcoming(Utc).next() + } + + fn call(&self) { + (self.func)() + } +} diff --git a/backend/api_server/src/scheduler/tasks/mod.rs b/backend/api_server/src/scheduler/tasks/mod.rs new file mode 100644 index 0000000..7e3b87a --- /dev/null +++ b/backend/api_server/src/scheduler/tasks/mod.rs @@ -0,0 +1,8 @@ +use chrono::{DateTime, Utc}; + +pub mod basic; + +pub trait Task: Send + Sync { + fn get_next_execution(&self) -> Option>; + fn call(&self); +} diff --git a/backend/api_server/src/utils/git.rs b/backend/api_server/src/utils/git.rs new file mode 100644 index 0000000..6743058 --- /dev/null +++ b/backend/api_server/src/utils/git.rs @@ -0,0 +1,195 @@ +/// git merge +/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs +/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 +/// +use git2::{ + build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, +}; +use std::path::PathBuf; +use tracing::{error, warn}; + + +#[derive(Debug, Eq, PartialEq)] +pub struct FileDelta { + pub old_path: Option, + pub new_path: Option, + pub status: Delta, +} + +fn do_fetch<'a>( + repo: &'a Repository, + refs: &[&str], + remote: &'a mut Remote, +) -> Result, git2::Error> { + let mut fetch_option = FetchOptions::new(); + + // Always fetch all tags. + // Perform a download and also update tips + fetch_option.download_tags(AutotagOption::All); + remote.fetch(refs, Some(&mut fetch_option), None)?; + + let fetch_head = repo.find_reference("FETCH_HEAD")?; + repo.reference_to_annotated_commit(&fetch_head) +} + +fn fast_forward( + repo: &Repository, + lb: &mut Reference, + rc: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let name = match lb.name() { + Some(s) => s.to_string(), + None => String::from_utf8_lossy(lb.name_bytes()).to_string(), + }; + let msg = format!("fast-forward({}) -> {}", name, rc.id()); + // println!("{}", msg); + lb.set_target(rc.id(), &msg)?; + repo.set_head(&name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + // For some reason, the force is required to make the working directory actually get updated + // I suspect we should be adding some logic to handle dirty working directory states, + // but this is just an example so maybe not. + .force(), + ))?; + Ok(()) +} + +fn normal_merge( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let ancestor = repo + .find_commit(repo.merge_base(local.id(), remote.id())?)? + .tree()?; + let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?; + + if idx.has_conflicts() { + warn!("merge conflicts detected"); + repo.checkout_index(Some(&mut idx), None)?; + return Ok(()); + } + let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; + // now create the merge commit + let msg = format!("merge({}) -> {}", remote.id(), local.id()); + let sig = repo.signature()?; + let local_commit = repo.find_commit(local.id())?; + let remote_commit = repo.find_commit(remote.id())?; + + // Do our merge commit and set the current branch head to that commit. + let _merge_commit = repo.commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &result_tree, + &[&local_commit, &remote_commit], + )?; + // Set working tree to match head. + repo.checkout_head(None)?; + Ok(()) +} + +fn do_merge<'a>( + repo: &'a Repository, + remote_branch: &str, + fetch_commit: AnnotatedCommit<'a>, +) -> Result, git2::Error> { + // 1. do a merge analysis + let analysis = repo.merge_analysis(&[&fetch_commit])?; + + let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; + + // 2. Do the appropriate merge + if analysis.0.is_fast_forward() { + // println!("Doing a fast-forward"); + // do a fast-forward + let ref_name = format!("refs/heads/{}", remote_branch); + match repo.find_reference(&ref_name) { + Ok(mut r) => { + fast_forward(repo, &mut r, &fetch_commit)?; + } + Err(_) => { + // The branch doesn't exist, so set the reference to the + // commit directly. Usually this is because you are pulling + // into an empty repository. + repo.reference( + &ref_name, + fetch_commit.id(), + true, + &format!("({}) -> {}", remote_branch, fetch_commit.id()), + )?; + repo.set_head(&ref_name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + .allow_conflicts(true) + .conflict_style_merge(true) + .force(), + ))?; + } + }; + } else if analysis.0.is_normal() { + // do a normal merge + normal_merge(repo, &head_commit, &fetch_commit)?; + } + // else { + // // already up-to-date + // println!("Nothing to do..."); + // } + + calculate_diff(repo, &head_commit, &fetch_commit) +} + +fn calculate_diff( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result, git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?; + + let mut updated_files: Vec = Vec::new(); + + if let Err(e) = diff.foreach( + &mut |d, _| { + // let delta = d.status(); + // + // let n_path = d.new_file().path(); + // let o_path = d.old_file().path(); + // + // if let Some(path) = n_path { + // updated_files.insert(path.to_owned()); + // } + // if let Some(path) = o_path { + // updated_files.insert(path.to_owned()); + // } + updated_files.push(FileDelta { + old_path: d.old_file().path().map(PathBuf::from), + new_path: d.new_file().path().map(PathBuf::from), + status: d.status(), + }); + true // Continue + }, + None, + None, + None, + ) { + error!("error at calculating diff: {:?}", e); + } + + Ok(updated_files.into_iter().collect()) +} + +pub fn sync( + remote_name: &str, + remote_branch: &str, + repo: &Repository, +) -> Result, git2::Error> { + let mut remote = repo.find_remote(remote_name)?; + let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; + do_merge(repo, remote_branch, fetch_commit) +} diff --git a/backend/api_server/src/utils/mod.rs b/backend/api_server/src/utils/mod.rs new file mode 100644 index 0000000..c2bf1c3 --- /dev/null +++ b/backend/api_server/src/utils/mod.rs @@ -0,0 +1 @@ +pub mod git; diff --git a/backend/constants/Cargo.toml b/backend/constants/Cargo.toml new file mode 100644 index 0000000..db345c7 --- /dev/null +++ b/backend/constants/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "constants" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/backend/constants/src/blog.rs b/backend/constants/src/blog.rs new file mode 100644 index 0000000..f0e80e6 --- /dev/null +++ b/backend/constants/src/blog.rs @@ -0,0 +1 @@ +pub const PAGE_SIZE: u32 = 6; diff --git a/backend/constants/src/config.rs b/backend/constants/src/config.rs new file mode 100644 index 0000000..bf6ba77 --- /dev/null +++ b/backend/constants/src/config.rs @@ -0,0 +1,48 @@ +// Database related +pub mod database { + pub mod var_name { + pub const URI: &str = "DATABASE_URL"; + pub const USERNAME: &str = "POSTGRES_USER"; + pub const PASSWORD: &str = "POSTGRES_PASSWORD"; + pub const HOST: &str = "POSTGRES_HOST"; + pub const PORT: &str = "POSTGRES_PORT"; + pub const DBNAME: &str = "POSTGRES_DB"; + } + + pub mod default_value { + pub const USERNAME: &str = ""; + pub const PASSWORD: &str = ""; + pub const HOST: &str = "localhost"; + pub const PORT: &str = "5432"; + pub const DBNAME: &str = "nocturne"; + pub const DATABASE_TYPE: &str = "postgresql"; + } +} + +pub mod server { + pub mod var_name { + pub const PORT: &str = "BACKEND_PORT"; + pub const HOST: &str = "BACKEND_HOST"; + } + + pub mod default_value { + pub const PORT: u32 = 3001; + pub const HOST: &str = "0.0.0.0"; + } +} + +pub mod general { + pub mod var_name { + pub const LOG_DIR: &str = "LOG_DIR"; + pub const WORK_DIR: &str = "WORK_DIR"; + pub const BLOG_GIT_URL: &str = "BLOG_GIT_URL"; + pub const BLOG_GIT_REMOTE_BRANCH: &str = "BLOG_GIT_REMOTE_BRANCH"; + pub const BLOG_GIT_REMOTE_NAME: &str = "BLOG_GIT_REMOTE_NAME"; + } + + pub mod default_value { + pub const WORK_DIR: &str = "~/.nocturne"; + pub const BLOG_GIT_REMOTE_BRANCH: &str = "master"; + pub const BLOG_GIT_REMOTE_NAME: &str = "origin"; + } +} diff --git a/backend/constants/src/lib.rs b/backend/constants/src/lib.rs new file mode 100644 index 0000000..1b58b94 --- /dev/null +++ b/backend/constants/src/lib.rs @@ -0,0 +1,3 @@ +pub mod blog; +pub mod config; +pub mod time; diff --git a/backend/constants/src/time.rs b/backend/constants/src/time.rs new file mode 100644 index 0000000..d7b29ac --- /dev/null +++ b/backend/constants/src/time.rs @@ -0,0 +1,2 @@ +pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; +pub const DEFAULT_DATE_FORMAT: &str = "%Y-%m-%d%zT%H%M%S"; diff --git a/backend/macros/Cargo.toml b/backend/macros/Cargo.toml new file mode 100644 index 0000000..6291521 --- /dev/null +++ b/backend/macros/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "macros" +version = "0.1.0" +edition = "2021" + +[dependencies] +tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs new file mode 100644 index 0000000..aad31d7 --- /dev/null +++ b/backend/macros/src/lib.rs @@ -0,0 +1 @@ +mod log; diff --git a/backend/macros/src/log.rs b/backend/macros/src/log.rs new file mode 100644 index 0000000..f7dd357 --- /dev/null +++ b/backend/macros/src/log.rs @@ -0,0 +1,45 @@ +/// log the message at specific log level and exit(1) +/// +/// # Example +/// ```rust +/// +/// use macros::panic_with_log; +/// use tracing::Level; +/// +/// panic_with_log!(Level::ERROR, "Test panicked: {}", 42); +/// ``` +#[macro_export] +macro_rules! panic_with_log { + ($level:expr, $msg:literal $(, $($arg:tt)*)?) => {{ + use std::process; + use tracing::event; + event!($level, $msg $(, $($arg)*)?); + process::exit(1); + }}; +} + +#[cfg(test)] +mod tests { + #[test] + fn test_panic_with_log() { + use std::io; + use tracing::Level; + use tracing_subscriber::prelude::*; + use tracing_subscriber::{fmt, Registry}; + + let format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true); + + let subscriber = Registry::default().with( + fmt::Layer::default() + .with_writer(io::stderr) + .event_format(format), + ); + + tracing::subscriber::set_global_default(subscriber).unwrap(); + + panic_with_log!(Level::ERROR, "Test panicked: {}", 42); + } +} diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml new file mode 100644 index 0000000..e0ca45e --- /dev/null +++ b/backend/markdown/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "markdown" +version = "0.1.0" +edition = "2021" + +[dependencies] +uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } +serde = { version = "1.0.209", features = ["derive"] } +serde_json = "1.0.128" +regex = "1.10.4" +yaml-rust2 = "0.9.0" +sha2 = "0.10.8" +base64 = "0.22.1" +thiserror = "2.0.3" diff --git a/backend/markdown/src/error.rs b/backend/markdown/src/error.rs new file mode 100644 index 0000000..a93585f --- /dev/null +++ b/backend/markdown/src/error.rs @@ -0,0 +1,16 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("missing front-matter field: id")] + MissingID, + + #[error("invalid file id: {}", msg)] + InvalidID { msg: String }, + + #[error("invalid front-matter: {}", msg)] + InvalidFrontMatter { msg: String }, + + #[error("fs error: {}", msg)] + FSError { msg: String }, +} diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs new file mode 100644 index 0000000..518d162 --- /dev/null +++ b/backend/markdown/src/lib.rs @@ -0,0 +1,68 @@ +mod error; +mod utils; + +use std::fs; +use std::path::Path; +use std::str::FromStr; + +use crate::error::Error; +use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + +use base64::{engine::general_purpose, Engine}; +use serde::Serialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +#[derive(Default, Debug, Serialize)] +pub struct MdFile { + pub file_id: Uuid, + pub filename: String, + pub meta: Value, + pub content: String, + pub hash: String, +} + +impl MdFile { + pub fn from_file(file_path: &Path) -> Result { + let raw_file = match fs::read_to_string(file_path) { + Ok(file) => file, + Err(e) => return Err(Error::FSError { msg: e.to_string() }), + }; + let raw_file_bytes = raw_file.as_bytes().to_vec(); + let (front_matter_str, content) = split_md_front_matter(raw_file); + let front_matter = match parse_front_matter(&front_matter_str) { + Ok(front_matter) => front_matter, + Err(e) => return Err(Error::InvalidFrontMatter { msg: e.to_string() }), + }; + let hash = calculate_hash_base64(&raw_file_bytes); + let file_id = match front_matter["id"].as_str() { + Some(id) => match Uuid::from_str(id) { + Ok(id) => id, + Err(e) => return Err(Error::InvalidID { msg: e.to_string() }), + }, + None => return Err(Error::MissingID), + }; + let file_name = Path::new(file_path) + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + + Ok(MdFile { + file_id, + filename: file_name, + meta: front_matter, + content, + hash, + }) + } +} + +fn calculate_hash_base64(bytes: &Vec) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let hash = hasher.finalize().to_vec(); + general_purpose::STANDARD.encode(hash) +} diff --git a/backend/markdown/src/utils/front_matter.rs b/backend/markdown/src/utils/front_matter.rs new file mode 100644 index 0000000..5bf77d2 --- /dev/null +++ b/backend/markdown/src/utils/front_matter.rs @@ -0,0 +1,81 @@ +use crate::utils::yaml::yaml_to_serde; +use regex::Regex; +use serde_json::Value; +use std::error::Error; +use yaml_rust2::YamlLoader; + +pub fn split_md_front_matter(raw_file: String) -> (String, String) { + let re = Regex::new(r"(?s)^(---\n.*?\n---\n)(.*)").unwrap(); + if let Some(captures) = re.captures(raw_file.as_str()) { + let front_matter = captures.get(1).map_or("", |m| m.as_str()).to_string(); + let main_content = captures.get(2).map_or("", |m| m.as_str()).to_string(); + (front_matter, main_content) + } else { + (String::new(), raw_file) + } +} + +pub fn parse_front_matter(front_matter: &str) -> Result> { + let parsed_f_matter = YamlLoader::load_from_str(front_matter)?; + if parsed_f_matter.is_empty() { + Ok(Value::String(String::new())) + } else { + Ok(yaml_to_serde(&parsed_f_matter[0])) + } +} + +#[cfg(test)] +mod tests { + use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; + + #[test] + fn front_matter_to_string() { + let (front_matter, main_content) = split_md_front_matter( + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- + +test body + +"## + .to_string(), + ); + assert_eq!( + front_matter, + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- +"## + ); + assert_eq!(main_content, "\ntest body\n\n"); + } + + #[test] + fn test_front_matter_parser() { + dbg!(parse_front_matter( + r##"--- +title: test_title +date: 0000-00-00+0000 +category: test +id: bf03afd8-61b4-4e78-a34a-8b3ee39adf16 +sub_title: sub_title +summary: summary +tags: [ "test1", "test2" ] +--- +"##, + )) + .expect("Failed to parse front matter"); + } +} diff --git a/backend/markdown/src/utils/mod.rs b/backend/markdown/src/utils/mod.rs new file mode 100644 index 0000000..27be997 --- /dev/null +++ b/backend/markdown/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod front_matter; +mod yaml; diff --git a/backend/markdown/src/utils/yaml.rs b/backend/markdown/src/utils/yaml.rs new file mode 100644 index 0000000..8c52f4a --- /dev/null +++ b/backend/markdown/src/utils/yaml.rs @@ -0,0 +1,29 @@ +use serde_json::Value; +use std::str::FromStr; +use yaml_rust2::Yaml; + +pub fn yaml_to_serde(yaml: &Yaml) -> Value { + match yaml { + Yaml::Null => Value::Null, + Yaml::Boolean(b) => Value::Bool(*b), + Yaml::Integer(i) => Value::Number(serde_json::Number::from(*i)), + Yaml::Real(s) => Value::Number(serde_json::Number::from_str(s).unwrap()), + Yaml::String(s) => Value::String(s.clone()), + Yaml::Array(arr) => Value::Array(arr.iter().map(yaml_to_serde).collect()), + Yaml::Hash(hash) => { + let obj: serde_json::map::Map = hash + .iter() + .map(|(k, v)| { + let key = match k { + Yaml::String(s) => s.clone(), + _ => "INVALID_KEY".to_string(), + }; + (key, yaml_to_serde(v)) + }) + .collect(); + Value::Object(obj) + } + Yaml::Alias(_) => Value::Null, + Yaml::BadValue => Value::Null, + } +} diff --git a/backend/src/api/mod.rs b/backend/src/api/mod.rs deleted file mode 100644 index 69e599d..0000000 --- a/backend/src/api/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod get; -pub mod post; diff --git a/backend/src/api/post/errors/mod.rs b/backend/src/api/post/errors/mod.rs deleted file mode 100644 index 1d2e0f5..0000000 --- a/backend/src/api/post/errors/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod refresh; diff --git a/backend/src/api/post/errors/refresh.rs b/backend/src/api/post/errors/refresh.rs deleted file mode 100644 index 1c4224a..0000000 --- a/backend/src/api/post/errors/refresh.rs +++ /dev/null @@ -1,61 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum PostIdxError { - #[error("Failed to parse UUID <{}>: {}", id, err_msg)] - InvalidUUID { id: String, err_msg: String }, - - #[error( - "Failed to parse `last update time` for <{}> when creating post index: {}", - id, - err_msg - )] - InvalidLastUpdate { id: String, err_msg: String }, - - #[error( - "Failed to parse `first update time` for <{}> when creating post index: {}", - id, - err_msg - )] - InvalidFirstUpdate { id: String, err_msg: String }, - - #[error( - "Failed to parse `tag` for <{}> when creating post index: <{}>: {}", - id, - invalid_string, - err_msg - )] - InvalidTagString { - id: String, - invalid_string: String, - err_msg: String, - }, - - #[error( - "Database error on writing `{}` to table `{}`: <{}>: {}", - data_desc, - db_table, - id, - err_msg - )] - DBWriteFailure { - data_desc: String, - db_table: String, - id: String, - err_msg: String, - }, - - #[error( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - data_desc, - db_table, - id, - err_msg - )] - DBReadFailure { - data_desc: String, - db_table: String, - id: String, - err_msg: String, - }, -} diff --git a/backend/src/api/post/mod.rs b/backend/src/api/post/mod.rs deleted file mode 100644 index 1bba1ec..0000000 --- a/backend/src/api/post/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod errors; -pub mod refresh; -mod utils; diff --git a/backend/src/api/post/refresh.rs b/backend/src/api/post/refresh.rs deleted file mode 100644 index 65d7be3..0000000 --- a/backend/src/api/post/refresh.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::path::Path; -use std::str::FromStr; -use std::{env, fs}; - -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::Json; -use futures::{stream, StreamExt}; -use serde::Deserialize; -use serde_json::{json, Map, Value}; -use sha2::{Digest, Sha256}; -use sqlx::{query, PgPool}; -use tracing::error; -use uuid::Uuid; - -use crate::api::post::errors::refresh::PostIdxError; -use crate::api::post::utils::{build_post_idx, post}; -use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL}; - -#[derive(Deserialize)] -pub struct Params { - /// refresh level - level: Option, -} - -pub async fn refresh( - State(db_connection): State, - Query(params): Query, -) -> Result)> { - let is_force_refresh = params.level.unwrap_or("".to_string()) == "force"; - - // search post - let data_repo_dir = match env::var("GIT_WORK_DIR") { - Ok(value) => value, - Err(_) => { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(( - "error", - "error at parsing environment variable: GIT_WORK_DIR" - ))), - )); - } - }; - let post_dir = Path::new(&data_repo_dir).join("posts"); - - if is_force_refresh { - if let Err(e) = query("DROP TABLE IF EXISTS hash, meta, post, tag;") - .execute(&db_connection) - .await - { - error!( - "DB query error when deleting table: `hash`, `meta`, `post`, `tag`: {}", - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(("error", e.to_string()))), - )); - } - } - - // generate post idx - let post_path = search_md(&post_dir); - - let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = post_path - .into_iter() - .filter_map(|post_path| match post::from_path(&post_path) { - Ok(post) => match fs::read(&post_path) { - Ok(bytes) => { - let mut hasher = Sha256::new(); - hasher.update(bytes); - let hash = hasher.finalize().to_vec(); - Some((post, hash)) - } - Err(e) => { - error!( - "Failed to read file while hashing the post <{}>, skipping: {}", - post_path, - e.to_string() - ); - None - } - }, - Err(e) => { - error!( - "Failed to parse post <{}>, skipping: {}", - post_path, - e.to_string() - ); - None - } - }) - .collect(); - - // db create - query(POST_BASE_MODEL) - .execute(&db_connection) - .await - .unwrap(); - query(POST_META_MODEL) - .execute(&db_connection) - .await - .unwrap(); - query(TAG_MODEL).execute(&db_connection).await.unwrap(); - query(POST_HASH_MODEL) - .execute(&db_connection) - .await - .unwrap(); - - let res: Vec<_> = stream::iter(post_list_with_hash) - .map(|(post, hash)| { - let db_clone = db_connection.clone(); - async move { build_post_idx(&db_clone, post, hash).await } - }) - .buffer_unordered(usize::MAX) - .collect() - .await; - - let errs: Vec<_> = res - .into_iter() - .filter_map(|res| res.err()) - .map(|e| e.to_string()) - .collect(); - - if !errs.is_empty() { - Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": errs})), - )) - } else { - Ok(StatusCode::OK) - } -} - -/// Search a dir, return a list of markdown files -/// -/// # Arguments -/// * `entry`: dir path -/// -/// # Returns -/// Vec -/// * `String`: path of a markdown file -/// -fn search_md(entry: &Path) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path.as_path()) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry.to_str().unwrap().to_string()] - } else { - vec![] - } -} - -async fn build_post_idx( - db_connection: &PgPool, - post: Map, - hash: Vec, -) -> Result<(), PostIdxError> { - let meta = &post["meta"]; - - // extract post id - let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { - Ok(id) => id, - Err(e) => { - return Err(PostIdxError::InvalidUUID { - id: post["post_id"].as_str().unwrap().to_string(), - err_msg: e.to_string(), - }) - } - }; - - if build_post_idx::hash(db_connection, post_id, hash).await? { - build_post_idx::base(db_connection, &post, post_id, meta).await?; - build_post_idx::tag(db_connection, post_id, meta).await?; - build_post_idx::meta(db_connection, post_id, meta).await?; - } - - Ok(()) -} diff --git a/backend/src/api/post/utils/build_post_idx.rs b/backend/src/api/post/utils/build_post_idx.rs deleted file mode 100644 index 47b2692..0000000 --- a/backend/src/api/post/utils/build_post_idx.rs +++ /dev/null @@ -1,288 +0,0 @@ -use base64::{Engine, engine::general_purpose}; -use chrono::DateTime; -use serde_json::{Map, Value}; -use sqlx::{PgPool, query, Row}; -use uuid::Uuid; - -use crate::api::post::errors::refresh::PostIdxError; -use crate::constants::GLOBAL_TIME_FORMAT; - -pub async fn base( - db_connection: &PgPool, - post: &Map, - post_id: Uuid, - meta: &Value, -) -> Result<(), PostIdxError> { - let title = post["title"].as_str(); - let summary = post - .get("summary") - .and_then(|summary| summary.as_str()) - .unwrap_or(""); - let content = post["content"].as_str(); - let last_update = - match DateTime::parse_from_str(post["last_update"].as_str().unwrap(), GLOBAL_TIME_FORMAT) { - Ok(time) => time, - Err(e) => { - return Err(PostIdxError::InvalidLastUpdate { - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - }; - let first_update = match DateTime::parse_from_str( - post["first_update"].as_str().unwrap(), - GLOBAL_TIME_FORMAT, - ) { - Ok(time) => time, - Err(e) => { - return Err(PostIdxError::InvalidFirstUpdate { - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - }; - let sub_title = meta - .get("sub_title") - .and_then(|sub_title| sub_title.as_str()) - .unwrap_or(""); - let category = meta - .get("category") - .and_then(|category| category.as_str()) - .unwrap_or("N/A"); - let header_img = meta - .get("header_img") - .and_then(|header_img| header_img.as_str()) - .unwrap_or(""); - - match query( - r##" -INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) -ON CONFLICT (post_id) DO -UPDATE SET - title = EXCLUDED.title, - summary = EXCLUDED.summary, - content = EXCLUDED.content, - last_update = EXCLUDED.last_update, - first_update = EXCLUDED.first_update, - sub_title = EXCLUDED.sub_title, - category = EXCLUDED.category, - header_img = EXCLUDED.header_img; - "##, - ) - .bind(post_id) - .bind(title) - .bind(summary) - .bind(content) - .bind(last_update) - .bind(first_update) - .bind(sub_title) - .bind(category) - .bind(header_img) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err( - PostIdxError::DBWriteFailure { - data_desc: "post base information".to_string(), - db_table: "Post".to_string(), - id: post_id.to_string(), - err_msg: e.to_string() - } - ) - } - } - - Ok(()) -} - -pub async fn tag(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> { - if let Some(tags_string) = meta.get("tags").to_owned() { - // parse tags string - let parsed: Value = match serde_json::from_str(tags_string.as_str().unwrap()) { - Ok(value) => value, - Err(e) => { - return Err(PostIdxError::InvalidTagString { - id: post_id.to_string(), - invalid_string: tags_string.to_string().to_string(), - err_msg: e.to_string(), - }) - } - }; - let string_array = parsed.as_array().unwrap(); - let tags: Vec = string_array - .iter() - .map(|value| value.as_str().unwrap().to_string()) - .collect(); - - // remove old record - if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") - .bind(post_id) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - - // add new record - for tag in tags { - match query("INSERT INTO Tag VALUES ($1, $2);") - .bind(post_id) - .bind(tag) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - } - } - } - - Ok(()) -} - -/// # Arguments -/// -/// * `db_connection`: database connection -/// * `post_id`: UUID of the post -/// * `hash`: Hex String -/// -/// # Returns -/// Result -/// - `bool`: if the hash is different from database, return true, else false -pub async fn hash( - db_connection: &PgPool, - post_id: Uuid, - hash: Vec, -) -> Result { - let base64_form = general_purpose::STANDARD.encode(hash); - match query("SELECT hash FROM Hash WHERE post_id = $1") - .bind(post_id) - .fetch_optional(db_connection) - .await - { - Ok(row) => { - let db_record: String = match row { - None => "".to_string(), - Some(row) => row.try_get("hash").unwrap_or("".to_string()), - }; - if base64_form == db_record { - Ok(false) - } else { - if let Err(e) = query( - r#" -INSERT INTO Hash (post_id, hash) VALUES ($1, $2) - ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, - ) - .bind(post_id) - .bind(&base64_form) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }); - } - Ok(true) - } - } - Err(e) => Err(PostIdxError::DBReadFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }), - } -} - -pub async fn meta(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> { - let mut err_trails = 5; - loop { - if let Err(e) = query( - format!( - r#" -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY['tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY(_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - "#, post_id.to_string().as_str(), meta.to_string() - ).as_str() - ) - .execute(db_connection) - .await { - if err_trails >= 0 { - err_trails -= 1; - continue - } else { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post meta data".to_string(), - db_table: "Meta".to_string(), - id: post_id.to_string(), - err_msg: e.to_string(), - }) - } - } else { - return Ok(()) - } - } -} diff --git a/backend/src/api/post/utils/mod.rs b/backend/src/api/post/utils/mod.rs deleted file mode 100644 index faee48e..0000000 --- a/backend/src/api/post/utils/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod build_post_idx; -pub mod post; diff --git a/backend/src/api/post/utils/post.rs b/backend/src/api/post/utils/post.rs deleted file mode 100644 index 901d799..0000000 --- a/backend/src/api/post/utils/post.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::{env, fs}; -use std::error::Error; -use std::process::Command; -use std::str; -use std::str::FromStr; - -use chrono::{DateTime, Utc}; -use serde_json::{Map, Value}; -use tracing::{error, warn}; -use uuid::Uuid; - -use crate::constants::GLOBAL_TIME_FORMAT; -use crate::markdown::meta::parse_meta; - -pub fn from_path(path: &str) -> Result, Box> { - let mut res = Map::new(); - - // field `content` - let md_content = fs::read_to_string(path)?; - res.insert("content".to_string(), Value::from(md_content.to_owned())); - - // field `meta` - let meta = parse_meta(&md_content); - let meta_map: Map = meta - .clone() - .into_iter() - .map(|(k, v)| (k, Value::from(v))) - .collect(); - res.insert("meta".to_string(), Value::from(meta_map)); - - // field `post_id` - let post_id = match meta.get("id") { - None => Uuid::new_v4(), - Some(id) => match Uuid::from_str(id.as_str()) { - Ok(id) => id, - Err(e) => { - error!("Failed to parse UUID <{}>: {}", id.as_str(), e.to_string()); - return Err(Box::new(e)); - } - }, - }; - res.insert("post_id".to_string(), Value::from(post_id.to_string())); - - // field `title` - let title = match meta.get("title") { - None => "Title Not Found".to_string(), - Some(title) => title.to_owned(), - }; - res.insert("title".to_string(), Value::from(title)); - - // field `summary` - let summary = match meta.get("summary") { - None => "".to_string(), - Some(summary) => summary.to_owned(), - }; - res.insert("summary".to_string(), Value::from(summary)); - - // field `last_update` - let git_output = match Command::new("git") - .arg("--no-pager") - .arg("log") - .arg("-1") - .arg("--format=%cI") - .arg("--") - .current_dir(env::var("GIT_WORK_DIR")?) - .arg(path) - .output() - { - Ok(res) => res, - Err(e) => { - error!("Failed to execute `git` command: {}", e.to_string()); - return Err(Box::new(e)); - } - }; - let last_update: DateTime = if git_output.status.success() { - chrono::DateTime::parse_from_str( - str::from_utf8(&git_output.stdout).unwrap().trim_end(), - GLOBAL_TIME_FORMAT, - ) - .unwrap_or_else(|_| DateTime::from(Utc::now())) - .to_utc() - } else { - Utc::now().to_utc() - }; - res.insert( - "last_update".to_string(), - Value::from(last_update.format(GLOBAL_TIME_FORMAT).to_string()), - ); - - // field `first_update` - let first_update: DateTime = match meta.get("date") { - None => { - warn!( - "Failed to parse first update time for post {}, using current time instead", - post_id.to_string() - ); - Utc::now().to_utc() - } - Some(date) => DateTime::from( - match DateTime::parse_from_str(format!("{}T000000", date).as_str(), "%Y-%m-%d%zT%H%M%S") - { - Ok(time) => time, - Err(e) => { - warn!( - "Failed to parse first update time for post {}, using current time instead: {}", - post_id.to_string(), - e.to_string() - ); - DateTime::from(Utc::now()) - } - }, - ), - }; - res.insert( - "first_update".to_string(), - Value::from(first_update.format(GLOBAL_TIME_FORMAT).to_string()), - ); - - Ok(res) -} diff --git a/backend/src/constants.rs b/backend/src/constants.rs deleted file mode 100644 index 5dd57e4..0000000 --- a/backend/src/constants.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub const GLOBAL_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; - -pub const PAGE_SIZE: u32 = 6; diff --git a/backend/src/logger.rs b/backend/src/logger.rs deleted file mode 100644 index 974a706..0000000 --- a/backend/src/logger.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::{env, io}; - -use chrono::Local; -use tracing::info; -use tracing_appender::{non_blocking, rolling}; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_subscriber::{fmt, Registry}; -use tracing_subscriber::fmt::format::Writer; -use tracing_subscriber::fmt::MakeWriter; -use tracing_subscriber::fmt::time::FormatTime; -use tracing_subscriber::layer::SubscriberExt; - -use crate::constants::GLOBAL_TIME_FORMAT; - -struct LocalTimer; - -impl FormatTime for LocalTimer { - fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { - write!(w, "{}", Local::now().format(GLOBAL_TIME_FORMAT)) - } -} - -type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; - -pub fn init() -> (WorkerGuard, WorkerGuard) { - let log_path = env::var("LOG_ROOT_DIR").unwrap_or_else(|e| { - panic!("[Fatal] failed to load LOG_ROOT_DIR: {}", e); - }); - - let log_path = format!("{}/backend", log_path); - - let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); - - let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); - - let log_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_timer(LocalTimer); - - let subscribers = Registry::default() - .with( - fmt::Layer::default() - .with_writer(info_log_writer) - .with_ansi(false), - ) - .with( - fmt::Layer::default() - .with_writer(err_log_writer) - .with_ansi(false), - ) - .with( - fmt::Layer::default() - .with_writer(io::stdout) - .with_ansi(true) - .event_format(log_format), - ); - - tracing::subscriber::set_global_default(subscribers) - .expect("[Fatal] Failed to setup logging system"); - - info!("Module Started: Logger"); - - // error!("test"); - (info_guard, error_guard) -} diff --git a/backend/src/main.rs b/backend/src/main.rs deleted file mode 100644 index eaf806d..0000000 --- a/backend/src/main.rs +++ /dev/null @@ -1,102 +0,0 @@ -use std::env; -use std::error::Error; -use std::num::ParseIntError; -use std::process::exit; -use std::str::FromStr; - -use crate::api::get::get_page_count::get_page_count; -use crate::api::get::get_post::get_post; -use crate::api::get::get_post_list::get_post_list; -use crate::api::post::refresh::refresh; -use axum::routing::{get, post}; -use axum::Router; -use dotenv::dotenv; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; -use sqlx::ConnectOptions; -use tracing::{error, info, warn}; - -mod api; -mod constants; -mod logger; -mod markdown; -mod model; - -#[tokio::main] -async fn main() -> Result<(), Box> { - if let Some(env_file) = env::args().nth(1) { - match dotenv::from_filename(&env_file) { - Ok(_) => { - println!("[Info] env loaded from {}", env_file); - } - Err(e) => { - panic!("[Fatal] failed to read {}: {}", env_file, e); - } - } - }; - match dotenv() { - Ok(_) => { - println!("[Info] env loaded from .env file, starting up...") - } - Err(_) => { - println!("[Info] .env not detected, starting up...") - } - }; - - let _guards = logger::init(); - - let port: u32 = match env::var("BACKEND_PORT") { - Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { - warn!("Failed to parse BACKEND_PORT: {}", e.to_string()); - 3001 - }), - Err(_) => 3001, - }; - - let db_connect_option = PgConnectOptions::from_str(dbg!(&parse_db_uri())) - .unwrap() - .disable_statement_logging(); - - let db_pool = match PgPoolOptions::new().connect_with(db_connect_option).await { - Ok(pool) => { - info!("Database connected"); - pool - } - Err(e) => { - error!("Failed to load database: {}", e.to_string()); - exit(1); - } - }; - - let app = Router::new() - .route("/refresh_posts", post(refresh)) - .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count)) - .route("/get_post/:post_id", get(get_post)) - .with_state(db_pool); - - let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)) - .await - .unwrap_or_else(|e| { - error!("Failed to build TCP listener: {}", e.to_string()); - exit(1); - }); - - axum::serve(listener, app).await.unwrap_or_else(|e| { - error!("Failed to start axum: {}", e.to_string()); - exit(1); - }); - - Ok(()) -} - -fn parse_db_uri() -> String { - let postgres_user = env::var("POSTGRES_USER").unwrap_or("".to_string()); - let postgres_password = env::var("POSTGRES_PASSWORD").unwrap_or("".to_string()); - let db_host = env::var("DB_HOST").unwrap_or("localhost".to_string()); - let db_port = env::var("DB_PORT") - .unwrap_or("5432".to_string()) - .to_string(); - let postgres_db = env::var("POSTGRES_DB").unwrap_or("nocturne".to_string()); - - format!("postgresql://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") -} diff --git a/backend/src/markdown/meta.rs b/backend/src/markdown/meta.rs deleted file mode 100644 index b2d7cac..0000000 --- a/backend/src/markdown/meta.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::collections::HashMap; - -use regex::Regex; - -/// return the meta info contained in the .md file front matter -pub fn parse_meta(md_content: &str) -> HashMap { - let mut fields: HashMap = HashMap::new(); - let match_regex = Regex::new(r"(?s)---\n(.*\n)---\n").unwrap(); - if let Some(front_matter) = match_regex.captures(md_content) { - let fm_content = front_matter.get(1).unwrap().as_str(); - for line in fm_content.lines() { - if let Some((k, v)) = extract_kv(line) { - fields.insert(k, v); - } - } - } - fields -} - -fn extract_kv(line: &str) -> Option<(String, String)> { - let split: Vec<&str> = line.splitn(2, ':').map(|s| s.trim()).collect(); - if split.len() == 2 { - let key = split[0].to_string(); - let value = split[1].to_string(); - Some((key, value)) - } else { - None - } -} - -#[cfg(test)] -mod meta_test { - use std::error::Error; - - use crate::markdown::meta::parse_meta; - - #[test] - fn test_parse_meta_front_matter() -> Result<(), Box> { - let parsed_meta = parse_meta( - r##" ---- -title: Hello World! -tags: [ "test", "hello_world" ] ---- - "##, - ); - dbg!(&parsed_meta); - assert_eq!(parsed_meta.get("title"), Some(&"Hello World!".to_string())); - assert_eq!( - parsed_meta.get("tags"), - Some(&r#"[ "test", "hello_world" ]"#.to_string()) - ); - Ok(()) - } -} diff --git a/backend/src/markdown/mod.rs b/backend/src/markdown/mod.rs deleted file mode 100644 index 6b50816..0000000 --- a/backend/src/markdown/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod meta; diff --git a/backend/src/model/blog/hash.sql b/backend/src/model/blog/hash.sql deleted file mode 100644 index 7dc91de..0000000 --- a/backend/src/model/blog/hash.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); \ No newline at end of file diff --git a/backend/src/model/blog/meta.sql b/backend/src/model/blog/meta.sql deleted file mode 100644 index 6cf903a..0000000 --- a/backend/src/model/blog/meta.sql +++ /dev/null @@ -1,79 +0,0 @@ -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); - -DROP table meta; - -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY ['id', 'tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY (_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - -SELECT jsonb_strip_nulls(to_jsonb(t)) AS meta -FROM meta t -WHERE post_id = '8ca3fa2b-f040-448c-8885-81aab56db9bb'; - -WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = '8ca3fa2b-f040-448c-8885-81aab56db9bb'), - prev AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date < (SELECT date FROM target_post) - ORDER BY t.date DESC - LIMIT 1), - next AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date > (SELECT date FROM target_post) - ORDER BY t.date - LIMIT 1) -SELECT json_strip_nulls(json_build_object( - 'prev', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM prev), - 'next', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM next) - )) AS result; - diff --git a/backend/src/model/blog/mod.rs b/backend/src/model/blog/mod.rs deleted file mode 100644 index c282af7..0000000 --- a/backend/src/model/blog/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -pub const POST_BASE_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - header_img TEXT, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, - first_update TIMESTAMPTZ NOT NULL -); -"##; - -// base mode, will add more cols in runtime -pub const POST_META_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); -"##; - -pub const POST_HASH_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); -"##; - -pub const TAG_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); -"##; diff --git a/backend/src/model/blog/post.sql b/backend/src/model/blog/post.sql deleted file mode 100644 index 1afc493..0000000 --- a/backend/src/model/blog/post.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - header_img TEXT, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, - first_update TIMESTAMPTZ NOT NULL -); diff --git a/backend/src/model/blog/tag.sql b/backend/src/model/blog/tag.sql deleted file mode 100644 index 6c84b76..0000000 --- a/backend/src/model/blog/tag.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); diff --git a/backend/src/model/mod.rs b/backend/src/model/mod.rs deleted file mode 100644 index 21aa2c6..0000000 --- a/backend/src/model/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod blog; diff --git a/frontend/app/blog/BlogPostCard.tsx b/frontend/app/blog/BlogPostCard.tsx index 3829fd2..0ca61a8 100644 --- a/frontend/app/blog/BlogPostCard.tsx +++ b/frontend/app/blog/BlogPostCard.tsx @@ -18,7 +18,7 @@ function BlogPostCard({ post }: Props) { // let last_update = new Date((post as any)['last_update']); let first_update = new Date((post as any)['first_update']); let post_id = (post as any)['post_id']; - let header_img = (post as any)['header_img']; + let header_img = ((post as any)['header_img'] as string | null); return ( @@ -29,7 +29,7 @@ function BlogPostCard({ post }: Props) { isHoverable > - { header_img !== '' &&
+ { header_img &&
{
-
+
{ category }

{ title }

diff --git a/frontend/app/blog/PostList.tsx b/frontend/app/blog/PostList.tsx index 4eff3a8..591c6ea 100644 --- a/frontend/app/blog/PostList.tsx +++ b/frontend/app/blog/PostList.tsx @@ -8,8 +8,8 @@ import { Card, CardBody } from '@nextui-org/card'; const PostList = async ({ page }: { page: number }) => { try { - const res = await fetchNocturneApi(`/get_post_list?page=${ page }`); - let posts = (await res.json() as any)['posts']; + const res = await fetchNocturneApi(`/blog/get_post_list?page=${ page }`); + const posts = await res.json(); return ( // blog list
diff --git a/frontend/app/blog/page.tsx b/frontend/app/blog/page.tsx index fb6d6cc..6690937 100644 --- a/frontend/app/blog/page.tsx +++ b/frontend/app/blog/page.tsx @@ -2,15 +2,15 @@ import PostList from '@/app/blog/PostList'; import Pagination from '@/app/blog/Pagination'; import { fetchNocturneApi } from '@/app/blog/api'; -async function Blog({ searchParams }: { - searchParams?: { - page?: number - }; +async function Blog({ + searchParams +}: { + searchParams: Promise<{ [key: string]: string | string[] | undefined }> }) { - const parsePage = Number(searchParams?.page); + const parsePage = Number((await searchParams)?.page); const currPage = parsePage > 0 ? parsePage : 1; - const ret = await fetchNocturneApi(`/get_page_count`); + const ret = await fetchNocturneApi(`/blog/get_page_count`); const pageCount = (await ret.json() as any)['res'] as number; return ( diff --git a/frontend/app/blog/post/[id]/Headers.tsx b/frontend/app/blog/post/[id]/Headers.tsx index cb44322..62e9090 100644 --- a/frontend/app/blog/post/[id]/Headers.tsx +++ b/frontend/app/blog/post/[id]/Headers.tsx @@ -18,11 +18,11 @@ const Headers = (props: Props) => { // let last_update = new Date((post as any)['last_update']); let first_update = new Date(post['first_update']); // let post_id = (post as any)['post_id']; - let header_img = post['header_img']; + let header_img = post['meta']['header_img'] as string | null; return ( <> - { header_img !== '' &&
+ { header_img &&
{ {
}
-
+
{ category }

{ title }

diff --git a/frontend/app/blog/post/[id]/PostCard.tsx b/frontend/app/blog/post/[id]/PostCard.tsx index 056f3d1..958979f 100644 --- a/frontend/app/blog/post/[id]/PostCard.tsx +++ b/frontend/app/blog/post/[id]/PostCard.tsx @@ -13,7 +13,7 @@ async function PostCard(props: Props) { try { - const res = await fetchNocturneApi(`/get_post/${ props.id }`); + const res = await fetchNocturneApi(`/blog/get_post/${ props.id }`); const ret: any = await res.json(); return (
diff --git a/frontend/app/blog/post/[id]/page.tsx b/frontend/app/blog/post/[id]/page.tsx index 13ea3b2..c29364d 100644 --- a/frontend/app/blog/post/[id]/page.tsx +++ b/frontend/app/blog/post/[id]/page.tsx @@ -3,7 +3,11 @@ import PostCard from '@/app/blog/post/[id]/PostCard'; import { MotionDiv } from '@/app/public/MotionDiv'; import ButtonGoBack from '@/app/blog/post/[id]/ButtonGoBack'; -function BlogReader({ params }: { params: { id: string } }) { +async function BlogReader({ + params +}: { + params: Promise<{ id: string }> +}) { return (
- +
diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index 3e71f6c..1c4b3c9 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -8,7 +8,7 @@ import Footer from '@/components/Footer'; export const metadata: Metadata = { title: 'Lumitherの酒馆', - description: '一间位于时间夹缝中的旅馆，静候下一位旅者的到来...' + description: '一间位于时间夹缝中的酒馆，静候下一位旅者的到来...' }; export default function RootLayout({ diff --git a/frontend/components/ThemeSwitcher.tsx b/frontend/components/ThemeSwitcher.tsx index 07f6f15..0dbc95f 100644 --- a/frontend/components/ThemeSwitcher.tsx +++ b/frontend/components/ThemeSwitcher.tsx @@ -49,9 +49,9 @@ function ThemeSwitcher() { fullWidth isIconOnly={ isMobile } > - { switcherElements.get(currTheme.values().next().value)?.icon } + { switcherElements.get(currTheme.values().next().value as string)?.icon }

- { switcherElements.get(currTheme.values().next().value)?.display_name } + { switcherElements.get(currTheme.values().next().value as string)?.display_name }

diff --git a/frontend/package.json b/frontend/package.json index ed666ec..92be224 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,7 +13,7 @@ "@types/react-syntax-highlighter": "^15.5.13", "framer-motion": "^11.3.24", "lxgw-wenkai-webfont": "^1.7.0", - "next": "14.2.3", + "next": "^15.0.3", "next-themes": "^0.3.0", "react": "^18.3.1", "react-dom": "^18.3.1",