From acd05255139d852f57e6aa6a4ac7c75dc110ecc4 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 17 Feb 2025 15:27:41 -0800 Subject: [PATCH] feat(backend) rebuild start --- backend/Cargo.toml | 3 +- backend/api_server/Cargo.toml | 25 -- .../api_server/src/blog/api/get_page_count.rs | 26 -- backend/api_server/src/blog/api/get_post.rs | 153 -------- .../api_server/src/blog/api/get_post_list.rs | 130 ------- backend/api_server/src/blog/api/mod.rs | 3 - .../src/blog/cron/check_update/error.rs | 49 --- .../src/blog/cron/check_update/index.rs | 351 ------------------ .../src/blog/cron/check_update/mod.rs | 146 -------- .../src/blog/cron/check_update/utils.rs | 86 ----- backend/api_server/src/blog/cron/mod.rs | 15 - backend/api_server/src/blog/mod.rs | 17 - backend/api_server/src/init/cron.rs | 12 - backend/api_server/src/init/database.rs | 41 -- backend/api_server/src/init/logger.rs | 106 ------ backend/api_server/src/init/misc.rs | 14 - backend/api_server/src/init/mod.rs | 58 --- backend/api_server/src/init/schema.rs | 49 --- backend/api_server/src/init/server.rs | 52 --- backend/api_server/src/main.rs | 13 - backend/api_server/src/scheduler/mod.rs | 194 ---------- .../src/scheduler/scheduler_error.rs | 44 --- backend/api_server/src/scheduler/sleeper.rs | 77 ---- backend/api_server/src/scheduler/task_func.rs | 3 - .../api_server/src/scheduler/tasks/basic.rs | 37 -- backend/api_server/src/scheduler/tasks/mod.rs | 8 - backend/api_server/src/utils/git.rs | 195 ---------- backend/api_server/src/utils/mod.rs | 1 - backend/constants/Cargo.toml | 6 - backend/macros/src/log.rs | 21 +- backend/server/schema.sql | 32 ++ .../src => server/src/constants}/blog.rs | 0 .../src => server/src/constants}/config.rs | 0 .../lib.rs => server/src/constants/mod.rs} | 0 .../src => server/src/constants}/time.rs | 0 backend/server/src/main.rs | 7 + loadbalancer/.gitignore | 3 + openresty/conf/nocturne.conf => nocturne.conf | 19 +- openresty/deploy.sh | 4 - openresty/lua/hello_world.lua | 1 - scripts/dev_server.sh | 14 - scripts/flush_env.sh | 6 - scripts/refresh.sh | 42 --- scripts/setup.sh | 15 - scripts/start_server.sh | 20 - 45 files changed, 69 insertions(+), 2029 deletions(-) delete mode 100644 backend/api_server/Cargo.toml delete mode 100644 backend/api_server/src/blog/api/get_page_count.rs delete mode 100644 backend/api_server/src/blog/api/get_post.rs delete mode 100644 backend/api_server/src/blog/api/get_post_list.rs delete mode 100644 backend/api_server/src/blog/api/mod.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/error.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/index.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/mod.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/utils.rs delete mode 100644 backend/api_server/src/blog/cron/mod.rs delete mode 100644 backend/api_server/src/blog/mod.rs delete mode 100644 backend/api_server/src/init/cron.rs delete mode 100644 backend/api_server/src/init/database.rs delete mode 100644 backend/api_server/src/init/logger.rs delete mode 100644 backend/api_server/src/init/misc.rs delete mode 100644 backend/api_server/src/init/mod.rs delete mode 100644 backend/api_server/src/init/schema.rs delete mode 100644 backend/api_server/src/init/server.rs delete mode 100644 backend/api_server/src/main.rs delete mode 100644 backend/api_server/src/scheduler/mod.rs delete mode 100644 backend/api_server/src/scheduler/scheduler_error.rs delete mode 100644 backend/api_server/src/scheduler/sleeper.rs delete mode 100644 backend/api_server/src/scheduler/task_func.rs delete mode 100644 backend/api_server/src/scheduler/tasks/basic.rs delete mode 100644 backend/api_server/src/scheduler/tasks/mod.rs delete mode 100644 backend/api_server/src/utils/git.rs delete mode 100644 backend/api_server/src/utils/mod.rs delete mode 100644 backend/constants/Cargo.toml create mode 100644 backend/server/schema.sql rename backend/{constants/src => server/src/constants}/blog.rs (100%) rename backend/{constants/src => server/src/constants}/config.rs (100%) rename backend/{constants/src/lib.rs => server/src/constants/mod.rs} (100%) rename backend/{constants/src => server/src/constants}/time.rs (100%) create mode 100644 backend/server/src/main.rs create mode 100644 loadbalancer/.gitignore rename openresty/conf/nocturne.conf => nocturne.conf (65%) delete mode 100755 openresty/deploy.sh delete mode 100644 openresty/lua/hello_world.lua delete mode 100755 scripts/dev_server.sh delete mode 100755 scripts/flush_env.sh delete mode 100755 scripts/refresh.sh delete mode 100755 scripts/setup.sh delete mode 100755 scripts/start_server.sh diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6c75b0a..154acf9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,8 +1,7 @@ [workspace] resolver = "2" members = [ - "api_server", - "constants", + "server", "macros", "markdown", ] diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml deleted file mode 100644 index af17326..0000000 --- a/backend/api_server/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "api_server" -version = "0.1.0" -edition = "2021" - -[dependencies] -macros = { path = "../macros" } -constants = { path = "../constants" } -markdown = { path = "../markdown" } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } -tracing-appender = "0.2.3" -dotenv = "0.15.0" -dirs = "5.0.1" -tokio = { version = "1.40.0", features = ["full"] } -sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -axum = { version = "0.7.5", features = ["macros"] } -serde = { version = "1.0.199", features = ["derive"] } -serde_json = "1.0.128" -uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } -futures = { version = "0.3.30", features = ["executor"] } -cron = "0.13.0" -git2 = "0.19.0" -thiserror = "2.0.3" diff --git a/backend/api_server/src/blog/api/get_page_count.rs b/backend/api_server/src/blog/api/get_page_count.rs deleted file mode 100644 index db540af..0000000 --- a/backend/api_server/src/blog/api/get_page_count.rs +++ /dev/null @@ -1,26 +0,0 @@ -use constants::blog::PAGE_SIZE; - -use axum::extract::State; -use axum::Json; -use serde_json::{json, Value}; -use sqlx::{query, PgPool, Row}; -use tracing::error; - -pub async fn get_page_count( - State(db_connection): State, -) -> Result, Json> { - match query("select cast(ceil(count(*) / $1) as integer) as res from post") - .bind(PAGE_SIZE as f32) - .fetch_one(&db_connection) - .await - { - Ok(res) => { - let res: i32 = res.get("res"); - Ok(Json::from(json!({"res": res}))) - } - Err(e) => { - error!("Failed to get blog page count: {}", e.to_string()); - Err(Json::from(json!({"error": e.to_string()}))) - } - } -} diff --git a/backend/api_server/src/blog/api/get_post.rs b/backend/api_server/src/blog/api/get_post.rs deleted file mode 100644 index 2374dec..0000000 --- a/backend/api_server/src/blog/api/get_post.rs +++ /dev/null @@ -1,153 +0,0 @@ -use axum::{ - extract::{Path, State}, - http::StatusCode, - Json, -}; -use serde_json::{json, Value}; -use sqlx::{query, Error, PgPool, Row}; -use std::str::FromStr; -use tracing::error; -use uuid::Uuid; - -pub async fn get_post( - State(db_connection): State, - Path(post_id): Path, -) -> Result, (StatusCode, Json)> { - let post_id = match Uuid::from_str(post_id.as_str()) { - Ok(id) => id, - Err(e) => { - error!( - "Failed to parse UUID <{}>: {}", - post_id.as_str(), - e.to_string() - ); - return Err(( - StatusCode::BAD_REQUEST, - Json::from(json!(("error", e.to_string()))), - )); - } - }; - - let post = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post -FROM post t -WHERE post_id = $1; -"#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("post"), - Err(e) => { - error!( - "Database error on Reading `base post information` from table `post`: <{}>: {}", - post_id.to_string(), - e.to_string() - ); - return match e { - Error::RowNotFound => Err(( - StatusCode::BAD_REQUEST, - Json::from(json!({"error": e.to_string()})), - )), - _ => Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )), - }; - } - }; - - let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;") - .bind(post_id) - .fetch_all(&db_connection) - .await - { - Ok(tags) => tags.iter().map(|tag| tag.get::("tag")).collect(), - Err(e) => { - error!("Database query error :{}", e.to_string()); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let meta = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta -FROM meta t -WHERE post_id = $1; - "#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("meta"), - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "post meta data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let adjacent = match query(r" -WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1), - prev AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date < (SELECT date FROM target_post) - ORDER BY t.date DESC - LIMIT 1), - next AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date > (SELECT date FROM target_post) - ORDER BY t.date - LIMIT 1) -SELECT json_strip_nulls(json_build_object( - 'prev', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM prev), - 'next', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM next) - )) AS result; -" - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(res) => {res.get::("result")} - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "adjoint posts data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - - }; - - let mut res = post.as_object().unwrap().to_owned(); - res.insert("tags".to_string(), tags.into()); - res.insert("meta".to_string(), meta); - res.insert("adj".to_string(), adjacent); - - Ok(Json::from(json!(res))) -} diff --git a/backend/api_server/src/blog/api/get_post_list.rs b/backend/api_server/src/blog/api/get_post_list.rs deleted file mode 100644 index 9db7c61..0000000 --- a/backend/api_server/src/blog/api/get_post_list.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::error::Error; - -use constants::blog::PAGE_SIZE; - -use axum::{extract::Query, extract::State, http::StatusCode, Json}; -use chrono::{DateTime, Utc}; -use futures::future::join_all; -use serde::Deserialize; -use serde_json::{json, Map, Value}; -use sqlx::{postgres::PgRow, query, PgPool, Row}; -use tracing::error; -use uuid::Uuid; - -#[derive(Deserialize)] -pub struct Params { - page: Option, -} - -#[axum::debug_handler] -pub async fn get_post_list( - State(db_connection): State, - Query(params): Query, -) -> Result, (StatusCode, Json)> { - let search_param_page = params.page.unwrap_or(1); - if search_param_page < 1 { - return Err(( - StatusCode::BAD_REQUEST, - Json::from(json!({"error": "`page` should be greater than 0"})), - )); - } - - let offset = PAGE_SIZE * (search_param_page - 1); - - let post_list = sqlx::query("SELECT * FROM post ORDER BY first_update DESC LIMIT $1 OFFSET $2") - .bind(PAGE_SIZE as i32) - .bind(offset as i32) - .fetch_all(&db_connection) - .await; - let post_list = match post_list { - Ok(post_list) => post_list, - Err(e) => { - error!( - "Database query error on fetching post list: {}", - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let mut res = Vec::new(); - let posts: Vec<_> = post_list - .into_iter() - .map(|post| async { analyze_post(&db_connection, post).await }) - .collect(); - let posts = join_all(posts).await; - for post in posts { - match post { - Ok(p) => res.push(p), - Err(e) => { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(("error", e.to_string()))), - )) - } - } - } - - // res.insert("posts".to_string(), Value::from(posts)); - Ok(Json::from(Value::from(res))) -} - -async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result> { - let post_id: Uuid = post.get("post_id"); - let tags = fetch_post_tags(db_connection, post_id).await; - let meta = fetch_post_meta(db_connection, post_id).await.unwrap(); - let header_img = meta.get("header_img"); - let last_update = meta.get("last_update"); - - Ok(json!({ - "post_id": post_id.to_string(), - "title": post.get::("title"), - "category": post.get::("category"), - "header_img": header_img, - "sub_title": post.get::("sub_title"), - "summary": post.get::("summary"), - "last_update": last_update, - "first_update": post.get::, _>("first_update"), - "tags": tags, - })) -} - -async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option { - match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta -FROM meta t -WHERE post_id = $1; - "#, - ) - .bind(post_id) - .fetch_one(db_connection) - .await - { - Ok(value) => Some(value.get::("meta")), - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "post meta data", - "meta", - post_id.to_string(), - e.to_string() - ); - None - } - } -} - -async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec { - match query("SELECT tag FROM Tag WHERE post_id = $1") - .bind(post_id) - .fetch_all(db_connection) - .await - { - Ok(res) => res.iter().map(|tag| tag.get::("tag")).collect(), - Err(_) => vec![], - } -} diff --git a/backend/api_server/src/blog/api/mod.rs b/backend/api_server/src/blog/api/mod.rs deleted file mode 100644 index 5e23c3d..0000000 --- a/backend/api_server/src/blog/api/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod get_page_count; -pub mod get_post; -pub mod get_post_list; diff --git a/backend/api_server/src/blog/cron/check_update/error.rs b/backend/api_server/src/blog/cron/check_update/error.rs deleted file mode 100644 index 0d27c29..0000000 --- a/backend/api_server/src/blog/cron/check_update/error.rs +++ /dev/null @@ -1,49 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum PostIdxError { - #[error("missing essential field `{}` for file `{}`", field, filename)] - MissingField { field: String, filename: String }, - - #[error( - "invalid field `{}` for file `{}`, <{}>: {}", - field_name, - filename, - err_field, - msg - )] - InvalidField { - field_name: String, - err_field: String, - filename: String, - msg: String, - }, - - #[error( - "database error on writing `{}` to table `{}`: <{}>: {}", - data_desc, - db_table, - id, - msg - )] - DBWriteFailure { - data_desc: String, - db_table: String, - id: String, - msg: String, - }, - - #[error( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - data_desc, - db_table, - id, - msg - )] - DBReadFailure { - data_desc: String, - db_table: String, - id: String, - msg: String, - }, -} diff --git a/backend/api_server/src/blog/cron/check_update/index.rs b/backend/api_server/src/blog/cron/check_update/index.rs deleted file mode 100644 index f391639..0000000 --- a/backend/api_server/src/blog/cron/check_update/index.rs +++ /dev/null @@ -1,351 +0,0 @@ -use std::error::Error; -use std::path::Path; - -use crate::blog::cron::check_update::utils::get_md_file_basename; -use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date}; -use markdown::MdFile; - -use sqlx::{query, PgPool, Pool, Postgres, Row}; -use tokio::runtime::Runtime; -use tracing::{info, warn}; - -async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - let post_id = post.file_id; - let title = match post.meta.get("title") { - None => { - return Err(PostIdxError::MissingField { - field: "title".to_string(), - filename: post.filename.clone(), - }) - } - Some(title) => title.as_str().unwrap_or_else(|| { - warn!("invalid title for `{}`", &post.filename); - "invalid title" - }), - }; - let sub_title = post - .meta - .get("sub_title") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let summary = post - .meta - .get("summary") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let content = post.content.as_str(); - let category = match post.meta.get("category") { - None => { - return Err(PostIdxError::MissingField { - field: "category".to_string(), - filename: post.filename.clone(), - }) - } - Some(title) => title.as_str().unwrap_or_else(|| { - warn!("invalid category for `{}`", &post.filename); - "invalid category" - }), - }; - let last_update = { - if let Some(last_update) = post.meta.get("last_update") { - match parse_date(last_update.as_str().unwrap()) { - Ok(date) => Some(date), - Err(e) => { - return Err(PostIdxError::InvalidField { - field_name: "last_update".to_string(), - err_field: last_update.to_string(), - filename: post.filename.clone(), - msg: e.to_string(), - }) - } - } - } else { - None - } - }; - let first_update = match post.meta.get("date") { - None => { - return Err(PostIdxError::MissingField { - field: "date".to_string(), - filename: post.filename.clone(), - }) - } - Some(first_update) => match parse_date(first_update.as_str().unwrap()) { - Ok(date) => date, - Err(e) => { - return Err(PostIdxError::InvalidField { - field_name: "date".to_string(), - err_field: first_update.to_string(), - filename: post.filename.clone(), - msg: e.to_string(), - }) - } - }, - }; - let filename = get_md_file_basename(&post.filename); - - if let Err(e) = query( - r##" -INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) -ON CONFLICT (post_id) DO -UPDATE SET - title = EXCLUDED.title, - summary = EXCLUDED.summary, - content = EXCLUDED.content, - last_update = EXCLUDED.last_update, - first_update = EXCLUDED.first_update, - sub_title = EXCLUDED.sub_title, - category = EXCLUDED.category, - filename = EXCLUDED.filename; - "##, - ) - .bind(post_id) - .bind(title) - .bind(summary) - .bind(content) - .bind(last_update) - .bind(first_update) - .bind(sub_title) - .bind(category) - .bind(filename) - .execute(db_connection) - .await - { - return Err( - PostIdxError::DBWriteFailure { - data_desc: "post base information".to_string(), - db_table: "Post".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - } - ); - } - Ok(()) -} - -async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - if let Some(tags_string) = post.meta.get("tags").to_owned() { - let tags = match tags_string.as_array().map(|vec| { - vec.iter() - .map(|val| val.as_str().unwrap()) - .collect::>() - }) { - None => { - return Err(PostIdxError::MissingField { - field: "tags".to_string(), - filename: post.filename.clone(), - }) - } - Some(val) => val, - }; - - let post_id = post.file_id; - - // remove old record - if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") - .bind(post_id) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - }); - } - - // add new record - for tag in tags { - match query("INSERT INTO Tag VALUES ($1, $2);") - .bind(post_id) - .bind(tag) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - }) - } - } - } - } - - Ok(()) -} - -async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { - let base64_form = post.hash.clone(); - match query("SELECT hash FROM Hash WHERE post_id = $1") - .bind(post.file_id) - .fetch_optional(db_connection) - .await - { - Ok(row) => { - let db_record: String = match row { - None => "".to_string(), - Some(row) => row.try_get("hash").unwrap_or("".to_string()), - }; - if base64_form == db_record { - Ok(false) - } else { - if let Err(e) = query( - r#" -INSERT INTO Hash (post_id, hash) VALUES ($1, $2) - ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, - ) - .bind(post.file_id) - .bind(&base64_form) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }); - } - Ok(true) - } - } - Err(e) => Err(PostIdxError::DBReadFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }), - } -} - -async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - if let Err(e) = query( - // todo: sanitization - format!( - r#" -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY['tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY(_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - "#, post.file_id.to_string().as_str(), post.meta - ).as_str() - ) - .execute(db_connection) - .await { - Err(PostIdxError::DBWriteFailure { - data_desc: "post meta data".to_string(), - db_table: "Meta".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }) - } else { - Ok(()) - } -} - -pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { - let md_file = MdFile::from_file(path)?; - - async fn trail(db_conn: &Pool, md_file: &MdFile) -> Result<(), PostIdxError> { - base(db_conn, md_file).await?; - tag(db_conn, md_file).await?; - meta(db_conn, md_file).await?; - hash(db_conn, md_file).await.map(|_| ()) - } - let mut cnt = 5; - - while let Err(e) = trail(db_conn, &md_file).await { - match e { - PostIdxError::DBWriteFailure { .. } => { - cnt -= 1; - } - PostIdxError::DBReadFailure { .. } => { - cnt -= 1; - } - _ => return Err(e.into()), - } - if cnt <= 0 { - return Err(e.into()); - } - } - - info!("indexed markdown file: {}", path.display()); - Ok(()) -} - -pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - if let Err(e) = query( - r#" -WITH deleted_post AS ( - DELETE FROM post - WHERE id = (SELECT id FROM post WHERE filename = $1) - RETURNING id -) -DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); -DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); -DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); - "#, - ) - .bind(file_basename) - .execute(db_conn) - .await - { - Err(Box::from( - format!("failed to drop index: {}", e).to_string(), - )) - } else { - Ok(()) - } - }) -} diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs deleted file mode 100644 index 2d531e3..0000000 --- a/backend/api_server/src/blog/cron/check_update/mod.rs +++ /dev/null @@ -1,146 +0,0 @@ -use std::env; -use std::error::Error; -use std::fs::remove_dir_all; -use std::path::{Path, PathBuf}; - -use crate::{ - blog::cron::check_update::{ - index::drop_index, - utils::{expand_path, get_md_file_basename, index_files, search_md}, - }, - utils::git, -}; -use constants::config::general::{default_value, var_name}; -use macros::panic_with_log; - -use git2::{Delta, Repository}; -use sqlx::{Pool, Postgres}; -use tracing::{error, info, warn, Level}; - -mod error; -mod index; -mod utils; - -pub fn task(db_connection: &Pool) { - // init - let git_work_dir = { - let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { - expand_path(default_log_path) - } else { - let fallback_path = expand_path(default_value::WORK_DIR.to_string()); - warn!( - "unset env var `{}`, using default path: {}/blog_git", - var_name::WORK_DIR, - fallback_path - ); - fallback_path - }; - Path::new(&path).join("blog_git") - }; - - // clone & check update - - let git_url = match env::var(var_name::BLOG_GIT_URL) { - Ok(url) => url, - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to read `{}`: {}", - var_name::BLOG_GIT_URL, - e - ) - } - }; - let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) - .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); - let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) - .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); - - match Repository::open(&git_work_dir) { - Ok(repo) => { - // prevent git remote url mismatch (clean cache) - match repo.find_remote(&git_remote_name) { - Ok(remote) => { - if remote.url().is_some_and(|url| url != git_url) { - remove_dir_all(&git_work_dir).unwrap(); - warn!("blog content git({}) url mismatch, reinitialized", &git_url); - return; - } - } - Err(e) => { - remove_dir_all(&git_work_dir).unwrap(); - error!( - "unexpected error on indexing blog content git({}), reinitialized: {}", - &git_url, e - ); - } - } - - match git::sync(&git_remote_name, &git_remote_branch, &repo) { - Ok(updates) => { - if !updates.is_empty() { - // incremental update - info!( - "blog content git({}) update detected: {:?}", - &git_url, updates - ); - - let mut update_md: Vec = vec![]; - - for update in updates { - if [Delta::Deleted, Delta::Untracked].contains(&update.status) { - if let Some(path) = &update.old_path { - match drop_index( - &get_md_file_basename( - path.file_name().unwrap().to_str().unwrap(), - ), - db_connection, - ) { - Ok(_) => info!( - "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", - &git_url, - &path.display() - ), - Err(e) => error!( - "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index: {}", - &git_url, - &path.display(), - e - ) - } - } - } else if let Some(path) = &update.new_path { - if path.components().any(|comp| comp.as_os_str() == "posts") - && path.extension().is_some_and(|ext| ext == "md") - { - // update_md.push(path.to_path_buf()); - update_md.push(dbg!(path.to_path_buf())); - } - } - } - - index_files(&update_md, db_connection); - } - } - Err(e) => { - error!( - "failed to sync blog content git({}) updates: {}", - &git_url, e - ); - } - } - } - Err(_) => { - match Repository::clone(&git_url, &git_work_dir) { - Ok(_repo) => { - // full repo index - let files = search_md(git_work_dir.join("posts")); - index_files(&files, db_connection); - } - Err(e) => { - error!("failed to clone blog content git({}): {}", &git_url, e); - } - } - } - }; -} diff --git a/backend/api_server/src/blog/cron/check_update/utils.rs b/backend/api_server/src/blog/cron/check_update/utils.rs deleted file mode 100644 index c20d98a..0000000 --- a/backend/api_server/src/blog/cron/check_update/utils.rs +++ /dev/null @@ -1,86 +0,0 @@ -use dirs::home_dir; -use std::path::PathBuf; - -use crate::blog::cron::check_update::index::index_file; -use chrono::{DateTime, FixedOffset, ParseResult}; -use constants::config::general::{default_value, var_name}; -use constants::time::DEFAULT_DATE_FORMAT; - -use futures::executor::block_on; -use futures::future::join_all; -use sqlx::{Pool, Postgres}; -use tokio::runtime::Runtime; -use tokio::spawn; -use tracing::{error, warn}; - -pub fn search_md(entry: PathBuf) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry] - } else { - vec![] - } -} - -pub fn index_files(files: &[PathBuf], db_conn: &Pool) { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let files_clone = dbg!(files).to_owned(); - let handlers = files_clone - .into_iter() - .map(|file: PathBuf| { - let conn_clone = db_conn.clone(); - spawn(async move { - if let Err(e) = index_file(&file, &conn_clone).await { - error!("failed to index markdown file `{}`: {}", file.display(), e); - } - }) - }) - .collect::>(); - - block_on(join_all(handlers)); - }); -} - -pub fn expand_path(path: String) -> String { - if path.starts_with('~') { - home_dir() - .map(|dir| { - dir.join(path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .unwrap() - }) - .unwrap_or_else(|| { - warn!( - "failed to parse env var `{}`, using default value `{}`", - var_name::WORK_DIR, - default_value::WORK_DIR - ); - default_value::WORK_DIR.to_string() - }) - } else { - path - } -} - -pub fn parse_date(date_str: &str) -> ParseResult> { - DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) -} - -pub fn get_md_file_basename(f_name: &str) -> String { - f_name.replace(".md", "") -} diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs deleted file mode 100644 index 1cb8398..0000000 --- a/backend/api_server/src/blog/cron/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -use crate::scheduler::tasks::basic::BasicTask; -use crate::scheduler::tasks::Task; -use sqlx::{Pool, Postgres}; - -pub mod check_update; - -pub fn get_tasks(db_connection: Pool) -> Vec> { - let db_conn_clone = db_connection.clone(); - vec![ - // BasicTask::new(move || check_update::task(&db_conn_clone), "0/15 * * * * *") - BasicTask::new(move || check_update::task(&db_conn_clone), "* * * * * *") - .unwrap() - .to_task(), - ] -} diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs deleted file mode 100644 index 6d14b15..0000000 --- a/backend/api_server/src/blog/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod api; -pub mod cron; - -use crate::blog::api::{ - get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, -}; - -use axum::routing::get; -use axum::Router; -use sqlx::{Pool, Postgres}; - -pub fn get_router() -> Router> { - Router::new() - .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count)) - .route("/get_post/:post_id", get(get_post)) -} diff --git a/backend/api_server/src/init/cron.rs b/backend/api_server/src/init/cron.rs deleted file mode 100644 index 429537a..0000000 --- a/backend/api_server/src/init/cron.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::blog; -use crate::scheduler::Scheduler; -use sqlx::{Pool, Postgres}; - -pub fn start(db_pool: Pool) { - let scheduler = Scheduler::new(); - let _ = vec![blog::cron::get_tasks(db_pool.clone())] - .into_iter() - .map(|list| scheduler.insert_list(list)) - .collect::>(); - scheduler.start().unwrap() -} diff --git a/backend/api_server/src/init/database.rs b/backend/api_server/src/init/database.rs deleted file mode 100644 index c2a8aad..0000000 --- a/backend/api_server/src/init/database.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::env; -use std::error::Error; -use std::str::FromStr; - -use constants::config::database::{default_value, var_name}; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; -use sqlx::{ConnectOptions, Pool, Postgres}; -use tracing::info; - -pub async fn init() -> Result, Box> { - let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) - .unwrap() - .disable_statement_logging(); - Ok(PgPoolOptions::new() - .max_connections(1) // todo: temporal solution, to be fixed - .connect_with(db_connect_option) - .await?) -} - -fn parse_db_uri() -> String { - env::var(var_name::URI).unwrap_or_else(|_| { - info!( - "`{}` env var is not set, \ - using `{}`, `{}`, `{}`, `{}` and `{}`", - var_name::URI, - var_name::USERNAME, - var_name::PASSWORD, - var_name::HOST, - var_name::PORT, - var_name::DBNAME - ); - let postgres_user = env::var(var_name::USERNAME).unwrap_or(default_value::USERNAME.into()); - let postgres_password = - env::var(var_name::PASSWORD).unwrap_or(default_value::PASSWORD.into()); - let db_host = env::var(var_name::HOST).unwrap_or(default_value::HOST.into()); - let db_port = env::var(var_name::PORT).unwrap_or(default_value::PORT.into()); - let postgres_db = env::var(var_name::DBNAME).unwrap_or(default_value::DBNAME.into()); - let db_type = default_value::DATABASE_TYPE; - format!("{db_type}://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") - }) -} diff --git a/backend/api_server/src/init/logger.rs b/backend/api_server/src/init/logger.rs deleted file mode 100644 index 5fd56b4..0000000 --- a/backend/api_server/src/init/logger.rs +++ /dev/null @@ -1,106 +0,0 @@ -use std::path::Path; -use std::{env, io}; - -use chrono::Local; -use constants::config::general::{default_value, var_name}; -use dirs::home_dir; -use tracing::info; -use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; -use tracing_subscriber::{ - filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, - layer::SubscriberExt, Layer, Registry, -}; - -struct LocalTimer; - -impl FormatTime for LocalTimer { - fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { - write!( - w, - "{}", - Local::now().format(constants::time::DEFAULT_TIME_FORMAT) - ) - } -} - -type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; - -pub fn init() -> (WorkerGuard, WorkerGuard) { - let log_path = match env::var(var_name::LOG_DIR) { - Ok(path) => Path::new(&path).to_path_buf(), - Err(_) => { - let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { - expand_home_path(default_log_path) - } else { - let fallback_path = expand_home_path(default_value::WORK_DIR.to_string()); - println!( - "[warn] unset env var `{}` and `{}`, trying to log on fallback path: {}/log", - var_name::LOG_DIR, - var_name::WORK_DIR, - fallback_path - ); - fallback_path - }; - Path::new(&path).join("log") - } - }; - - let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); - let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); - - let info_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_ansi(false); - let err_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_ansi(false); - let stdout_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_timer(LocalTimer); - - let subscribers = Registry::default() - .with( - fmt::Layer::default() - .with_writer(info_log_writer) - .event_format(info_format), - ) - .with( - fmt::Layer::default() - .with_writer(err_log_writer) - .event_format(err_format) - .with_filter(LevelFilter::ERROR), - ) - .with( - fmt::Layer::default() - .with_writer(io::stdout) - .with_ansi(true) - .event_format(stdout_format), - ); - - tracing::subscriber::set_global_default(subscribers) - .expect("[fatal] failed to setup logging system"); - - info!("logging to dir: {}", log_path.to_str().unwrap()); - - (info_guard, error_guard) -} - -fn expand_home_path(path: String) -> String { - if path.starts_with('~') { - let home_path = home_dir() - .unwrap_or_else(|| panic!("[fatal] unable to locate home dir when parsing env `{}`, try to use absolute path instead", var_name::WORK_DIR)); - home_path - .join(path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .unwrap() - } else { - path - } -} diff --git a/backend/api_server/src/init/misc.rs b/backend/api_server/src/init/misc.rs deleted file mode 100644 index 6a08bc2..0000000 --- a/backend/api_server/src/init/misc.rs +++ /dev/null @@ -1,14 +0,0 @@ -use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds}; -use tracing::warn; - -pub fn init() { - // git2 option - unsafe { - if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) { - warn!("failed to set git2 server connect timeout: {}", e); - } - if let Err(e) = set_server_timeout_in_milliseconds(60_000) { - warn!("failed to set git2 server timeout: {}", e); - } - } -} diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs deleted file mode 100644 index 507d52a..0000000 --- a/backend/api_server/src/init/mod.rs +++ /dev/null @@ -1,58 +0,0 @@ -mod cron; -mod database; -pub mod logger; -mod misc; -mod schema; -pub mod server; - -use macros::panic_with_log; -use std::{env, panic}; -use tracing::{info, Level}; - -pub fn load_env() { - if let Some(env_file) = env::args().nth(1) { - match dotenv::from_filename(&env_file) { - Ok(_) => { - println!("[info] env loaded from {}, starting up", env_file); - } - Err(e) => { - panic!("[fatal] failed to read {}: {}", env_file, e); - } - } - } else { - println!("[info] no .env file referred, starting up") - } -} - -pub async fn start() { - misc::init(); - - // connect database - let db_pool = match database::init().await { - Ok(pool) => { - info!("database connection pool initialized"); - pool - } - Err(e) => { - panic_with_log!( - Level::ERROR, - "database connection pool failed to initialize: {}", - e - ); - } - }; - - // schema init - schema::init(&db_pool).await; - - // load cron task - cron::start(db_pool.clone()); - - // api serve - match server::start(db_pool).await { - Ok(_) => {} - Err(e) => { - panic_with_log!(Level::ERROR, "server panicked: {}", e) - } - } -} diff --git a/backend/api_server/src/init/schema.rs b/backend/api_server/src/init/schema.rs deleted file mode 100644 index ef9def0..0000000 --- a/backend/api_server/src/init/schema.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sqlx::{query, Pool, Postgres}; -use tracing::info; - -const POST_BASE_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - filename TEXT UNIQUE NOT NULL, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ, - first_update TIMESTAMPTZ NOT NULL -); -"##; - -// base model, will add more cols at runtime -const POST_META_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); -"##; - -const POST_HASH_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); -"##; - -const TAG_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); -"##; - -pub async fn init(db_connection: &Pool) { - query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); - query(POST_META_MODEL).execute(db_connection).await.unwrap(); - query(TAG_MODEL).execute(db_connection).await.unwrap(); - query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); - info!("database schema initialized"); -} diff --git a/backend/api_server/src/init/server.rs b/backend/api_server/src/init/server.rs deleted file mode 100644 index 703a043..0000000 --- a/backend/api_server/src/init/server.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::env; -use std::error::Error; -use std::num::ParseIntError; -use std::process::exit; - -use crate::blog; -use axum::Router; -use constants::config::server::{default_value, var_name}; -use macros::panic_with_log; -use sqlx::{Pool, Postgres}; -use tracing::{error, info, warn, Level}; - -pub async fn start(db_pool: Pool) -> Result<(), Box> { - let app = Router::new() - .nest("/blog", blog::get_router()) - .with_state(db_pool); - - let port: u32 = match env::var(var_name::PORT) { - Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { - warn!( - "failed to parse `{}`, using default port: {}", - var_name::PORT, - e.to_string() - ); - default_value::PORT - }), - Err(_) => default_value::PORT, - }; - - let listener = - match tokio::net::TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await { - Ok(listener) => { - info!("server started on {}:{}", default_value::HOST, port); - listener - } - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to start server on {}:{}: {}", - default_value::HOST, - port, - e.to_string() - ); - } - }; - - axum::serve(listener, app).await.unwrap_or_else(|e| { - error!("failed to start axum: {}", e.to_string()); - exit(1); - }); - Ok(()) -} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs deleted file mode 100644 index 8aab2c7..0000000 --- a/backend/api_server/src/main.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod blog; -mod init; -mod scheduler; -mod utils; - -#[tokio::main] -async fn main() { - init::load_env(); - - let _guards = init::logger::init(); - - init::start().await; -} diff --git a/backend/api_server/src/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs deleted file mode 100644 index ce86797..0000000 --- a/backend/api_server/src/scheduler/mod.rs +++ /dev/null @@ -1,194 +0,0 @@ -use crate::{ - scheduler::scheduler_error::SchedulerError, - scheduler::scheduler_error::SchedulerError::IdAlreadyRunning, - scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper, - scheduler::tasks::Task, -}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::RwLock; -use std::{collections::HashMap, sync::Arc, thread}; -use tracing::log::error; -use uuid::Uuid; - -pub mod scheduler_error; -mod sleeper; -mod task_func; -pub mod tasks; - -struct TaskStatus { - sleeper: Option>, - is_alive: AtomicBool, -} - -#[derive(Default)] -pub struct Scheduler { - tasks: Arc>>>>, - running: Arc>>>, -} - -impl Scheduler { - /// Start all tasks in the scheduler - pub fn start(&self) -> Result<(), SchedulerError> { - for (k, _v) in self.tasks.read()?.iter() { - self.run_non_blocking(k)? - } - Ok(()) - } - - /// Terminate all tasks in the scheduler - pub fn halt() {} - - fn run_task( - running: Arc>>>, - id: Uuid, - task: Arc>, - ) -> Result<(), SchedulerError> { - if running.read()?.get(&id).is_some() { - return Err(IdAlreadyRunning); - } else { - running.write()?.insert( - id, - TaskStatus { - sleeper: None, - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - loop { - if let Some(next_execution) = task.get_next_execution() { - // guarantee is alive - let task_status = { - let running_reading_guard = running.read()?; - running_reading_guard.get(&id).cloned().clone() - }; - if let Some(task_status) = task_status { - if !&task_status.is_alive.load(Ordering::SeqCst) { - { - let mut running_lock_guard = running.write()?; - running_lock_guard.remove(&id); - } - return Ok(()); - } else { - let sleeper = Arc::from(SkippableSleeper::from(next_execution)); - { - let mut running_lock_guard = running.write()?; - running_lock_guard.insert( - id, - TaskStatus { - sleeper: Some(sleeper.clone()), - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - sleeper.start(); - task.call(); - } - } - } else { - error!("unexpected error "); - break; - } - } - Ok(()) - } - - /// Run a task with `id` - pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - Self::run_task(self.running.clone(), id, task.clone()) - } else { - Err(IdNotFound) - } - } - - pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - let running_clone = self.running.clone(); - let task_clone = task.clone(); - thread::spawn(move || Self::run_task(running_clone, id, task_clone)); - Ok(()) - } else { - Err(IdNotFound) - } - } - - fn get_running(&self, id: &Uuid) -> Option> { - let ret = self.running.read().unwrap().get(id).cloned(); - ret - } - - /// Stop a task with `id` - pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - status.is_alive.store(false, Ordering::SeqCst); - if let Some(s) = &status.sleeper { - s.skip(); - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } - } - } - - pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - if let Some(s) = &status.sleeper { - s.skip() - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } - } - } - - pub fn insert(&self, task: Box) -> Result { - let id = Uuid::new_v4(); - self.tasks.write()?.insert(id, Arc::new(task)); - Ok(id) - } - - pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { - tasks.into_iter().map(|t| self.insert(t)).collect() - } - - pub fn new() -> Self { - Self::default() - } -} - -#[cfg(test)] -mod tests { - use crate::scheduler::tasks::basic::BasicTask; - use crate::scheduler::Scheduler; - use std::error::Error; - use std::thread; - use std::time::Duration; - - #[test] - fn test_run() -> Result<(), Box> { - let scheduler = Scheduler::default(); - let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; - let id = scheduler.insert(Box::new(task))?; - for i in 1..=2 { - println!("Loop {} start", i); - println!("Run {}", i); - scheduler.run_non_blocking(&id)?; - thread::sleep(Duration::from_secs(4)); - scheduler.stop(&id)?; - println!("Stop {}", i); - thread::sleep(Duration::from_secs(2)); - println!("Loop {} finished", i); - } - Ok(()) - } -} diff --git a/backend/api_server/src/scheduler/scheduler_error.rs b/backend/api_server/src/scheduler/scheduler_error.rs deleted file mode 100644 index 43f30dd..0000000 --- a/backend/api_server/src/scheduler/scheduler_error.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; -use std::error::Error; -use std::fmt::{Debug, Display, Formatter}; -use std::sync::PoisonError; - -pub enum SchedulerError { - IdNotFound, - IdAlreadyRunning, - MutexPoison(String), -} - -impl From> for SchedulerError { - fn from(value: PoisonError) -> Self { - MutexPoison(value.to_string()) - } -} - -impl Debug for SchedulerError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self) - } -} - -impl Display for SchedulerError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - SchedulerError::IdNotFound => { - "SchedulerError::IdNotFound".to_string() - } - SchedulerError::IdAlreadyRunning => { - "SchedulerError::IdAlreadyRunning".to_string() - } - MutexPoison(str) => { - format!("SchedulerError::MutexPoison: {}", str) - } - } - ) - } -} - -impl Error for SchedulerError {} diff --git a/backend/api_server/src/scheduler/sleeper.rs b/backend/api_server/src/scheduler/sleeper.rs deleted file mode 100644 index 4155e57..0000000 --- a/backend/api_server/src/scheduler/sleeper.rs +++ /dev/null @@ -1,77 +0,0 @@ -use chrono::{DateTime, Utc}; -use std::sync::{Arc, Condvar, Mutex}; -use std::thread; -use std::time::Duration; - -#[derive(Clone, Debug, Default)] -pub struct SkippableSleeper { - pub duration: Duration, - state: Arc<(Mutex, Condvar)>, -} - -impl SkippableSleeper { - pub fn new(duration: Duration) -> Self { - SkippableSleeper { - duration, - state: Arc::new((Mutex::new(false), Condvar::new())), - } - } - - // Blocks until the duration has passed or skip() is called - pub fn start(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - let state_clone = Arc::clone(&self.state); - let duration_clone = self.duration; - thread::spawn(move || { - thread::sleep(duration_clone); - let (lock, cvar) = &*state_clone; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); - }); - while !*received { - received = cvar.wait(received).unwrap(); - } - } - - // Skip the blocking delay - pub fn skip(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); - } -} - -impl From> for SkippableSleeper { - fn from(value: DateTime) -> Self { - let from_utc_timestamp = value.timestamp() as u64; - let curr_utc_timestamp = Utc::now().timestamp() as u64; - SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) - } -} - -#[cfg(test)] -mod test { - use crate::scheduler::sleeper::SkippableSleeper; - use std::thread; - use std::time::Duration; - - #[test] - fn usage() { - let sleeper = SkippableSleeper::new(Duration::new(10, 0)); - thread::spawn({ - let sleeper = sleeper.clone(); - move || { - println!("Sleeping..."); - sleeper.start(); - println!("Woke up!"); - } - }); - println!("Start Sleeping..."); - thread::sleep(Duration::from_secs(4)); - println!("Skipping sleep..."); - sleeper.skip(); - } -} diff --git a/backend/api_server/src/scheduler/task_func.rs b/backend/api_server/src/scheduler/task_func.rs deleted file mode 100644 index af27ded..0000000 --- a/backend/api_server/src/scheduler/task_func.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub trait TaskFunc: Fn() + Send + Sync + 'static {} - -impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} diff --git a/backend/api_server/src/scheduler/tasks/basic.rs b/backend/api_server/src/scheduler/tasks/basic.rs deleted file mode 100644 index 71fc3a6..0000000 --- a/backend/api_server/src/scheduler/tasks/basic.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::scheduler::task_func::TaskFunc; -use crate::scheduler::tasks::Task; -use chrono::{DateTime, Utc}; -use cron::Schedule; -use std::error::Error; -use std::str::FromStr; - -pub struct BasicTask { - schedule: Schedule, - func: Box, -} - -impl BasicTask { - pub fn new(func: F, cron_expr: &str) -> Result> - where - F: TaskFunc, - { - Ok(BasicTask { - schedule: Schedule::from_str(cron_expr)?, - func: Box::new(func), - }) - } - - pub fn to_task(self) -> Box { - Box::new(self) - } -} - -impl Task for BasicTask { - fn get_next_execution(&self) -> Option> { - self.schedule.upcoming(Utc).next() - } - - fn call(&self) { - (self.func)() - } -} diff --git a/backend/api_server/src/scheduler/tasks/mod.rs b/backend/api_server/src/scheduler/tasks/mod.rs deleted file mode 100644 index 7e3b87a..0000000 --- a/backend/api_server/src/scheduler/tasks/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -use chrono::{DateTime, Utc}; - -pub mod basic; - -pub trait Task: Send + Sync { - fn get_next_execution(&self) -> Option>; - fn call(&self); -} diff --git a/backend/api_server/src/utils/git.rs b/backend/api_server/src/utils/git.rs deleted file mode 100644 index 6743058..0000000 --- a/backend/api_server/src/utils/git.rs +++ /dev/null @@ -1,195 +0,0 @@ -/// git merge -/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs -/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 -/// -use git2::{ - build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, -}; -use std::path::PathBuf; -use tracing::{error, warn}; - - -#[derive(Debug, Eq, PartialEq)] -pub struct FileDelta { - pub old_path: Option, - pub new_path: Option, - pub status: Delta, -} - -fn do_fetch<'a>( - repo: &'a Repository, - refs: &[&str], - remote: &'a mut Remote, -) -> Result, git2::Error> { - let mut fetch_option = FetchOptions::new(); - - // Always fetch all tags. - // Perform a download and also update tips - fetch_option.download_tags(AutotagOption::All); - remote.fetch(refs, Some(&mut fetch_option), None)?; - - let fetch_head = repo.find_reference("FETCH_HEAD")?; - repo.reference_to_annotated_commit(&fetch_head) -} - -fn fast_forward( - repo: &Repository, - lb: &mut Reference, - rc: &AnnotatedCommit, -) -> Result<(), git2::Error> { - let name = match lb.name() { - Some(s) => s.to_string(), - None => String::from_utf8_lossy(lb.name_bytes()).to_string(), - }; - let msg = format!("fast-forward({}) -> {}", name, rc.id()); - // println!("{}", msg); - lb.set_target(rc.id(), &msg)?; - repo.set_head(&name)?; - repo.checkout_head(Some( - build::CheckoutBuilder::default() - // For some reason, the force is required to make the working directory actually get updated - // I suspect we should be adding some logic to handle dirty working directory states, - // but this is just an example so maybe not. - .force(), - ))?; - Ok(()) -} - -fn normal_merge( - repo: &Repository, - local: &AnnotatedCommit, - remote: &AnnotatedCommit, -) -> Result<(), git2::Error> { - let local_tree = repo.find_commit(local.id())?.tree()?; - let remote_tree = repo.find_commit(remote.id())?.tree()?; - let ancestor = repo - .find_commit(repo.merge_base(local.id(), remote.id())?)? - .tree()?; - let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?; - - if idx.has_conflicts() { - warn!("merge conflicts detected"); - repo.checkout_index(Some(&mut idx), None)?; - return Ok(()); - } - let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; - // now create the merge commit - let msg = format!("merge({}) -> {}", remote.id(), local.id()); - let sig = repo.signature()?; - let local_commit = repo.find_commit(local.id())?; - let remote_commit = repo.find_commit(remote.id())?; - - // Do our merge commit and set the current branch head to that commit. - let _merge_commit = repo.commit( - Some("HEAD"), - &sig, - &sig, - &msg, - &result_tree, - &[&local_commit, &remote_commit], - )?; - // Set working tree to match head. - repo.checkout_head(None)?; - Ok(()) -} - -fn do_merge<'a>( - repo: &'a Repository, - remote_branch: &str, - fetch_commit: AnnotatedCommit<'a>, -) -> Result, git2::Error> { - // 1. do a merge analysis - let analysis = repo.merge_analysis(&[&fetch_commit])?; - - let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; - - // 2. Do the appropriate merge - if analysis.0.is_fast_forward() { - // println!("Doing a fast-forward"); - // do a fast-forward - let ref_name = format!("refs/heads/{}", remote_branch); - match repo.find_reference(&ref_name) { - Ok(mut r) => { - fast_forward(repo, &mut r, &fetch_commit)?; - } - Err(_) => { - // The branch doesn't exist, so set the reference to the - // commit directly. Usually this is because you are pulling - // into an empty repository. - repo.reference( - &ref_name, - fetch_commit.id(), - true, - &format!("({}) -> {}", remote_branch, fetch_commit.id()), - )?; - repo.set_head(&ref_name)?; - repo.checkout_head(Some( - build::CheckoutBuilder::default() - .allow_conflicts(true) - .conflict_style_merge(true) - .force(), - ))?; - } - }; - } else if analysis.0.is_normal() { - // do a normal merge - normal_merge(repo, &head_commit, &fetch_commit)?; - } - // else { - // // already up-to-date - // println!("Nothing to do..."); - // } - - calculate_diff(repo, &head_commit, &fetch_commit) -} - -fn calculate_diff( - repo: &Repository, - local: &AnnotatedCommit, - remote: &AnnotatedCommit, -) -> Result, git2::Error> { - let local_tree = repo.find_commit(local.id())?.tree()?; - let remote_tree = repo.find_commit(remote.id())?.tree()?; - let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?; - - let mut updated_files: Vec = Vec::new(); - - if let Err(e) = diff.foreach( - &mut |d, _| { - // let delta = d.status(); - // - // let n_path = d.new_file().path(); - // let o_path = d.old_file().path(); - // - // if let Some(path) = n_path { - // updated_files.insert(path.to_owned()); - // } - // if let Some(path) = o_path { - // updated_files.insert(path.to_owned()); - // } - updated_files.push(FileDelta { - old_path: d.old_file().path().map(PathBuf::from), - new_path: d.new_file().path().map(PathBuf::from), - status: d.status(), - }); - true // Continue - }, - None, - None, - None, - ) { - error!("error at calculating diff: {:?}", e); - } - - Ok(updated_files.into_iter().collect()) -} - -pub fn sync( - remote_name: &str, - remote_branch: &str, - repo: &Repository, -) -> Result, git2::Error> { - let mut remote = repo.find_remote(remote_name)?; - let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; - do_merge(repo, remote_branch, fetch_commit) -} diff --git a/backend/api_server/src/utils/mod.rs b/backend/api_server/src/utils/mod.rs deleted file mode 100644 index c2bf1c3..0000000 --- a/backend/api_server/src/utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod git; diff --git a/backend/constants/Cargo.toml b/backend/constants/Cargo.toml deleted file mode 100644 index db345c7..0000000 --- a/backend/constants/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "constants" -version = "0.1.0" -edition = "2021" - -[dependencies] diff --git a/backend/macros/src/log.rs b/backend/macros/src/log.rs index f7dd357..b3aed4e 100644 --- a/backend/macros/src/log.rs +++ b/backend/macros/src/log.rs @@ -6,7 +6,7 @@ /// use macros::panic_with_log; /// use tracing::Level; /// -/// panic_with_log!(Level::ERROR, "Test panicked: {}", 42); +/// panic_with_log!(Level::ERROR, "test panicked: {}", 42); /// ``` #[macro_export] macro_rules! panic_with_log { @@ -18,6 +18,25 @@ macro_rules! panic_with_log { }}; } +/// log the message with `Level::ERROR` and exit(1) +/// +/// # Example +/// ```rust +/// +/// use macros::error_panic; +/// +/// error_panic!("test panicked: {}", 42); +/// ``` +#[macro_export] +macro_rules! error_panic { + ($msg:literal $(, $($arg:tt)*)?) => {{ + use std::process; + use tracing::{event, Level}; + event!(Level::ERROR, $msg $(, $($arg)*)?); + process::exit(1); + }}; +} + #[cfg(test)] mod tests { #[test] diff --git a/backend/server/schema.sql b/backend/server/schema.sql new file mode 100644 index 0000000..675175e --- /dev/null +++ b/backend/server/schema.sql @@ -0,0 +1,32 @@ +CREATE TABLE tag +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT +); + +CREATE TABLE category +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT +); + +CREATE TABLE post +( + id SERIAL PRIMARY KEY, + identifier VARCHAR(50) UNIQUE NOT NULL, + title VARCHAR(255) NOT NULL, + subtitle VARCHAR(255) NOT NULL, + published_date DATE NOT NULL, + last_update DATE, + content TEXT NOT NULL, + + category SERIAL REFERENCES category (id) NOT NULL +); + +CREATE TABLE post_tag +( + post SERIAL REFERENCES post (id), + tag SERIAL REFERENCES tag (id) +) \ No newline at end of file diff --git a/backend/constants/src/blog.rs b/backend/server/src/constants/blog.rs similarity index 100% rename from backend/constants/src/blog.rs rename to backend/server/src/constants/blog.rs diff --git a/backend/constants/src/config.rs b/backend/server/src/constants/config.rs similarity index 100% rename from backend/constants/src/config.rs rename to backend/server/src/constants/config.rs diff --git a/backend/constants/src/lib.rs b/backend/server/src/constants/mod.rs similarity index 100% rename from backend/constants/src/lib.rs rename to backend/server/src/constants/mod.rs diff --git a/backend/constants/src/time.rs b/backend/server/src/constants/time.rs similarity index 100% rename from backend/constants/src/time.rs rename to backend/server/src/constants/time.rs diff --git a/backend/server/src/main.rs b/backend/server/src/main.rs new file mode 100644 index 0000000..5e6abc2 --- /dev/null +++ b/backend/server/src/main.rs @@ -0,0 +1,7 @@ +mod constants; + +const BUILD_ID: &str = env!("BUILD_ID"); + +fn main() { + println!("{}", BUILD_ID); +} diff --git a/loadbalancer/.gitignore b/loadbalancer/.gitignore new file mode 100644 index 0000000..596b9e4 --- /dev/null +++ b/loadbalancer/.gitignore @@ -0,0 +1,3 @@ +target/ + +Cargo.lock \ No newline at end of file diff --git a/openresty/conf/nocturne.conf b/nocturne.conf similarity index 65% rename from openresty/conf/nocturne.conf rename to nocturne.conf index 787c98f..00df227 100644 --- a/openresty/conf/nocturne.conf +++ b/nocturne.conf @@ -1,24 +1,17 @@ -# server { -# listen 80; -# server_name lumither.com; - -# return 301 https://$host$request_uri; -# } - server { listen 80; - # listen 443 ssl; - # server_name lumither.com; + listen 443 ssl; + server_name lumither.com; # ssl_certificate certs/cert.pem; # ssl_certificate_key certs/key.pem; - # ssl_session_cache shared:SSL:1m; - # ssl_session_timeout 5m; + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 5m; - # ssl_ciphers HIGH:!aNULL:!MD5; - # ssl_prefer_server_ciphers on; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; location / { default_type text/html; diff --git a/openresty/deploy.sh b/openresty/deploy.sh deleted file mode 100755 index dc0a79c..0000000 --- a/openresty/deploy.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -rm -rf /opt/openresty/nginx/lua -cp -r ./lua /opt/openresty/nginx diff --git a/openresty/lua/hello_world.lua b/openresty/lua/hello_world.lua deleted file mode 100644 index 7d90d01..0000000 --- a/openresty/lua/hello_world.lua +++ /dev/null @@ -1 +0,0 @@ -ngx.say("hello world") diff --git a/scripts/dev_server.sh b/scripts/dev_server.sh deleted file mode 100755 index 2082756..0000000 --- a/scripts/dev_server.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# refresh env -/bin/bash ./flush_env.sh - -# start frontend -cd frontend || exit -screen -S nocturne_frontend -d -m bun dev -cd .. - -# start backend -cd backend || exit -screen -S nocturne_backend -d -m cargo r -cd .. diff --git a/scripts/flush_env.sh b/scripts/flush_env.sh deleted file mode 100755 index f746f85..0000000 --- a/scripts/flush_env.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/env bash - -cp .env ./backend -cp .env ./frontend -cp .env.local ./backend -cp .env.local ./frontend diff --git a/scripts/refresh.sh b/scripts/refresh.sh deleted file mode 100755 index f2fa921..0000000 --- a/scripts/refresh.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -source .env - -if [ -z "$BLOG_ENDPOINT" ]; then - echo "BLOG_ENDPOINT is not set in the .env file" - exit 1 -fi - -if [ -z "$GIT_WORK_DIR" ]; then - echo "GIT_WORK_DIR is not set in the .env file" - exit 1 -fi - -if [ -z "$BACKEND_PORT" ]; then - echo "BACKEND_PORT is not set in the .env file" - exit 1 -fi - -check_for_updates() { - cd "$GIT_WORK_DIR" || exit - res=$(git pull) - if echo "$res" | grep -q "Already up to date"; then - echo "Repo is up to date." - else - echo "Repo has updated; calling db update" - curl --request POST -sL \ - --url "localhost:$BACKEND_PORT" - #todo: to be continued - fi -} - -if [ -d "$GIT_WORK_DIR" ]; then - check_for_updates -else - git clone "$BLOG_ENDPOINT" "$GIT_WORK_DIR" - if [ $? -eq 0 ]; then - check_for_updates - else - echo "Failed to clone the repo" - fi -fi diff --git a/scripts/setup.sh b/scripts/setup.sh deleted file mode 100755 index 2efab7c..0000000 --- a/scripts/setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. "$DIR/.env" - -sudo groupadd nocturne -USERNAME="$(whoami)" -echo "$USERNAME" -sudo usermod -aG nocturne "$USERNAME" - -sudo mkdir -p "$LOG_ROOT_DIR/backend" "$LOG_ROOT_DIR/frontend" -sudo chown -R :nocturne "$LOG_ROOT_DIR" -sudo chmod -R 775 "$LOG_ROOT_DIR" - diff --git a/scripts/start_server.sh b/scripts/start_server.sh deleted file mode 100755 index 50c2199..0000000 --- a/scripts/start_server.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# refresh env -/bin/bash ./flush_env.sh - -# start frontend -( -cd frontend || exit -bun i -bun next build -screen -S nocturne_frontend -d -m bun start -cd .. -) & - -# start backend -( -cd backend || exit -screen -S nocturne_backend -d -m cargo r -r -cd .. -) & wait