diff --git a/backend/macros/src/include_str.rs b/backend/macros/src/include_str.rs new file mode 100644 index 0000000..4e4a624 --- /dev/null +++ b/backend/macros/src/include_str.rs @@ -0,0 +1,24 @@ +/// Includes a UTF-8 encoded file as a string +/// +/// The file is located relative to `CARGO_MANIFEST_DIR`. +/// +/// # Example +/// ```rust +/// use macros::include_str_from_crate_root; +/// +/// let str: &str = include_str_from_crate_root!("/query/scheme.sql"); +/// +// todo: impl +// const PATH_PREFIX: &str = "/src"; +// let str: &str = include_str_from_crate_root!("{}/query/scheme.sql", PATH_PREFIX); +/// ``` +#[macro_export] +macro_rules! include_str_from_crate_root { + // ($path:literal $(, $($arg:tt)*)?) => { + // ($path:expr, $($args:tt)*) => { + // include_str!(concat!(env!("CARGO_MANIFEST_DIR"), format_args!($path, $($args)*))) + // }; + ($path:expr) => { + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), $path)) + }; +} diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs index aad31d7..6003095 100644 --- a/backend/macros/src/lib.rs +++ b/backend/macros/src/lib.rs @@ -1 +1,2 @@ +mod include_str; mod log; diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml index d02915f..33944ed 100644 --- a/backend/server/Cargo.toml +++ b/backend/server/Cargo.toml @@ -5,14 +5,25 @@ edition = "2021" [dependencies] macros = { path = "../macros" } +markdown = { path = "../markdown" } - +axum = { version = "0.8.1", features = ["macros"] } tracing = { version = "0.1.41", features = ["log"] } git2 = "0.20.0" -chrono = "0.4.39" -uuid = { version = "1.13.2", features = ["v4"] } +chrono = { version = "0.4.39", features = ["serde"] } cron = "0.15.0" +sqlx = { version = "0.8.3", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono"] } +serde_json = "1.0.133" +serde = { version = "1.0.215", features = ["derive"] } +tracing-subscriber = "0.3.18" +tracing-appender = "0.2.3" +dirs = "6.0.0" +thiserror = "2.0.11" +tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros"] } +dotenv = "0.15.0" +uuid = { version = "1.13.2", features = ["v4"] } +futures = "0.3.31" [build-dependencies] git2 = "0.20.0" -chrono = "0.4.39" \ No newline at end of file +chrono = "0.4.39" diff --git a/backend/server/src/init/database.rs b/backend/server/src/init/database.rs new file mode 100644 index 0000000..f237e1e --- /dev/null +++ b/backend/server/src/init/database.rs @@ -0,0 +1,42 @@ +use std::env; +use std::error::Error; +use std::str::FromStr; + +use crate::constants::config::database::{default_value, var_name}; + +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, Pool, Postgres}; +use tracing::info; + +pub async fn init() -> Result, Box> { + let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) + .unwrap() + .disable_statement_logging(); + Ok(PgPoolOptions::new() + .max_connections(1) // todo: temporary solution, to be fixed + .connect_with(db_connect_option) + .await?) +} + +fn parse_db_uri() -> String { + env::var(var_name::URI).unwrap_or_else(|_| { + info!( + "`{}` env var is not set, \ + using `{}`, `{}`, `{}`, `{}` and `{}`", + var_name::URI, + var_name::USERNAME, + var_name::PASSWORD, + var_name::HOST, + var_name::PORT, + var_name::DBNAME + ); + let postgres_user = env::var(var_name::USERNAME).unwrap_or(default_value::USERNAME.into()); + let postgres_password = + env::var(var_name::PASSWORD).unwrap_or(default_value::PASSWORD.into()); + let db_host = env::var(var_name::HOST).unwrap_or(default_value::HOST.into()); + let db_port = env::var(var_name::PORT).unwrap_or(default_value::PORT.into()); + let postgres_db = env::var(var_name::DBNAME).unwrap_or(default_value::DBNAME.into()); + let db_type = default_value::DATABASE_TYPE; + format!("{db_type}://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") + }) +} diff --git a/backend/server/src/init/logger.rs b/backend/server/src/init/logger.rs new file mode 100644 index 0000000..57ee6be --- /dev/null +++ b/backend/server/src/init/logger.rs @@ -0,0 +1,107 @@ +use std::path::Path; +use std::{env, io}; + +use crate::constants::config::general::{default_value, var_name}; + +use chrono::Local; +use dirs::home_dir; +use tracing::info; +use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; +use tracing_subscriber::{ + filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, + layer::SubscriberExt, Layer, Registry, +}; + +struct LocalTimer; + +impl FormatTime for LocalTimer { + fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { + write!( + w, + "{}", + Local::now().format(crate::constants::time::DEFAULT_TIME_FORMAT) + ) + } +} + +type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; + +pub fn init() -> (WorkerGuard, WorkerGuard) { + let log_path = match env::var(var_name::LOG_DIR) { + Ok(path) => Path::new(&path).to_path_buf(), + Err(_) => { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_home_path(default_log_path) + } else { + let fallback_path = expand_home_path(default_value::WORK_DIR.to_string()); + println!( + "[warn] unset env var `{}` and `{}`, trying to log on fallback path: {}/log", + var_name::LOG_DIR, + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("log") + } + }; + + let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); + let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); + + let info_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let err_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let stdout_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_timer(LocalTimer); + + let subscribers = Registry::default() + .with( + fmt::Layer::default() + .with_writer(info_log_writer) + .event_format(info_format), + ) + .with( + fmt::Layer::default() + .with_writer(err_log_writer) + .event_format(err_format) + .with_filter(LevelFilter::ERROR), + ) + .with( + fmt::Layer::default() + .with_writer(io::stdout) + .with_ansi(true) + .event_format(stdout_format), + ); + + tracing::subscriber::set_global_default(subscribers) + .expect("[fatal] failed to setup logging system"); + + info!("logging to dir: {}", log_path.to_str().unwrap()); + + (info_guard, error_guard) +} + +fn expand_home_path(path: String) -> String { + if path.starts_with('~') { + let home_path = home_dir() + .unwrap_or_else(|| panic!("[fatal] unable to locate home dir when parsing env `{}`, try to use absolute path instead", var_name::WORK_DIR)); + home_path + .join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + } else { + path + } +} diff --git a/backend/server/src/init/misc.rs b/backend/server/src/init/misc.rs new file mode 100644 index 0000000..6a08bc2 --- /dev/null +++ b/backend/server/src/init/misc.rs @@ -0,0 +1,14 @@ +use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds}; +use tracing::warn; + +pub fn init() { + // git2 option + unsafe { + if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) { + warn!("failed to set git2 server connect timeout: {}", e); + } + if let Err(e) = set_server_timeout_in_milliseconds(60_000) { + warn!("failed to set git2 server timeout: {}", e); + } + } +} diff --git a/backend/server/src/init/mod.rs b/backend/server/src/init/mod.rs new file mode 100644 index 0000000..4da997a --- /dev/null +++ b/backend/server/src/init/mod.rs @@ -0,0 +1,51 @@ +mod database; +pub mod logger; +mod misc; +mod schema; + +use std::{env, panic}; + +use crate::modules::{blog::Blog, ModuleTree}; +use macros::error_panic; + +use tracing::info; + +pub fn load_env() { + if let Some(env_file) = env::args().nth(1) { + match dotenv::from_filename(&env_file) { + Ok(_) => { + println!("[info] env loaded from {}, starting up", env_file); + } + Err(e) => { + panic!("[fatal] failed to read {}: {}", env_file, e); + } + } + } else { + println!("[info] no .env file referred, starting up") + } +} + +pub async fn start() { + misc::init(); + + // connect database + let db_pool = match database::init().await { + Ok(pool) => { + info!("database connection pool initialized"); + pool + } + Err(e) => { + error_panic!("database connection pool failed to initialize: {}", e); + } + }; + + // // schema init + // schema::init(&db_pool).await; + + ModuleTree::new() + .add_module(Blog { + db_handler: db_pool, + }) + .blocking_serve() + .await +} diff --git a/backend/server/src/init/schema.rs b/backend/server/src/init/schema.rs new file mode 100644 index 0000000..6cd1363 --- /dev/null +++ b/backend/server/src/init/schema.rs @@ -0,0 +1,49 @@ +use sqlx::{query, Pool, Postgres}; +use tracing::info; +// +// const POST_BASE_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Post +// ( +// post_id UUID PRIMARY KEY, +// title VARCHAR(255) NOT NULL, +// category VARCHAR(255) NOT NULL, +// filename TEXT UNIQUE NOT NULL, +// summary TEXT, +// sub_title TEXT, +// content TEXT NOT NULL, +// last_update TIMESTAMPTZ, +// first_update TIMESTAMPTZ NOT NULL +// ); +// "##; +// +// // base model, will add more cols at runtime +// const POST_META_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Meta +// ( +// post_id UUID PRIMARY KEY +// ); +// "##; +// +// const POST_HASH_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Hash +// ( +// post_id UUID PRIMARY KEY, +// hash VARCHAR(44) +// ); +// "##; +// +// const TAG_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Tag +// ( +// post_id UUID NOT NULL, +// tag VARCHAR(255) NOT NULL +// ); +// "##; + +pub async fn init(db_connection: &Pool) { + // query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); + // query(POST_META_MODEL).execute(db_connection).await.unwrap(); + // query(TAG_MODEL).execute(db_connection).await.unwrap(); + // query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); + info!("database schema initialized"); +} diff --git a/backend/server/src/main.rs b/backend/server/src/main.rs index 655a616..8971580 100644 --- a/backend/server/src/main.rs +++ b/backend/server/src/main.rs @@ -1,8 +1,19 @@ mod constants; +mod init; +mod modules; mod scheduler; +mod utils; + +use crate::init::{logger, start}; const BUILD_ID: &str = env!("BUILD_ID"); -fn main() { +#[tokio::main] +async fn main() { println!("{}", BUILD_ID); + + init::load_env(); + let _guards = logger::init(); + + start().await; } diff --git a/backend/server/src/modules/blog/api/get_page_count.rs b/backend/server/src/modules/blog/api/get_page_count.rs new file mode 100644 index 0000000..771098f --- /dev/null +++ b/backend/server/src/modules/blog/api/get_page_count.rs @@ -0,0 +1,27 @@ +use crate::constants::blog::PAGE_SIZE; + +use axum::extract::State; +use axum::Json; +use serde_json::{json, Value}; +use sqlx::{query, PgPool, Row}; +use tracing::error; + +pub async fn get_page_count( + State(db_connection): State, +) -> Result, Json> { + match query("select cast(ceil(count(*) / $1) as integer) as res from post") + .bind(PAGE_SIZE as f32) + .fetch_one(&db_connection) + .await + { + Ok(res) => { + let res: i32 = res.get("res"); + Ok(Json::from(json!({"res": res}))) + } + Err(e) => { + // todo: rename + error!("Failed to get blog page count: {}", e.to_string()); + Err(Json::from(json!({"error": e.to_string()}))) + } + } +} diff --git a/backend/server/src/modules/blog/api/get_post.rs b/backend/server/src/modules/blog/api/get_post.rs new file mode 100644 index 0000000..2374dec --- /dev/null +++ b/backend/server/src/modules/blog/api/get_post.rs @@ -0,0 +1,153 @@ +use axum::{ + extract::{Path, State}, + http::StatusCode, + Json, +}; +use serde_json::{json, Value}; +use sqlx::{query, Error, PgPool, Row}; +use std::str::FromStr; +use tracing::error; +use uuid::Uuid; + +pub async fn get_post( + State(db_connection): State, + Path(post_id): Path, +) -> Result, (StatusCode, Json)> { + let post_id = match Uuid::from_str(post_id.as_str()) { + Ok(id) => id, + Err(e) => { + error!( + "Failed to parse UUID <{}>: {}", + post_id.as_str(), + e.to_string() + ); + return Err(( + StatusCode::BAD_REQUEST, + Json::from(json!(("error", e.to_string()))), + )); + } + }; + + let post = match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post +FROM post t +WHERE post_id = $1; +"#, + ) + .bind(post_id) + .fetch_one(&db_connection) + .await + { + Ok(value) => value.get::("post"), + Err(e) => { + error!( + "Database error on Reading `base post information` from table `post`: <{}>: {}", + post_id.to_string(), + e.to_string() + ); + return match e { + Error::RowNotFound => Err(( + StatusCode::BAD_REQUEST, + Json::from(json!({"error": e.to_string()})), + )), + _ => Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )), + }; + } + }; + + let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;") + .bind(post_id) + .fetch_all(&db_connection) + .await + { + Ok(tags) => tags.iter().map(|tag| tag.get::("tag")).collect(), + Err(e) => { + error!("Database query error :{}", e.to_string()); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + }; + + let meta = match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta +FROM meta t +WHERE post_id = $1; + "#, + ) + .bind(post_id) + .fetch_one(&db_connection) + .await + { + Ok(value) => value.get::("meta"), + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "post meta data", + "meta", + post_id.to_string(), + e.to_string() + ); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + }; + + let adjacent = match query(r" +WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1), + prev AS (SELECT post_id, t.title, t.sub_title, t.header_img + FROM meta t + WHERE t.date < (SELECT date FROM target_post) + ORDER BY t.date DESC + LIMIT 1), + next AS (SELECT post_id, t.title, t.sub_title, t.header_img + FROM meta t + WHERE t.date > (SELECT date FROM target_post) + ORDER BY t.date + LIMIT 1) +SELECT json_strip_nulls(json_build_object( + 'prev', + (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) + FROM prev), + 'next', + (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) + FROM next) + )) AS result; +" + ) + .bind(post_id) + .fetch_one(&db_connection) + .await + { + Ok(res) => {res.get::("result")} + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "adjoint posts data", + "meta", + post_id.to_string(), + e.to_string() + ); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + + }; + + let mut res = post.as_object().unwrap().to_owned(); + res.insert("tags".to_string(), tags.into()); + res.insert("meta".to_string(), meta); + res.insert("adj".to_string(), adjacent); + + Ok(Json::from(json!(res))) +} diff --git a/backend/server/src/modules/blog/api/get_post_list.rs b/backend/server/src/modules/blog/api/get_post_list.rs new file mode 100644 index 0000000..81800aa --- /dev/null +++ b/backend/server/src/modules/blog/api/get_post_list.rs @@ -0,0 +1,130 @@ +use std::error::Error; + +use crate::constants::blog::PAGE_SIZE; + +use axum::{extract::Query, extract::State, http::StatusCode, Json}; +use chrono::{DateTime, Utc}; +use futures::future::join_all; +use serde::Deserialize; +use serde_json::{json, Map, Value}; +use sqlx::{postgres::PgRow, query, PgPool, Row}; +use tracing::error; +use uuid::Uuid; + +#[derive(Deserialize)] +pub struct Params { + page: Option, +} + +#[axum::debug_handler] +pub async fn get_post_list( + State(db_connection): State, + Query(params): Query, +) -> Result, (StatusCode, Json)> { + let search_param_page = params.page.unwrap_or(1); + if search_param_page < 1 { + return Err(( + StatusCode::BAD_REQUEST, + Json::from(json!({"error": "`page` should be greater than 0"})), + )); + } + + let offset = PAGE_SIZE * (search_param_page - 1); + + let post_list = sqlx::query("SELECT * FROM post ORDER BY first_update DESC LIMIT $1 OFFSET $2") + .bind(PAGE_SIZE as i32) + .bind(offset as i32) + .fetch_all(&db_connection) + .await; + let post_list = match post_list { + Ok(post_list) => post_list, + Err(e) => { + error!( + "Database query error on fetching post list: {}", + e.to_string() + ); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + }; + + let mut res = Vec::new(); + let posts: Vec<_> = post_list + .into_iter() + .map(|post| async { analyze_post(&db_connection, post).await }) + .collect(); + let posts = join_all(posts).await; + for post in posts { + match post { + Ok(p) => res.push(p), + Err(e) => { + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!(("error", e.to_string()))), + )) + } + } + } + + // res.insert("posts".to_string(), Value::from(posts)); + Ok(Json::from(Value::from(res))) +} + +async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result> { + let post_id: Uuid = post.get("post_id"); + let tags = fetch_post_tags(db_connection, post_id).await; + let meta = fetch_post_meta(db_connection, post_id).await.unwrap(); + let header_img = meta.get("header_img"); + let last_update = meta.get("last_update"); + + Ok(json!({ + "post_id": post_id.to_string(), + "title": post.get::("title"), + "category": post.get::("category"), + "header_img": header_img, + "sub_title": post.get::("sub_title"), + "summary": post.get::("summary"), + "last_update": last_update, + "first_update": post.get::, _>("first_update"), + "tags": tags, + })) +} + +async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option { + match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta +FROM meta t +WHERE post_id = $1; + "#, + ) + .bind(post_id) + .fetch_one(db_connection) + .await + { + Ok(value) => Some(value.get::("meta")), + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "post meta data", + "meta", + post_id.to_string(), + e.to_string() + ); + None + } + } +} + +async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec { + match query("SELECT tag FROM Tag WHERE post_id = $1") + .bind(post_id) + .fetch_all(db_connection) + .await + { + Ok(res) => res.iter().map(|tag| tag.get::("tag")).collect(), + Err(_) => vec![], + } +} diff --git a/backend/server/src/modules/blog/api/mod.rs b/backend/server/src/modules/blog/api/mod.rs new file mode 100644 index 0000000..5e23c3d --- /dev/null +++ b/backend/server/src/modules/blog/api/mod.rs @@ -0,0 +1,3 @@ +pub mod get_page_count; +pub mod get_post; +pub mod get_post_list; diff --git a/backend/server/src/modules/blog/cron/check_update/error.rs b/backend/server/src/modules/blog/cron/check_update/error.rs new file mode 100644 index 0000000..0d27c29 --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/error.rs @@ -0,0 +1,49 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PostIdxError { + #[error("missing essential field `{}` for file `{}`", field, filename)] + MissingField { field: String, filename: String }, + + #[error( + "invalid field `{}` for file `{}`, <{}>: {}", + field_name, + filename, + err_field, + msg + )] + InvalidField { + field_name: String, + err_field: String, + filename: String, + msg: String, + }, + + #[error( + "database error on writing `{}` to table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBWriteFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, + + #[error( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBReadFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, +} diff --git a/backend/server/src/modules/blog/cron/check_update/index.rs b/backend/server/src/modules/blog/cron/check_update/index.rs new file mode 100644 index 0000000..fabd478 --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/index.rs @@ -0,0 +1,353 @@ +use std::error::Error; +use std::path::Path; + +use crate::{ + modules::blog::cron::check_update::utils::get_md_file_basename, + modules::blog::cron::check_update::{error::PostIdxError, utils::parse_date}, +}; +use markdown::MdFile; + +use sqlx::{query, PgPool, Pool, Postgres, Row}; +use tokio::runtime::Runtime; +use tracing::{info, warn}; + +async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + let post_id = post.file_id; + let title = match post.meta.get("title") { + None => { + return Err(PostIdxError::MissingField { + field: "title".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid title for `{}`", &post.filename); + "invalid title" + }), + }; + let sub_title = post + .meta + .get("sub_title") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let summary = post + .meta + .get("summary") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let content = post.content.as_str(); + let category = match post.meta.get("category") { + None => { + return Err(PostIdxError::MissingField { + field: "category".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid category for `{}`", &post.filename); + "invalid category" + }), + }; + let last_update = { + if let Some(last_update) = post.meta.get("last_update") { + match parse_date(last_update.as_str().unwrap()) { + Ok(date) => Some(date), + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "last_update".to_string(), + err_field: last_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + } + } else { + None + } + }; + let first_update = match post.meta.get("date") { + None => { + return Err(PostIdxError::MissingField { + field: "date".to_string(), + filename: post.filename.clone(), + }) + } + Some(first_update) => match parse_date(first_update.as_str().unwrap()) { + Ok(date) => date, + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "date".to_string(), + err_field: first_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + }, + }; + let filename = get_md_file_basename(&post.filename); + + if let Err(e) = query( + r##" +INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (post_id) DO +UPDATE SET + title = EXCLUDED.title, + summary = EXCLUDED.summary, + content = EXCLUDED.content, + last_update = EXCLUDED.last_update, + first_update = EXCLUDED.first_update, + sub_title = EXCLUDED.sub_title, + category = EXCLUDED.category, + filename = EXCLUDED.filename; + "##, + ) + .bind(post_id) + .bind(title) + .bind(summary) + .bind(content) + .bind(last_update) + .bind(first_update) + .bind(sub_title) + .bind(category) + .bind(filename) + .execute(db_connection) + .await + { + return Err( + PostIdxError::DBWriteFailure { + data_desc: "post base information".to_string(), + db_table: "Post".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + } + ); + } + Ok(()) +} + +async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Some(tags_string) = post.meta.get("tags").to_owned() { + let tags = match tags_string.as_array().map(|vec| { + vec.iter() + .map(|val| val.as_str().unwrap()) + .collect::>() + }) { + None => { + return Err(PostIdxError::MissingField { + field: "tags".to_string(), + filename: post.filename.clone(), + }) + } + Some(val) => val, + }; + + let post_id = post.file_id; + + // remove old record + if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") + .bind(post_id) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }); + } + + // add new record + for tag in tags { + match query("INSERT INTO Tag VALUES ($1, $2);") + .bind(post_id) + .bind(tag) + .execute(db_connection) + .await + { + Ok(_) => {} + Err(e) => { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }) + } + } + } + } + + Ok(()) +} + +async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { + let base64_form = post.hash.clone(); + match query("SELECT hash FROM Hash WHERE post_id = $1") + .bind(post.file_id) + .fetch_optional(db_connection) + .await + { + Ok(row) => { + let db_record: String = match row { + None => "".to_string(), + Some(row) => row.try_get("hash").unwrap_or("".to_string()), + }; + if base64_form == db_record { + Ok(false) + } else { + if let Err(e) = query( + r#" +INSERT INTO Hash (post_id, hash) VALUES ($1, $2) + ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, + ) + .bind(post.file_id) + .bind(&base64_form) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }); + } + Ok(true) + } + } + Err(e) => Err(PostIdxError::DBReadFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }), + } +} + +async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Err(e) = query( + // todo: sanitization + format!( + r#" +DO +$$ + DECLARE + _post_id UUID := '{}'; + _meta_cols JSONB := '{}'; + _exists BOOLEAN; + _col_name TEXT; + _col_value TEXT; + _sql TEXT; + _insert_columns TEXT := 'post_id'; + _insert_values TEXT := quote_literal(_post_id); + _update_set TEXT := ''; + _blacklist TEXT[] := ARRAY['tags']; + BEGIN + SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; + + FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) + LOOP + IF _col_name = ANY(_blacklist) THEN + CONTINUE; + END IF; + + IF NOT EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'meta' + AND column_name = _col_name + -- may change in the future + AND table_schema = 'public') THEN + _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; + EXECUTE _sql; + END IF; + _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); + _insert_values := _insert_values || ', ' || quote_literal(_col_value); + _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); + END LOOP; + + _update_set := substring(_update_set FROM 3); + + IF _exists THEN + _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); + EXECUTE _sql; + ELSE + _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; + EXECUTE _sql; + END IF; + END; +$$ LANGUAGE plpgsql; + "#, post.file_id.to_string().as_str(), post.meta + ).as_str() + ) + .execute(db_connection) + .await { + Err(PostIdxError::DBWriteFailure { + data_desc: "post meta data".to_string(), + db_table: "Meta".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }) + } else { + Ok(()) + } +} + +pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { + let md_file = MdFile::from_file(path)?; + + async fn trail(db_conn: &Pool, md_file: &MdFile) -> Result<(), PostIdxError> { + base(db_conn, md_file).await?; + tag(db_conn, md_file).await?; + meta(db_conn, md_file).await?; + hash(db_conn, md_file).await.map(|_| ()) + } + let mut cnt = 5; + + while let Err(e) = trail(db_conn, &md_file).await { + match e { + PostIdxError::DBWriteFailure { .. } => { + cnt -= 1; + } + PostIdxError::DBReadFailure { .. } => { + cnt -= 1; + } + _ => return Err(e.into()), + } + if cnt <= 0 { + return Err(e.into()); + } + } + + info!("indexed markdown file: {}", path.display()); + Ok(()) +} + +pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + if let Err(e) = query( + r#" +WITH deleted_post AS ( + DELETE FROM post + WHERE id = (SELECT id FROM post WHERE filename = $1) + RETURNING id +) +DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); + "#, + ) + .bind(file_basename) + .execute(db_conn) + .await + { + Err(Box::from( + format!("failed to drop index: {}", e).to_string(), + )) + } else { + Ok(()) + } + }) +} diff --git a/backend/server/src/modules/blog/cron/check_update/mod.rs b/backend/server/src/modules/blog/cron/check_update/mod.rs new file mode 100644 index 0000000..4471e04 --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/mod.rs @@ -0,0 +1,148 @@ +use std::{ + env, + fs::remove_dir_all, + path::{Path, PathBuf}, +}; + +use crate::{ + constants::config::general::{default_value, var_name}, + modules::blog::cron::check_update::{ + index::drop_index, + utils::{expand_path, get_md_file_basename, index_files, search_md}, + }, + utils::git, +}; +use macros::panic_with_log; + +use git2::{Delta, Repository}; +use sqlx::{Pool, Postgres}; +use tracing::{error, info, warn, Level}; + +mod error; +mod index; +mod utils; + +pub fn task(db_connection: &Pool) { + // init + let git_work_dir = { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_path(default_log_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") + }; + + // clone & check update + + let git_url = match env::var(var_name::BLOG_GIT_URL) { + Ok(url) => url, + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to read `{}`: {}", + var_name::BLOG_GIT_URL, + e + ) + } + }; + let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) + .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); + let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) + .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); + + match Repository::open(&git_work_dir) { + Ok(repo) => { + // prevent git remote url mismatch (clean cache) + match repo.find_remote(&git_remote_name) { + Ok(remote) => { + if remote.url().is_some_and(|url| url != git_url) { + remove_dir_all(&git_work_dir).unwrap(); + warn!("blog content git({}) url mismatch, reinitialized", &git_url); + return; + } + } + Err(e) => { + remove_dir_all(&git_work_dir).unwrap(); + error!( + "unexpected error on indexing blog content git({}), reinitialized: {}", + &git_url, e + ); + } + } + + match git::sync(&git_remote_name, &git_remote_branch, &repo) { + Ok(updates) => { + if !updates.is_empty() { + // incremental update + info!( + "blog content git({}) update detected: {:?}", + &git_url, updates + ); + + let mut update_md: Vec = vec![]; + + for update in updates { + if [Delta::Deleted, Delta::Untracked].contains(&update.status) { + if let Some(path) = &update.old_path { + if drop_index( + &get_md_file_basename( + path.file_name().unwrap().to_str().unwrap(), + ), + db_connection, + ) + .is_ok() + { + info!( + "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", + &git_url, + &path.display() + ); + } else { + error!( + "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", + &git_url, + &path.display() + ) + } + } + } else if let Some(path) = &update.new_path { + if path.components().any(|comp| comp.as_os_str() == "posts") + && path.extension().is_some_and(|ext| ext == "md") + { + update_md.push(path.to_path_buf()); + } + } + } + + index_files(&update_md, db_connection); + } + } + Err(e) => { + error!( + "failed to sync blog content git({}) updates: {}", + &git_url, e + ); + } + } + } + Err(_) => { + match Repository::clone(&git_url, &git_work_dir) { + Ok(_repo) => { + // full repo index + let files = search_md(git_work_dir.join("posts")); + index_files(&files, db_connection); + } + Err(e) => { + error!("failed to clone blog content git({}): {}", &git_url, e); + } + } + } + }; +} diff --git a/backend/server/src/modules/blog/cron/check_update/utils.rs b/backend/server/src/modules/blog/cron/check_update/utils.rs new file mode 100644 index 0000000..85086ca --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/utils.rs @@ -0,0 +1,88 @@ +use std::path::PathBuf; + +use crate::{ + constants::{ + config::general::{default_value, var_name}, + time::DEFAULT_DATE_FORMAT, + }, + modules::blog::cron::check_update::index::index_file, +}; + +use chrono::{DateTime, FixedOffset, ParseResult}; +use dirs::home_dir; +use futures::{executor::block_on, future::join_all}; +use sqlx::{Pool, Postgres}; +use tokio::{runtime::Runtime, spawn}; +use tracing::{error, warn}; + +pub fn search_md(entry: PathBuf) -> Vec { + if entry.ends_with(".git") { + return vec![]; + } + if entry.is_dir() { + if let Ok(files) = entry.read_dir() { + files + .flat_map(|file| { + let file_path = file.unwrap().path(); + search_md(file_path) + }) + .collect() + } else { + vec![] + } + } else if entry.extension().is_some_and(|ext| ext == "md") { + vec![entry] + } else { + vec![] + } +} + +pub fn index_files(files: &[PathBuf], db_conn: &Pool) { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let files_clone = files.to_owned(); + let handlers = files_clone + .into_iter() + .map(|file: PathBuf| { + let conn_clone = db_conn.clone(); + spawn(async move { + if let Err(e) = index_file(&file, &conn_clone).await { + error!("failed to index markdown file `{}`: {}", file.display(), e); + } + }) + }) + .collect::>(); + + block_on(join_all(handlers)); + }); +} + +pub fn expand_path(path: String) -> String { + if path.starts_with('~') { + home_dir() + .map(|dir| { + dir.join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + }) + .unwrap_or_else(|| { + warn!( + "failed to parse env var `{}`, using default value `{}`", + var_name::WORK_DIR, + default_value::WORK_DIR + ); + default_value::WORK_DIR.to_string() + }) + } else { + path + } +} + +pub fn parse_date(date_str: &str) -> ParseResult> { + DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) +} + +pub fn get_md_file_basename(f_name: &str) -> String { + f_name.replace(".md", "") +} diff --git a/backend/server/src/modules/blog/cron/mod.rs b/backend/server/src/modules/blog/cron/mod.rs new file mode 100644 index 0000000..be1305e --- /dev/null +++ b/backend/server/src/modules/blog/cron/mod.rs @@ -0,0 +1 @@ +pub mod check_update; diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs new file mode 100644 index 0000000..7832bf6 --- /dev/null +++ b/backend/server/src/modules/blog/mod.rs @@ -0,0 +1,50 @@ +mod api; +mod cron; + +use crate::{ + modules::{ + blog::{ + api::{ + get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, + }, + cron::check_update, + }, + Module, + }, + scheduler::tasks::basic::BasicTask, + scheduler::tasks::CronTask, +}; + +use axum::{routing::get, Router}; +use sqlx::PgPool; + +const MOUNT_POINT: &str = "/blog"; + +pub struct Blog { + pub db_handler: PgPool, +} + +impl Module for Blog { + fn get_server_router(&self) -> Router { + let db_handler = self.db_handler.clone(); + // todo: remove new in get func? + Router::new() + .route("/get_post_list", get(get_post_list)) + .route("/get_page_count", get(get_page_count)) + .route("/get_post/:post_id", get(get_post)) + .with_state(db_handler) + } + + fn get_mount_point(&self) -> &'static str { + MOUNT_POINT + } + + fn get_cron_tasks(&self) -> Vec> { + let db_handler = self.db_handler.clone(); + vec![ + BasicTask::new(move || check_update::task(&db_handler), "0/15 * * * * *") + .unwrap() + .to_task(), + ] + } +} diff --git a/backend/server/src/modules/mod.rs b/backend/server/src/modules/mod.rs new file mode 100644 index 0000000..dd7e83e --- /dev/null +++ b/backend/server/src/modules/mod.rs @@ -0,0 +1,84 @@ +pub mod blog; + +use std::env; +use std::num::ParseIntError; + +use crate::{ + constants::config::server::{default_value, var_name}, + scheduler::{tasks::CronTask, Scheduler}, +}; +use macros::error_panic; + +use axum::Router; +use tracing::{info, warn}; + +// todo: merge `get_server_router` & `get_mount_point`, make Option +pub trait Module { + fn get_server_router(&self) -> Router; + + fn get_mount_point(&self) -> &'static str; + + fn get_cron_tasks(&self) -> Vec>; +} + +pub struct ModuleTree { + pub modules: Vec>, +} + +impl ModuleTree { + pub fn new() -> Self { + ModuleTree { modules: vec![] } + } + + pub fn add_module(&mut self, module: impl Module + 'static) -> &mut Self { + self.modules.push(Box::new(module)); + self + } + + pub async fn blocking_serve(&self) { + let scheduler = Scheduler::new(); + let mut app = Router::new(); + + let port: u32 = match env::var(var_name::PORT) { + Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { + warn!( + "failed to parse `{}`, using default port: {}", + var_name::PORT, + e.to_string() + ); + default_value::PORT + }), + Err(_) => default_value::PORT, + }; + + let listener = match tokio::net::TcpListener::bind(format!( + "{}:{}", + default_value::HOST, + port + )) + .await + { + Ok(listener) => { + info!("server started on {}:{}", default_value::HOST, port); + listener + } + Err(e) => { + error_panic!( + "failed to start server on {}:{}: {}", + default_value::HOST, + port, + e.to_string() + ); + } + }; + + for module in &self.modules { + let _ = scheduler.insert_list(module.get_cron_tasks()); + app = app.nest(module.get_mount_point(), module.get_server_router()); + } + + axum::serve(listener, app).await.unwrap_or_else(|e| { + error_panic!("failed to start axum: {}", e.to_string()); + }); + } +} diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index 71cb62e..edf5b1a 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -12,7 +12,7 @@ use crate::scheduler::{ SchedulerError, SchedulerError::IdAlreadyRunning, SchedulerError::IdNotFound, }, sleeper::SkippableSleeper, - tasks::Task, + tasks::CronTask, }; use tracing::log::error; @@ -30,7 +30,7 @@ struct TaskStatus { #[derive(Default)] pub struct Scheduler { - tasks: Arc>>>>, + tasks: Arc>>>>, running: Arc>>>, } @@ -49,7 +49,7 @@ impl Scheduler { fn run_task( running: Arc>>>, id: Uuid, - task: Arc>, + task: Arc>, ) -> Result<(), SchedulerError> { if running.read()?.get(&id).is_some() { return Err(IdAlreadyRunning); @@ -159,13 +159,13 @@ impl Scheduler { } } - pub fn insert(&self, task: Box) -> Result { + pub fn insert(&self, task: Box) -> Result { let id = Uuid::new_v4(); self.tasks.write()?.insert(id, Arc::new(task)); Ok(id) } - pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { + pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { tasks.into_iter().map(|t| self.insert(t)).collect() } diff --git a/backend/server/src/scheduler/scheduler_error.rs b/backend/server/src/scheduler/scheduler_error.rs index d9783f7..8961bf6 100644 --- a/backend/server/src/scheduler/scheduler_error.rs +++ b/backend/server/src/scheduler/scheduler_error.rs @@ -6,6 +6,7 @@ use std::{ use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; +// todo: refactor with thiserror pub enum SchedulerError { IdNotFound, IdAlreadyRunning, diff --git a/backend/server/src/scheduler/tasks/basic.rs b/backend/server/src/scheduler/tasks/basic.rs index 24124e3..4c69db7 100644 --- a/backend/server/src/scheduler/tasks/basic.rs +++ b/backend/server/src/scheduler/tasks/basic.rs @@ -1,7 +1,7 @@ use std::error::Error; use std::str::FromStr; -use crate::scheduler::{task_func::TaskFunc, tasks::Task}; +use crate::scheduler::{task_func::TaskFunc, tasks::CronTask}; use chrono::{DateTime, Utc}; use cron::Schedule; @@ -22,12 +22,12 @@ impl BasicTask { }) } - pub fn to_task(self) -> Box { + pub fn to_task(self) -> Box { Box::new(self) } } -impl Task for BasicTask { +impl CronTask for BasicTask { fn get_next_execution(&self) -> Option> { self.schedule.upcoming(Utc).next() } diff --git a/backend/server/src/scheduler/tasks/mod.rs b/backend/server/src/scheduler/tasks/mod.rs index 7e3b87a..8058bf6 100644 --- a/backend/server/src/scheduler/tasks/mod.rs +++ b/backend/server/src/scheduler/tasks/mod.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; pub mod basic; -pub trait Task: Send + Sync { +pub trait CronTask: Send + Sync { fn get_next_execution(&self) -> Option>; fn call(&self); } diff --git a/backend/server/src/utils/git.rs b/backend/server/src/utils/git.rs new file mode 100644 index 0000000..6743058 --- /dev/null +++ b/backend/server/src/utils/git.rs @@ -0,0 +1,195 @@ +/// git merge +/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs +/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 +/// +use git2::{ + build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, +}; +use std::path::PathBuf; +use tracing::{error, warn}; + + +#[derive(Debug, Eq, PartialEq)] +pub struct FileDelta { + pub old_path: Option, + pub new_path: Option, + pub status: Delta, +} + +fn do_fetch<'a>( + repo: &'a Repository, + refs: &[&str], + remote: &'a mut Remote, +) -> Result, git2::Error> { + let mut fetch_option = FetchOptions::new(); + + // Always fetch all tags. + // Perform a download and also update tips + fetch_option.download_tags(AutotagOption::All); + remote.fetch(refs, Some(&mut fetch_option), None)?; + + let fetch_head = repo.find_reference("FETCH_HEAD")?; + repo.reference_to_annotated_commit(&fetch_head) +} + +fn fast_forward( + repo: &Repository, + lb: &mut Reference, + rc: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let name = match lb.name() { + Some(s) => s.to_string(), + None => String::from_utf8_lossy(lb.name_bytes()).to_string(), + }; + let msg = format!("fast-forward({}) -> {}", name, rc.id()); + // println!("{}", msg); + lb.set_target(rc.id(), &msg)?; + repo.set_head(&name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + // For some reason, the force is required to make the working directory actually get updated + // I suspect we should be adding some logic to handle dirty working directory states, + // but this is just an example so maybe not. + .force(), + ))?; + Ok(()) +} + +fn normal_merge( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let ancestor = repo + .find_commit(repo.merge_base(local.id(), remote.id())?)? + .tree()?; + let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?; + + if idx.has_conflicts() { + warn!("merge conflicts detected"); + repo.checkout_index(Some(&mut idx), None)?; + return Ok(()); + } + let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; + // now create the merge commit + let msg = format!("merge({}) -> {}", remote.id(), local.id()); + let sig = repo.signature()?; + let local_commit = repo.find_commit(local.id())?; + let remote_commit = repo.find_commit(remote.id())?; + + // Do our merge commit and set the current branch head to that commit. + let _merge_commit = repo.commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &result_tree, + &[&local_commit, &remote_commit], + )?; + // Set working tree to match head. + repo.checkout_head(None)?; + Ok(()) +} + +fn do_merge<'a>( + repo: &'a Repository, + remote_branch: &str, + fetch_commit: AnnotatedCommit<'a>, +) -> Result, git2::Error> { + // 1. do a merge analysis + let analysis = repo.merge_analysis(&[&fetch_commit])?; + + let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; + + // 2. Do the appropriate merge + if analysis.0.is_fast_forward() { + // println!("Doing a fast-forward"); + // do a fast-forward + let ref_name = format!("refs/heads/{}", remote_branch); + match repo.find_reference(&ref_name) { + Ok(mut r) => { + fast_forward(repo, &mut r, &fetch_commit)?; + } + Err(_) => { + // The branch doesn't exist, so set the reference to the + // commit directly. Usually this is because you are pulling + // into an empty repository. + repo.reference( + &ref_name, + fetch_commit.id(), + true, + &format!("({}) -> {}", remote_branch, fetch_commit.id()), + )?; + repo.set_head(&ref_name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + .allow_conflicts(true) + .conflict_style_merge(true) + .force(), + ))?; + } + }; + } else if analysis.0.is_normal() { + // do a normal merge + normal_merge(repo, &head_commit, &fetch_commit)?; + } + // else { + // // already up-to-date + // println!("Nothing to do..."); + // } + + calculate_diff(repo, &head_commit, &fetch_commit) +} + +fn calculate_diff( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result, git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?; + + let mut updated_files: Vec = Vec::new(); + + if let Err(e) = diff.foreach( + &mut |d, _| { + // let delta = d.status(); + // + // let n_path = d.new_file().path(); + // let o_path = d.old_file().path(); + // + // if let Some(path) = n_path { + // updated_files.insert(path.to_owned()); + // } + // if let Some(path) = o_path { + // updated_files.insert(path.to_owned()); + // } + updated_files.push(FileDelta { + old_path: d.old_file().path().map(PathBuf::from), + new_path: d.new_file().path().map(PathBuf::from), + status: d.status(), + }); + true // Continue + }, + None, + None, + None, + ) { + error!("error at calculating diff: {:?}", e); + } + + Ok(updated_files.into_iter().collect()) +} + +pub fn sync( + remote_name: &str, + remote_branch: &str, + repo: &Repository, +) -> Result, git2::Error> { + let mut remote = repo.find_remote(remote_name)?; + let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; + do_merge(repo, remote_branch, fetch_commit) +} diff --git a/backend/server/src/utils/mod.rs b/backend/server/src/utils/mod.rs new file mode 100644 index 0000000..c2bf1c3 --- /dev/null +++ b/backend/server/src/utils/mod.rs @@ -0,0 +1 @@ +pub mod git;