From 0864f0b9506fa07b8fea371cb9837f07d6d9f884 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Sun, 21 Jul 2024 01:12:15 +0800 Subject: [PATCH] feat(blog::refresh) add hash check to boost idx speed --- backend/Cargo.toml | 3 + backend/post.sql | 24 -- backend/src/api/post/errors/mod.rs | 1 + backend/src/api/post/errors/refresh.rs | 61 +++++ backend/src/api/post/mod.rs | 2 + backend/src/api/post/refresh.rs | 227 +++++++----------- backend/src/api/post/utils/build_post_idx.rs | 214 +++++++++++++++++ backend/src/api/post/utils/mod.rs | 2 + backend/src/{model => api/post/utils}/post.rs | 0 backend/src/model/blog/hash.sql | 5 + backend/src/model/blog/meta.sql | 4 + backend/src/model/blog/mod.rs | 38 +++ backend/src/model/blog/post.sql | 12 + backend/{ => src/model/blog}/tag.sql | 7 - backend/src/model/mod.rs | 25 +- 15 files changed, 426 insertions(+), 199 deletions(-) delete mode 100644 backend/post.sql create mode 100644 backend/src/api/post/errors/mod.rs create mode 100644 backend/src/api/post/errors/refresh.rs create mode 100644 backend/src/api/post/utils/build_post_idx.rs create mode 100644 backend/src/api/post/utils/mod.rs rename backend/src/{model => api/post/utils}/post.rs (100%) create mode 100644 backend/src/model/blog/hash.sql create mode 100644 backend/src/model/blog/meta.sql create mode 100644 backend/src/model/blog/mod.rs create mode 100644 backend/src/model/blog/post.sql rename backend/{ => src/model/blog}/tag.sql (58%) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 22d1a90..5d3cbd3 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -20,3 +20,6 @@ tracing = "0.1.40" tracing-subscriber = "0.3.18" tracing-appender = "0.2.3" log = "0.4.21" +sha2 = "0.10.8" +thiserror = "1.0.63" +base64 = "0.22.1" diff --git a/backend/post.sql b/backend/post.sql deleted file mode 100644 index b27b701..0000000 --- a/backend/post.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - summary TEXT NOT NULL, - content TEXT NOT NULL, - last_update TIMESTAMP, - first_update TIMESTAMP NOT NULL -); - -drop table if exists post; - -INSERT INTO Post (post_id, title, summary, content, last_update, first_update) -VALUES ($1, $2, $3, $4, $5, $6) -ON CONFLICT (post_id) - DO UPDATE SET title = excluded.title, - summary = excluded.summary, - content = excluded.content, - last_update = excluded.last_update, - first_update = excluded.first_update; - -SELECT * -FROM Post -WHERE post_id = $1; diff --git a/backend/src/api/post/errors/mod.rs b/backend/src/api/post/errors/mod.rs new file mode 100644 index 0000000..1d2e0f5 --- /dev/null +++ b/backend/src/api/post/errors/mod.rs @@ -0,0 +1 @@ +pub mod refresh; diff --git a/backend/src/api/post/errors/refresh.rs b/backend/src/api/post/errors/refresh.rs new file mode 100644 index 0000000..20a7c70 --- /dev/null +++ b/backend/src/api/post/errors/refresh.rs @@ -0,0 +1,61 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PostIdxError { + #[error("Failed to parse UUID <{}>: {}", id, err_msg)] + InvalidUUID { id: String, err_msg: String }, + + #[error( + "Failed to parse `last update time` for <{}> when creating post index: {}", + id, + err_msg + )] + InvalidLastUpdate { id: String, err_msg: String }, + + #[error( + "Failed to parse `first update time` for <{}> when creating post index: {}", + id, + err_msg + )] + InvalidFirstUpdate { id: String, err_msg: String }, + + #[error( + "Failed to parse `tag` for <{}> when creating post index: <{}>: {}", + id, + invalid_string, + err_msg + )] + InvalidTagString { + id: String, + invalid_string: String, + err_msg: String, + }, + + #[error( + "Database error on writing `{}` to table `{}`: <{}>: {}", + data_desc, + db_table, + id, + err_msg + )] + DBWriteFailure { + data_desc: String, + db_table: String, + id: String, + err_msg: String, + }, + + #[error( + "Database error on Reading `{}` to table `{}`: <{}>: {}", + data_desc, + db_table, + id, + err_msg + )] + DBReadFailure { + data_desc: String, + db_table: String, + id: String, + err_msg: String, + }, +} diff --git a/backend/src/api/post/mod.rs b/backend/src/api/post/mod.rs index 1d2e0f5..1bba1ec 100644 --- a/backend/src/api/post/mod.rs +++ b/backend/src/api/post/mod.rs @@ -1 +1,3 @@ +mod errors; pub mod refresh; +mod utils; diff --git a/backend/src/api/post/refresh.rs b/backend/src/api/post/refresh.rs index f073eb8..5c674af 100644 --- a/backend/src/api/post/refresh.rs +++ b/backend/src/api/post/refresh.rs @@ -1,53 +1,60 @@ -use std::env; +use std::{env, fs}; use std::path::Path; use std::str::FromStr; use axum::extract::State; use axum::http::StatusCode; use axum::Json; -use chrono::DateTime; +use futures::{stream, StreamExt}; use serde_json::{json, Map, Value}; +use sha2::{Digest, Sha256}; use sqlx::{PgPool, query}; use tracing::error; use uuid::Uuid; -use crate::constants::GLOBAL_TIME_FORMAT; -use crate::model::{post, POST_DB_MODEL, TAG_DB_MODEL}; +use crate::api::post::errors::refresh::PostIdxError; +use crate::api::post::utils::{build_post_idx, post}; +use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL}; pub async fn refresh( State(db_connection): State, ) -> Result)> { + // search post let data_repo_dir = match env::var("GIT_WORK_DIR") { Ok(value) => value, Err(_) => { return Err(( StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(format!( - "{{error: {}}}", + Json::from(json!(( + "error", "error at parsing environment variable: GIT_WORK_DIR" ))), )); } }; - let md_list = search_md(Path::new(&data_repo_dir)); + // generate post idx + let md_path = search_md(Path::new(&data_repo_dir)); - // dump old idx - query("DROP TABLE IF EXISTS Post;") - .execute(&db_connection) - .await - .unwrap(); - query("DROP TABLE IF EXISTS Tag;") - .execute(&db_connection) - .await - .unwrap(); - - // generate new idx - - let post_list: Vec> = md_list + let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = md_path .into_iter() .filter_map(|post_path| match post::from_path(&post_path) { - Ok(post) => Some(post), + Ok(post) => match fs::read(&post_path) { + Ok(bytes) => { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let hash = hasher.finalize().to_vec(); + Some((post, hash)) + } + Err(e) => { + error!( + "Failed to read file while hashing the post <{}>, skipping: {}", + post_path, + e.to_string() + ); + None + } + }, Err(e) => { error!( "Failed to parse post <{}>, skipping: {}", @@ -59,138 +66,44 @@ pub async fn refresh( }) .collect(); - query(POST_DB_MODEL).execute(&db_connection).await.unwrap(); - - query(TAG_DB_MODEL).execute(&db_connection).await.unwrap(); - - // todo: concurrency - for post in post_list { - let meta = &post["meta"]; - - let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { - Ok(id) => id, - Err(e) => { - error!( - "Failed to parse UUID <{}>: {}", - post["post_id"].as_str().unwrap(), - e.to_string() - ); - continue; - } - }; - let title = post["title"].as_str(); - let summary = post - .get("summary") - .and_then(|summary| summary.as_str()) - .unwrap_or(""); - let content = post["content"].as_str(); - let last_update = match DateTime::parse_from_str( - post["last_update"].as_str().unwrap(), - GLOBAL_TIME_FORMAT, - ) { - Ok(time) => time, - Err(e) => { - error!( - "Failed to parse last update time for <{}> on db table initialization: {}", - post_id.to_string(), - e.to_string() - ); - continue; - } - }; - let first_update = match DateTime::parse_from_str( - post["first_update"].as_str().unwrap(), - GLOBAL_TIME_FORMAT, - ) { - Ok(time) => time, - Err(e) => { - error!( - "Failed to parse first update time for <{}> on db table initialization: {}", - post_id.to_string(), - e.to_string() - ); - continue; - } - }; - let sub_title = meta - .get("sub_title") - .and_then(|sub_title| sub_title.as_str()) - .unwrap_or(""); - let category = meta - .get("category") - .and_then(|category| category.as_str()) - .unwrap_or("N/A"); - let header_img = meta - .get("header_img") - .and_then(|header_img| header_img.as_str()) - .unwrap_or(""); - - // table Post - match query( - r##" -INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - "##, - ) - .bind(post_id) - .bind(title) - .bind(summary) - .bind(content) - .bind(last_update) - .bind(first_update) - .bind(sub_title) - .bind(category) - .bind(header_img) + // db create + query(POST_BASE_MODEL) .execute(&db_connection) .await - { - Ok(_) => {} - Err(e) => { - error!("Database error on initializing post data (insert):<{}> : {}", post_id.to_string(), e.to_string()); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(("error:", e.to_string()))), - )); + .unwrap(); + query(POST_META_MODEL) + .execute(&db_connection) + .await + .unwrap(); + query(TAG_MODEL).execute(&db_connection).await.unwrap(); + query(POST_HASH_MODEL) + .execute(&db_connection) + .await + .unwrap(); + + let stream = stream::iter(post_list_with_hash).for_each_concurrent(None, |(post, hash)| { + let db_clone = db_connection.clone(); + async move { + if let Err(e) = build_post_idx(&db_clone, post, hash).await { + error!("{}", e.to_string()); } } + }); - // table Tag - if let Some(tags_string) = meta.get("tags").to_owned() { - // parse tags string - let parsed: Value = serde_json::from_str(tags_string.as_str().unwrap()).unwrap(); - let string_array = parsed.as_array().unwrap(); - let tags: Vec = string_array - .iter() - .map(|value| value.as_str().unwrap().to_string()) - .collect(); - - for tag in tags { - match query( - r#" -INSERT INTO Tag -VALUES ($1, $2); - "#, - ) - .bind(Uuid::from_str(post["post_id"].as_str().unwrap()).unwrap()) - .bind(tag) - .execute(&db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(format!("{{error: {}}}", e.to_string()))), - )); - } - } - } - } - } + stream.await; Ok(StatusCode::OK) } +/// Search a dir, return a list of markdown files +/// +/// # Arguments +/// * `entry`: dir path +/// +/// # Returns +/// Vec +/// * `String`: path of a markdown file +/// fn search_md(entry: &Path) -> Vec { if entry.ends_with(".git") { return vec![]; @@ -212,3 +125,29 @@ fn search_md(entry: &Path) -> Vec { vec![] } } + +async fn build_post_idx( + db_connection: &PgPool, + post: Map, + hash: Vec, +) -> Result<(), PostIdxError> { + let meta = &post["meta"]; + + // extract post id + let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) { + Ok(id) => id, + Err(e) => { + return Err(PostIdxError::InvalidUUID { + id: post["post_id"].as_str().unwrap().to_string(), + err_msg: e.to_string(), + }) + } + }; + + if build_post_idx::hash(db_connection, post_id, hash).await? { + build_post_idx::base(db_connection, &post, post_id, meta).await?; + build_post_idx::tag(db_connection, post_id, meta).await?; + } + + Ok(()) +} diff --git a/backend/src/api/post/utils/build_post_idx.rs b/backend/src/api/post/utils/build_post_idx.rs new file mode 100644 index 0000000..426ec63 --- /dev/null +++ b/backend/src/api/post/utils/build_post_idx.rs @@ -0,0 +1,214 @@ +use base64::Engine; +use base64::engine::general_purpose; +use chrono::DateTime; +use serde_json::{Map, Value}; +use sqlx::{PgPool, query, Row}; +use uuid::Uuid; + +use crate::api::post::errors::refresh::PostIdxError; +use crate::constants::GLOBAL_TIME_FORMAT; + +pub async fn base( + db_connection: &PgPool, + post: &Map, + post_id: Uuid, + meta: &Value, +) -> Result<(), PostIdxError> { + let title = post["title"].as_str(); + let summary = post + .get("summary") + .and_then(|summary| summary.as_str()) + .unwrap_or(""); + let content = post["content"].as_str(); + let last_update = + match DateTime::parse_from_str(post["last_update"].as_str().unwrap(), GLOBAL_TIME_FORMAT) { + Ok(time) => time, + Err(e) => { + return Err(PostIdxError::InvalidLastUpdate { + id: post_id.to_string(), + err_msg: e.to_string(), + }); + } + }; + let first_update = match DateTime::parse_from_str( + post["first_update"].as_str().unwrap(), + GLOBAL_TIME_FORMAT, + ) { + Ok(time) => time, + Err(e) => { + return Err(PostIdxError::InvalidFirstUpdate { + id: post_id.to_string(), + err_msg: e.to_string(), + }) + } + }; + let sub_title = meta + .get("sub_title") + .and_then(|sub_title| sub_title.as_str()) + .unwrap_or(""); + let category = meta + .get("category") + .and_then(|category| category.as_str()) + .unwrap_or("N/A"); + let header_img = meta + .get("header_img") + .and_then(|header_img| header_img.as_str()) + .unwrap_or(""); + + match query( + r##" +INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (post_id) DO +UPDATE SET + title = EXCLUDED.title, + summary = EXCLUDED.summary, + content = EXCLUDED.content, + last_update = EXCLUDED.last_update, + first_update = EXCLUDED.first_update, + sub_title = EXCLUDED.sub_title, + category = EXCLUDED.category, + header_img = EXCLUDED.header_img; + "##, + ) + .bind(post_id) + .bind(title) + .bind(summary) + .bind(content) + .bind(last_update) + .bind(first_update) + .bind(sub_title) + .bind(category) + .bind(header_img) + .execute(db_connection) + .await + { + Ok(_) => {} + Err(e) => { + return Err( + PostIdxError::DBWriteFailure { + data_desc: "post base information".to_string(), + db_table: "Post".to_string(), + id: post_id.to_string(), + err_msg: e.to_string() + } + ) + } + } + + Ok(()) +} + +pub async fn tag(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> { + if let Some(tags_string) = meta.get("tags").to_owned() { + // parse tags string + let parsed: Value = match serde_json::from_str(tags_string.as_str().unwrap()) { + Ok(value) => value, + Err(e) => { + return Err(PostIdxError::InvalidTagString { + id: post_id.to_string(), + invalid_string: tags_string.to_string().to_string(), + err_msg: e.to_string(), + }) + } + }; + let string_array = parsed.as_array().unwrap(); + let tags: Vec = string_array + .iter() + .map(|value| value.as_str().unwrap().to_string()) + .collect(); + + // remove old record + if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") + .bind(post_id) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + err_msg: e.to_string(), + }); + } + + // add new record + for tag in tags { + match query("INSERT INTO Tag VALUES ($1, $2);") + .bind(post_id) + .bind(tag) + .execute(db_connection) + .await + { + Ok(_) => {} + Err(e) => { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + err_msg: e.to_string(), + }) + } + } + } + } + + Ok(()) +} + +/// # Arguments +/// +/// * `db_connection`: database connection +/// * `post_id`: UUID of the post +/// * `hash`: Hex String +/// +/// # Returns +/// Result +/// - `bool`: if the hash is different from database, return true, else false +pub async fn hash( + db_connection: &PgPool, + post_id: Uuid, + hash: Vec, +) -> Result { + let base64_form = general_purpose::STANDARD.encode(hash); + match query("SELECT hash FROM Hash WHERE post_id = $1") + .bind(post_id) + .fetch_optional(db_connection) + .await + { + Ok(row) => { + let db_record: String = match row { + None => "".to_string(), + Some(row) => row.try_get("hash").unwrap_or("".to_string()), + }; + if base64_form == db_record { + Ok(false) + } else { + if let Err(e) = query( + r#" +INSERT INTO Hash (post_id, hash) VALUES ($1, $2) + ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, + ) + .bind(post_id) + .bind(&base64_form) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post_id.to_string(), + err_msg: e.to_string(), + }); + } + Ok(true) + } + } + Err(e) => Err(PostIdxError::DBReadFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post_id.to_string(), + err_msg: e.to_string(), + }), + } +} diff --git a/backend/src/api/post/utils/mod.rs b/backend/src/api/post/utils/mod.rs new file mode 100644 index 0000000..faee48e --- /dev/null +++ b/backend/src/api/post/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod build_post_idx; +pub mod post; diff --git a/backend/src/model/post.rs b/backend/src/api/post/utils/post.rs similarity index 100% rename from backend/src/model/post.rs rename to backend/src/api/post/utils/post.rs diff --git a/backend/src/model/blog/hash.sql b/backend/src/model/blog/hash.sql new file mode 100644 index 0000000..7dc91de --- /dev/null +++ b/backend/src/model/blog/hash.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS Hash +( + post_id UUID PRIMARY KEY, + hash VARCHAR(44) +); \ No newline at end of file diff --git a/backend/src/model/blog/meta.sql b/backend/src/model/blog/meta.sql new file mode 100644 index 0000000..1848324 --- /dev/null +++ b/backend/src/model/blog/meta.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS Meta +( + post_id UUID PRIMARY KEY +); diff --git a/backend/src/model/blog/mod.rs b/backend/src/model/blog/mod.rs new file mode 100644 index 0000000..c282af7 --- /dev/null +++ b/backend/src/model/blog/mod.rs @@ -0,0 +1,38 @@ +pub const POST_BASE_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Post +( + post_id UUID PRIMARY KEY, + title VARCHAR(255) NOT NULL, + category VARCHAR(255) NOT NULL, + header_img TEXT, + summary TEXT, + sub_title TEXT, + content TEXT NOT NULL, + last_update TIMESTAMPTZ NOT NULL, + first_update TIMESTAMPTZ NOT NULL +); +"##; + +// base mode, will add more cols in runtime +pub const POST_META_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Meta +( + post_id UUID PRIMARY KEY +); +"##; + +pub const POST_HASH_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Hash +( + post_id UUID PRIMARY KEY, + hash VARCHAR(44) +); +"##; + +pub const TAG_MODEL: &str = r##" +CREATE TABLE IF NOT EXISTS Tag +( + post_id UUID NOT NULL, + tag VARCHAR(255) NOT NULL +); +"##; diff --git a/backend/src/model/blog/post.sql b/backend/src/model/blog/post.sql new file mode 100644 index 0000000..1afc493 --- /dev/null +++ b/backend/src/model/blog/post.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS Post +( + post_id UUID PRIMARY KEY, + title VARCHAR(255) NOT NULL, + category VARCHAR(255) NOT NULL, + header_img TEXT, + summary TEXT, + sub_title TEXT, + content TEXT NOT NULL, + last_update TIMESTAMPTZ NOT NULL, + first_update TIMESTAMPTZ NOT NULL +); diff --git a/backend/tag.sql b/backend/src/model/blog/tag.sql similarity index 58% rename from backend/tag.sql rename to backend/src/model/blog/tag.sql index 80699c4..6c84b76 100644 --- a/backend/tag.sql +++ b/backend/src/model/blog/tag.sql @@ -3,10 +3,3 @@ CREATE TABLE IF NOT EXISTS Tag post_id UUID NOT NULL, tag VARCHAR(255) NOT NULL ); - -INSERT INTO Tag -VALUES ($1, $2); - -SELECT * -FROM Tag -WHERE post_id = $1; diff --git a/backend/src/model/mod.rs b/backend/src/model/mod.rs index b477e57..21aa2c6 100644 --- a/backend/src/model/mod.rs +++ b/backend/src/model/mod.rs @@ -1,24 +1 @@ -pub mod post; - -pub const POST_DB_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - header_img TEXT, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ NOT NULL, - first_update TIMESTAMPTZ NOT NULL -); -"##; - -pub const TAG_DB_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); -"##; +pub mod blog;