feat(blog::refresh) add hash check to boost idx speed

backend/Cargo.toml

@@ -20,3 +20,6 @@ tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tracing-appender = "0.2.3"
 log = "0.4.21"
+sha2 = "0.10.8"
+thiserror = "1.0.63"
+base64 = "0.22.1"

backend/post.sql

@@ -1,24 +0,0 @@
-CREATE TABLE IF NOT EXISTS Post
-(
-    post_id      UUID PRIMARY KEY,
-    title        VARCHAR(255) NOT NULL,
-    summary      TEXT         NOT NULL,
-    content      TEXT         NOT NULL,
-    last_update  TIMESTAMP,
-    first_update TIMESTAMP    NOT NULL
-);
-
-drop table if exists post;
-
-INSERT INTO Post (post_id, title, summary, content, last_update, first_update)
-VALUES ($1, $2, $3, $4, $5, $6)
-ON CONFLICT (post_id)
-    DO UPDATE SET title        = excluded.title,
-                  summary      = excluded.summary,
-                  content      = excluded.content,
-                  last_update  = excluded.last_update,
-                  first_update = excluded.first_update;
-
-SELECT *
-FROM Post
-WHERE post_id = $1;

backend/src/api/post/errors/mod.rs

@@ -0,0 +1 @@
+pub mod refresh;

backend/src/api/post/errors/refresh.rs

@@ -0,0 +1,61 @@
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum PostIdxError {
+    #[error("Failed to parse UUID <{}>: {}", id, err_msg)]
+    InvalidUUID { id: String, err_msg: String },
+
+    #[error(
+        "Failed to parse `last update time` for <{}> when creating post index: {}",
+        id,
+        err_msg
+    )]
+    InvalidLastUpdate { id: String, err_msg: String },
+
+    #[error(
+        "Failed to parse `first update time` for <{}> when creating post index: {}",
+        id,
+        err_msg
+    )]
+    InvalidFirstUpdate { id: String, err_msg: String },
+
+    #[error(
+        "Failed to parse `tag` for <{}> when creating post index: <{}>: {}",
+        id,
+        invalid_string,
+        err_msg
+    )]
+    InvalidTagString {
+        id: String,
+        invalid_string: String,
+        err_msg: String,
+    },
+
+    #[error(
+        "Database error on writing `{}` to table `{}`: <{}>: {}",
+        data_desc,
+        db_table,
+        id,
+        err_msg
+    )]
+    DBWriteFailure {
+        data_desc: String,
+        db_table: String,
+        id: String,
+        err_msg: String,
+    },
+
+    #[error(
+        "Database error on Reading `{}` to table `{}`: <{}>: {}",
+        data_desc,
+        db_table,
+        id,
+        err_msg
+    )]
+    DBReadFailure {
+        data_desc: String,
+        db_table: String,
+        id: String,
+        err_msg: String,
+    },
+}

backend/src/api/post/mod.rs

@@ -1 +1,3 @@
+mod errors;
 pub mod refresh;
+mod utils;

backend/src/api/post/refresh.rs

@@ -1,53 +1,60 @@
-use std::env;
+use std::{env, fs};
 use std::path::Path;
 use std::str::FromStr;
 
 use axum::extract::State;
 use axum::http::StatusCode;
 use axum::Json;
-use chrono::DateTime;
+use futures::{stream, StreamExt};
 use serde_json::{json, Map, Value};
+use sha2::{Digest, Sha256};
 use sqlx::{PgPool, query};
 use tracing::error;
 use uuid::Uuid;
 
-use crate::constants::GLOBAL_TIME_FORMAT;
+use crate::api::post::errors::refresh::PostIdxError;
-use crate::model::{post, POST_DB_MODEL, TAG_DB_MODEL};
+use crate::api::post::utils::{build_post_idx, post};
+use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL};
 
 pub async fn refresh(
     State(db_connection): State<PgPool>,
 ) -> Result<StatusCode, (StatusCode, Json<Value>)> {
+    // search post
     let data_repo_dir = match env::var("GIT_WORK_DIR") {
         Ok(value) => value,
         Err(_) => {
             return Err((
                 StatusCode::INTERNAL_SERVER_ERROR,
-                Json::from(json!(format!(
+                Json::from(json!((
-                    "{{error: {}}}",
+                    "error",
                     "error at parsing environment variable: GIT_WORK_DIR"
                 ))),
             ));
         }
     };
 
-    let md_list = search_md(Path::new(&data_repo_dir));
+    // generate post idx
+    let md_path = search_md(Path::new(&data_repo_dir));
 
-    // dump old idx
+    let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = md_path
-    query("DROP TABLE IF EXISTS Post;")
-        .execute(&db_connection)
-        .await
-        .unwrap();
-    query("DROP TABLE IF EXISTS Tag;")
-        .execute(&db_connection)
-        .await
-        .unwrap();
-
-    // generate new idx
-
-    let post_list: Vec<Map<_, _>> = md_list
         .into_iter()
         .filter_map(|post_path| match post::from_path(&post_path) {
-            Ok(post) => Some(post),
+            Ok(post) => match fs::read(&post_path) {
+                Ok(bytes) => {
+                    let mut hasher = Sha256::new();
+                    hasher.update(bytes);
+                    let hash = hasher.finalize().to_vec();
+                    Some((post, hash))
+                }
+                Err(e) => {
+                    error!(
+                        "Failed to read file while hashing the post <{}>, skipping: {}",
+                        post_path,
+                        e.to_string()
+                    );
+                    None
+                }
+            },
             Err(e) => {
                 error!(
                     "Failed to parse post <{}>, skipping: {}",
@@ -59,138 +66,44 @@ pub async fn refresh(
         })
         .collect();
 
-    query(POST_DB_MODEL).execute(&db_connection).await.unwrap();
+    // db create
-
+    query(POST_BASE_MODEL)
-    query(TAG_DB_MODEL).execute(&db_connection).await.unwrap();
-
-    // todo: concurrency
-    for post in post_list {
-        let meta = &post["meta"];
-
-        let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) {
-            Ok(id) => id,
-            Err(e) => {
-                error!(
-                    "Failed to parse UUID <{}>: {}",
-                    post["post_id"].as_str().unwrap(),
-                    e.to_string()
-                );
-                continue;
-            }
-        };
-        let title = post["title"].as_str();
-        let summary = post
-            .get("summary")
-            .and_then(|summary| summary.as_str())
-            .unwrap_or("");
-        let content = post["content"].as_str();
-        let last_update = match DateTime::parse_from_str(
-            post["last_update"].as_str().unwrap(),
-            GLOBAL_TIME_FORMAT,
-        ) {
-            Ok(time) => time,
-            Err(e) => {
-                error!(
-                    "Failed to parse last update time for <{}> on db table initialization: {}",
-                    post_id.to_string(),
-                    e.to_string()
-                );
-                continue;
-            }
-        };
-        let first_update = match DateTime::parse_from_str(
-            post["first_update"].as_str().unwrap(),
-            GLOBAL_TIME_FORMAT,
-        ) {
-            Ok(time) => time,
-            Err(e) => {
-                error!(
-                    "Failed to parse first update time for <{}> on db table initialization: {}",
-                    post_id.to_string(),
-                    e.to_string()
-                );
-                continue;
-            }
-        };
-        let sub_title = meta
-            .get("sub_title")
-            .and_then(|sub_title| sub_title.as_str())
-            .unwrap_or("");
-        let category = meta
-            .get("category")
-            .and_then(|category| category.as_str())
-            .unwrap_or("N/A");
-        let header_img = meta
-            .get("header_img")
-            .and_then(|header_img| header_img.as_str())
-            .unwrap_or("");
-
-        // table Post
-        match query(
-            r##"
-INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
-        "##,
-        )
-        .bind(post_id)
-        .bind(title)
-        .bind(summary)
-        .bind(content)
-        .bind(last_update)
-        .bind(first_update)
-        .bind(sub_title)
-        .bind(category)
-        .bind(header_img)
         .execute(&db_connection)
         .await
-        {
+        .unwrap();
-            Ok(_) => {}
+    query(POST_META_MODEL)
-            Err(e) => {
+        .execute(&db_connection)
-                error!("Database error on initializing post data (insert):<{}> : {}", post_id.to_string(), e.to_string());
+        .await
-                return Err((
+        .unwrap();
-                    StatusCode::INTERNAL_SERVER_ERROR,
+    query(TAG_MODEL).execute(&db_connection).await.unwrap();
-                    Json::from(json!(("error:", e.to_string()))),
+    query(POST_HASH_MODEL)
-                ));
+        .execute(&db_connection)
+        .await
+        .unwrap();
+
+    let stream = stream::iter(post_list_with_hash).for_each_concurrent(None, |(post, hash)| {
+        let db_clone = db_connection.clone();
+        async move {
+            if let Err(e) = build_post_idx(&db_clone, post, hash).await {
+                error!("{}", e.to_string());
             }
         }
+    });
 
-        // table Tag
+    stream.await;
-        if let Some(tags_string) = meta.get("tags").to_owned() {
-            // parse tags string
-            let parsed: Value = serde_json::from_str(tags_string.as_str().unwrap()).unwrap();
-            let string_array = parsed.as_array().unwrap();
-            let tags: Vec<String> = string_array
-                .iter()
-                .map(|value| value.as_str().unwrap().to_string())
-                .collect();
-
-            for tag in tags {
-                match query(
-                    r#"
-INSERT INTO Tag
-VALUES ($1, $2); 
-                    "#,
-                )
-                .bind(Uuid::from_str(post["post_id"].as_str().unwrap()).unwrap())
-                .bind(tag)
-                .execute(&db_connection)
-                .await
-                {
-                    Ok(_) => {}
-                    Err(e) => {
-                        return Err((
-                            StatusCode::INTERNAL_SERVER_ERROR,
-                            Json::from(json!(format!("{{error: {}}}", e.to_string()))),
-                        ));
-                    }
-                }
-            }
-        }
-    }
 
     Ok(StatusCode::OK)
 }
 
+/// Search a dir, return a list of markdown files
+///
+/// # Arguments
+/// * `entry`: dir path
+///
+/// # Returns
+/// Vec<String, Global>
+/// * `String`: path of a markdown file
+///
 fn search_md(entry: &Path) -> Vec<String> {
     if entry.ends_with(".git") {
         return vec![];
@@ -212,3 +125,29 @@ fn search_md(entry: &Path) -> Vec<String> {
         vec![]
     }
 }
+
+async fn build_post_idx(
+    db_connection: &PgPool,
+    post: Map<String, Value>,
+    hash: Vec<u8>,
+) -> Result<(), PostIdxError> {
+    let meta = &post["meta"];
+
+    // extract post id
+    let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) {
+        Ok(id) => id,
+        Err(e) => {
+            return Err(PostIdxError::InvalidUUID {
+                id: post["post_id"].as_str().unwrap().to_string(),
+                err_msg: e.to_string(),
+            })
+        }
+    };
+
+    if build_post_idx::hash(db_connection, post_id, hash).await? {
+        build_post_idx::base(db_connection, &post, post_id, meta).await?;
+        build_post_idx::tag(db_connection, post_id, meta).await?;
+    }
+
+    Ok(())
+}

backend/src/api/post/utils/build_post_idx.rs

@@ -0,0 +1,214 @@
+use base64::Engine;
+use base64::engine::general_purpose;
+use chrono::DateTime;
+use serde_json::{Map, Value};
+use sqlx::{PgPool, query, Row};
+use uuid::Uuid;
+
+use crate::api::post::errors::refresh::PostIdxError;
+use crate::constants::GLOBAL_TIME_FORMAT;
+
+pub async fn base(
+    db_connection: &PgPool,
+    post: &Map<String, Value>,
+    post_id: Uuid,
+    meta: &Value,
+) -> Result<(), PostIdxError> {
+    let title = post["title"].as_str();
+    let summary = post
+        .get("summary")
+        .and_then(|summary| summary.as_str())
+        .unwrap_or("");
+    let content = post["content"].as_str();
+    let last_update =
+        match DateTime::parse_from_str(post["last_update"].as_str().unwrap(), GLOBAL_TIME_FORMAT) {
+            Ok(time) => time,
+            Err(e) => {
+                return Err(PostIdxError::InvalidLastUpdate {
+                    id: post_id.to_string(),
+                    err_msg: e.to_string(),
+                });
+            }
+        };
+    let first_update = match DateTime::parse_from_str(
+        post["first_update"].as_str().unwrap(),
+        GLOBAL_TIME_FORMAT,
+    ) {
+        Ok(time) => time,
+        Err(e) => {
+            return Err(PostIdxError::InvalidFirstUpdate {
+                id: post_id.to_string(),
+                err_msg: e.to_string(),
+            })
+        }
+    };
+    let sub_title = meta
+        .get("sub_title")
+        .and_then(|sub_title| sub_title.as_str())
+        .unwrap_or("");
+    let category = meta
+        .get("category")
+        .and_then(|category| category.as_str())
+        .unwrap_or("N/A");
+    let header_img = meta
+        .get("header_img")
+        .and_then(|header_img| header_img.as_str())
+        .unwrap_or("");
+
+    match query(
+        r##"
+INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ON CONFLICT (post_id) DO
+UPDATE SET
+    title = EXCLUDED.title,
+    summary = EXCLUDED.summary,
+    content = EXCLUDED.content,
+    last_update = EXCLUDED.last_update,
+    first_update = EXCLUDED.first_update,
+    sub_title = EXCLUDED.sub_title,
+    category = EXCLUDED.category,
+    header_img = EXCLUDED.header_img;
+        "##,
+    )
+        .bind(post_id)
+        .bind(title)
+        .bind(summary)
+        .bind(content)
+        .bind(last_update)
+        .bind(first_update)
+        .bind(sub_title)
+        .bind(category)
+        .bind(header_img)
+        .execute(db_connection)
+        .await
+    {
+        Ok(_) => {}
+        Err(e) => {
+            return Err(
+                PostIdxError::DBWriteFailure {
+                    data_desc: "post base information".to_string(),
+                    db_table: "Post".to_string(),
+                    id: post_id.to_string(),
+                    err_msg: e.to_string()
+                }
+            )
+        }
+    }
+
+    Ok(())
+}
+
+pub async fn tag(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> {
+    if let Some(tags_string) = meta.get("tags").to_owned() {
+        // parse tags string
+        let parsed: Value = match serde_json::from_str(tags_string.as_str().unwrap()) {
+            Ok(value) => value,
+            Err(e) => {
+                return Err(PostIdxError::InvalidTagString {
+                    id: post_id.to_string(),
+                    invalid_string: tags_string.to_string().to_string(),
+                    err_msg: e.to_string(),
+                })
+            }
+        };
+        let string_array = parsed.as_array().unwrap();
+        let tags: Vec<String> = string_array
+            .iter()
+            .map(|value| value.as_str().unwrap().to_string())
+            .collect();
+
+        // remove old record
+        if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1")
+            .bind(post_id)
+            .execute(db_connection)
+            .await
+        {
+            return Err(PostIdxError::DBWriteFailure {
+                data_desc: "post tag information".to_string(),
+                db_table: "Tag".to_string(),
+                id: post_id.to_string(),
+                err_msg: e.to_string(),
+            });
+        }
+
+        // add new record
+        for tag in tags {
+            match query("INSERT INTO Tag VALUES ($1, $2);")
+                .bind(post_id)
+                .bind(tag)
+                .execute(db_connection)
+                .await
+            {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(PostIdxError::DBWriteFailure {
+                        data_desc: "post tag information".to_string(),
+                        db_table: "Tag".to_string(),
+                        id: post_id.to_string(),
+                        err_msg: e.to_string(),
+                    })
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// # Arguments
+///
+/// * `db_connection`: database connection
+/// * `post_id`: UUID of the post
+/// * `hash`: Hex String
+///
+/// # Returns
+/// Result<bool, PostIdxError>
+/// - `bool`: if the hash is different from database, return true, else false
+pub async fn hash(
+    db_connection: &PgPool,
+    post_id: Uuid,
+    hash: Vec<u8>,
+) -> Result<bool, PostIdxError> {
+    let base64_form = general_purpose::STANDARD.encode(hash);
+    match query("SELECT hash FROM Hash WHERE post_id = $1")
+        .bind(post_id)
+        .fetch_optional(db_connection)
+        .await
+    {
+        Ok(row) => {
+            let db_record: String = match row {
+                None => "".to_string(),
+                Some(row) => row.try_get("hash").unwrap_or("".to_string()),
+            };
+            if base64_form == db_record {
+                Ok(false)
+            } else {
+                if let Err(e) = query(
+                    r#"
+INSERT INTO Hash (post_id, hash) VALUES ($1, $2)
+    ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#,
+                )
+                .bind(post_id)
+                .bind(&base64_form)
+                .execute(db_connection)
+                .await
+                {
+                    return Err(PostIdxError::DBWriteFailure {
+                        data_desc: "post hash".to_string(),
+                        db_table: "Hash".to_string(),
+                        id: post_id.to_string(),
+                        err_msg: e.to_string(),
+                    });
+                }
+                Ok(true)
+            }
+        }
+        Err(e) => Err(PostIdxError::DBReadFailure {
+            data_desc: "post hash".to_string(),
+            db_table: "Hash".to_string(),
+            id: post_id.to_string(),
+            err_msg: e.to_string(),
+        }),
+    }
+}

backend/src/api/post/utils/mod.rs

@@ -0,0 +1,2 @@
+pub mod build_post_idx;
+pub mod post;

backend/src/model/blog/hash.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS Hash
+(
+    post_id UUID PRIMARY KEY,
+    hash    VARCHAR(44)
+);

backend/src/model/blog/meta.sql

@@ -0,0 +1,4 @@
+CREATE TABLE IF NOT EXISTS Meta
+(
+    post_id UUID PRIMARY KEY
+);

backend/src/model/blog/mod.rs

@@ -0,0 +1,38 @@
+pub const POST_BASE_MODEL: &str = r##"
+CREATE TABLE IF NOT EXISTS Post
+(
+    post_id      UUID PRIMARY KEY,
+    title        VARCHAR(255) NOT NULL,
+    category     VARCHAR(255) NOT NULL,
+    header_img   TEXT,
+    summary      TEXT,
+    sub_title    TEXT,
+    content      TEXT         NOT NULL,
+    last_update  TIMESTAMPTZ  NOT NULL,
+    first_update TIMESTAMPTZ  NOT NULL
+);
+"##;
+
+// base mode, will add more cols in runtime
+pub const POST_META_MODEL: &str = r##"
+CREATE TABLE IF NOT EXISTS Meta
+(
+    post_id      UUID PRIMARY KEY
+);
+"##;
+
+pub const POST_HASH_MODEL: &str = r##"
+CREATE TABLE IF NOT EXISTS Hash
+(
+    post_id      UUID PRIMARY KEY,
+    hash         VARCHAR(44)
+);
+"##;
+
+pub const TAG_MODEL: &str = r##"
+CREATE TABLE IF NOT EXISTS Tag
+(
+    post_id UUID         NOT NULL,
+    tag     VARCHAR(255) NOT NULL
+);
+"##;

backend/src/model/blog/post.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS Post
+(
+    post_id      UUID PRIMARY KEY,
+    title        VARCHAR(255) NOT NULL,
+    category     VARCHAR(255) NOT NULL,
+    header_img   TEXT,
+    summary      TEXT,
+    sub_title    TEXT,
+    content      TEXT         NOT NULL,
+    last_update  TIMESTAMPTZ  NOT NULL,
+    first_update TIMESTAMPTZ  NOT NULL
+);

backend/src/model/blog/tag.sql

@@ -3,10 +3,3 @@ CREATE TABLE IF NOT EXISTS Tag
     post_id UUID         NOT NULL,
     tag     VARCHAR(255) NOT NULL
 );
-
-INSERT INTO Tag
-VALUES ($1, $2);
-
-SELECT *
-FROM Tag
-WHERE post_id = $1;

backend/src/model/mod.rs

@@ -1,24 +1 @@
-pub mod post;
+pub mod blog;
-
-pub const POST_DB_MODEL: &str = r##"
-CREATE TABLE IF NOT EXISTS Post
-(
-    post_id      UUID PRIMARY KEY,
-    title        VARCHAR(255) NOT NULL,
-    category     VARCHAR(255) NOT NULL,
-    header_img   TEXT,
-    summary      TEXT,
-    sub_title    TEXT,
-    content      TEXT         NOT NULL,
-    last_update  TIMESTAMPTZ  NOT NULL,
-    first_update TIMESTAMPTZ  NOT NULL
-);
-"##;
-
-pub const TAG_DB_MODEL: &str = r##"
-CREATE TABLE IF NOT EXISTS Tag
-(
-    post_id UUID         NOT NULL,
-    tag     VARCHAR(255) NOT NULL
-);
-"##;