feat(blog) add force refresh

backend/src/api/post/refresh.rs

@@ -2,10 +2,11 @@ use std::{env, fs};
 use std::path::Path;
 use std::str::FromStr;
 
-use axum::extract::State;
+use axum::extract::{Query, State};
 use axum::http::StatusCode;
 use axum::Json;
 use futures::{stream, StreamExt};
+use serde::Deserialize;
 use serde_json::{json, Map, Value};
 use sha2::{Digest, Sha256};
 use sqlx::{PgPool, query};
@@ -16,9 +17,18 @@ use crate::api::post::errors::refresh::PostIdxError;
 use crate::api::post::utils::{build_post_idx, post};
 use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL};
 
+#[derive(Deserialize)]
+pub struct Params {
+    /// refresh level
+    level: Option<String>,
+}
+
 pub async fn refresh(
     State(db_connection): State<PgPool>,
+    Query(params): Query<Params>,
 ) -> Result<StatusCode, (StatusCode, Json<Value>)> {
+    let is_force_refresh = params.level.unwrap_or("".to_string()) == "force";
+
     // search post
     let data_repo_dir = match env::var("GIT_WORK_DIR") {
         Ok(value) => value,
@@ -33,6 +43,22 @@ pub async fn refresh(
         }
     };
 
+    if is_force_refresh {
+        if let Err(e) = query("DROP TABLE hash, meta, post, tag;")
+            .execute(&db_connection)
+            .await
+        {
+            error!(
+                "DB query error when deleting table: `hash`, `meta`, `post`, `tag`: {}",
+                e.to_string()
+            );
+            return Err((
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json::from(json!(("error", e.to_string()))),
+            ));
+        }
+    }
+
     // generate post idx
     let md_path = search_md(Path::new(&data_repo_dir));
 
@@ -81,19 +107,30 @@ pub async fn refresh(
         .await
         .unwrap();
 
-    let stream = stream::iter(post_list_with_hash).for_each_concurrent(None, |(post, hash)| {
+    let res: Vec<_> = stream::iter(post_list_with_hash)
+        .map(|(post, hash)| {
             let db_clone = db_connection.clone();
-        async move {
+            async move { build_post_idx(&db_clone, post, hash).await }
-            if let Err(e) = build_post_idx(&db_clone, post, hash).await {
+        })
-                error!("{}", e.to_string());
+        .buffer_unordered(usize::MAX)
-            }
+        .collect()
-        }
+        .await;
-    });
 
-    stream.await;
+    let errs: Vec<_> = res
+        .into_iter()
+        .filter_map(|res| res.err())
+        .map(|e| e.to_string())
+        .collect();
 
+    if !errs.is_empty() {
+        Err((
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json::from(json!({"error": errs})),
+        ))
+    } else {
         Ok(StatusCode::OK)
     }
+}
 
 /// Search a dir, return a list of markdown files
 ///

backend/src/api/post/utils/build_post_idx.rs

@@ -214,6 +214,8 @@ INSERT INTO Hash (post_id, hash) VALUES ($1, $2)
 }
 
 pub async fn meta(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> {
+    let mut err_trails = 5;
+    loop {
         if let Err(e) = query(
             format!(
                 r"
@@ -264,13 +266,19 @@ $$ LANGUAGE plpgsql;
         )
             .execute(db_connection)
             .await {
-        Err(PostIdxError::DBWriteFailure {
+            if err_trails >= 0 {
+                err_trails -= 1;
+                continue
+            } else {
+                return Err(PostIdxError::DBWriteFailure {
                     data_desc: "post meta data".to_string(),
                     db_table: "Meta".to_string(),
                     id: post_id.to_string(),
                     err_msg: e.to_string(),
                 })
+            }
         } else {
-        Ok(())
+            return Ok(())
+        }
     }
 }

backend/src/api/post/utils/post.rs

@@ -63,7 +63,7 @@ pub fn from_path(path: &str) -> Result<Map<String, Value>, Box<dyn Error>> {
         .arg("--format=%cI")
         .arg("--")
         .current_dir(env::var("GIT_WORK_DIR")?)
-        .arg(&path)
+        .arg(path)
         .output()
     {
         Ok(res) => res,