feat(backend::scheduler) deadlock spotted

backend/server/src/modules/blog/components/check_update/index.rs

@@ -1,12 +1,10 @@
 use std::error::Error;
 use std::path::Path;
 
-use crate::{
-    modules::blog::cron::check_update::utils::get_md_file_basename,
-    modules::blog::cron::check_update::{error::PostIdxError, utils::parse_date},
-};
 use markdown::MdFile;
 
+use crate::modules::blog::components::check_update::utils::get_md_file_basename;
+use crate::modules::blog::components::check_update::{error::PostIdxError, utils::parse_date};
 use sqlx::{query, PgPool, Pool, Postgres, Row};
 use tokio::runtime::Runtime;
 use tracing::{info, warn};
@@ -18,7 +16,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError>
             return Err(PostIdxError::MissingField {
                 field: "title".to_string(),
                 filename: post.filename.clone(),
-            })
+            });
         }
         Some(title) => title.as_str().unwrap_or_else(|| {
             warn!("invalid title for `{}`", &post.filename);
@@ -41,7 +39,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError>
             return Err(PostIdxError::MissingField {
                 field: "category".to_string(),
                 filename: post.filename.clone(),
-            })
+            });
         }
         Some(title) => title.as_str().unwrap_or_else(|| {
             warn!("invalid category for `{}`", &post.filename);
@@ -58,7 +56,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError>
                         err_field: last_update.to_string(),
                         filename: post.filename.clone(),
                         msg: e.to_string(),
-                    })
+                    });
                 }
             }
         } else {
@@ -70,7 +68,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError>
             return Err(PostIdxError::MissingField {
                 field: "date".to_string(),
                 filename: post.filename.clone(),
-            })
+            });
         }
         Some(first_update) => match parse_date(first_update.as_str().unwrap()) {
             Ok(date) => date,
@@ -80,7 +78,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError>
                     err_field: first_update.to_string(),
                     filename: post.filename.clone(),
                     msg: e.to_string(),
-                })
+                });
             }
         },
     };
@@ -137,7 +135,7 @@ async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError>
                 return Err(PostIdxError::MissingField {
                     field: "tags".to_string(),
                     filename: post.filename.clone(),
-                })
+                });
             }
             Some(val) => val,
         };
@@ -173,7 +171,7 @@ async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError>
                         db_table: "Tag".to_string(),
                         id: post_id.to_string(),
                         msg: e.to_string(),
-                    })
+                    });
                 }
             }
         }

backend/server/src/modules/blog/components/check_update/mod.rs

@@ -0,0 +1,78 @@
+use crate::constants::config::general::{default_value, var_name};
+use macros::panic_with_log;
+use std::sync::Arc;
+use std::{
+    env,
+    path::{Path, PathBuf},
+};
+
+use crate::modules::blog::components::check_update::{
+    // index::drop_index,
+    // utils::{expand_path, index_files, search_md},
+    utils::expand_path,
+};
+use crate::scheduler::task_func::AsyncTaskFunc;
+use sqlx::PgPool;
+use tracing::{info, warn, Level};
+
+// mod error;
+// mod index;
+mod utils;
+
+#[derive(Debug)]
+struct Args {
+    git_work_dir: PathBuf,
+    git_url: String,
+    git_remote_name: String,
+    git_remote_branch: String,
+}
+
+pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc {
+    // init
+    let git_work_dir = {
+        let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
+            expand_path(default_log_path)
+        } else {
+            let fallback_path = expand_path(default_value::WORK_DIR.to_string());
+            warn!(
+                "unset env var `{}`, using default path: {}/blog_git",
+                var_name::WORK_DIR,
+                fallback_path
+            );
+            fallback_path
+        };
+        Path::new(&path).join("blog_git")
+    };
+
+    // clone & check update
+
+    let git_url = match env::var(var_name::BLOG_GIT_URL) {
+        Ok(url) => url,
+        Err(e) => {
+            panic_with_log!(
+                Level::ERROR,
+                "failed to read `{}`: {}",
+                var_name::BLOG_GIT_URL,
+                e
+            )
+        }
+    };
+    let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME)
+        .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string());
+    let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH)
+        .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string());
+
+    let arc_args = Arc::new(Args {
+        git_work_dir,
+        git_url,
+        git_remote_name,
+        git_remote_branch,
+    });
+
+    move || Box::pin(test(arc_args.clone()))
+}
+
+async fn test(args: Arc<Args>) {
+    info!("test");
+    dbg!(&args);
+}

backend/server/src/modules/blog/components/check_update/utils.rs

@@ -0,0 +1,81 @@
+use crate::constants::{
+    config::general::{default_value, var_name},
+    time::DEFAULT_DATE_FORMAT,
+};
+
+// use crate::modules::blog::components::check_update::index::index_file;
+use chrono::{DateTime, FixedOffset, ParseResult};
+use dirs::home_dir;
+use tracing::warn;
+
+// pub fn search_md(entry: PathBuf) -> Vec<PathBuf> {
+//     if entry.ends_with(".git") {
+//         return vec![];
+//     }
+//     if entry.is_dir() {
+//         if let Ok(files) = entry.read_dir() {
+//             files
+//                 .flat_map(|file| {
+//                     let file_path = file.unwrap().path();
+//                     search_md(file_path)
+//                 })
+//                 .collect()
+//         } else {
+//             vec![]
+//         }
+//     } else if entry.extension().is_some_and(|ext| ext == "md") {
+//         vec![entry]
+//     } else {
+//         vec![]
+//     }
+// }
+
+// pub fn index_files(files: &[PathBuf], db_conn: &Pool<Postgres>) {
+//     let rt = Runtime::new().unwrap();
+//     rt.block_on(async {
+//         let files_clone = files.to_owned();
+//         let handlers = files_clone
+//             .into_iter()
+//             .map(|file: PathBuf| {
+//                 let conn_clone = db_conn.clone();
+//                 spawn(async move {
+//                     if let Err(e) = index_file(&file, &conn_clone).await {
+//                         error!("failed to index markdown file `{}`: {}", file.display(), e);
+//                     }
+//                 })
+//             })
+//             .collect::<Vec<_>>();
+//
+//         block_on(join_all(handlers));
+//     });
+// }
+
+pub fn expand_path(path: String) -> String {
+    if path.starts_with('~') {
+        home_dir()
+            .map(|dir| {
+                dir.join(path.strip_prefix("~/").unwrap())
+                    .into_os_string()
+                    .into_string()
+                    .unwrap()
+            })
+            .unwrap_or_else(|| {
+                warn!(
+                    "failed to parse env var `{}`, using default value `{}`",
+                    var_name::WORK_DIR,
+                    default_value::WORK_DIR
+                );
+                default_value::WORK_DIR.to_string()
+            })
+    } else {
+        path
+    }
+}
+
+pub fn parse_date(date_str: &str) -> ParseResult<DateTime<FixedOffset>> {
+    DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT)
+}
+
+// pub fn get_md_file_basename(f_name: &str) -> String {
+//     f_name.replace(".md", "")
+// }

backend/server/src/modules/blog/components/mod.rs

@@ -0,0 +1,2 @@
+pub mod check_update;
+mod rag;

backend/server/src/modules/blog/cron/check_update/mod.rs

@@ -1,148 +0,0 @@
-use std::{
-    env,
-    fs::remove_dir_all,
-    path::{Path, PathBuf},
-};
-
-use crate::{
-    constants::config::general::{default_value, var_name},
-    modules::blog::cron::check_update::{
-        index::drop_index,
-        utils::{expand_path, get_md_file_basename, index_files, search_md},
-    },
-    utils::git,
-};
-use macros::panic_with_log;
-
-use git2::{Delta, Repository};
-use sqlx::{PgPool, Pool, Postgres};
-use tracing::{error, info, warn, Level};
-
-mod error;
-mod index;
-mod utils;
-
-pub fn task(db_connection: &PgPool) {
-    // init
-    let git_work_dir = {
-        let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
-            expand_path(default_log_path)
-        } else {
-            let fallback_path = expand_path(default_value::WORK_DIR.to_string());
-            warn!(
-                "unset env var `{}`, using default path: {}/blog_git",
-                var_name::WORK_DIR,
-                fallback_path
-            );
-            fallback_path
-        };
-        Path::new(&path).join("blog_git")
-    };
-
-    // clone & check update
-
-    let git_url = match env::var(var_name::BLOG_GIT_URL) {
-        Ok(url) => url,
-        Err(e) => {
-            panic_with_log!(
-                Level::ERROR,
-                "failed to read `{}`: {}",
-                var_name::BLOG_GIT_URL,
-                e
-            )
-        }
-    };
-    let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME)
-        .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string());
-    let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH)
-        .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string());
-
-    match Repository::open(&git_work_dir) {
-        Ok(repo) => {
-            // prevent git remote url mismatch (clean cache)
-            match repo.find_remote(&git_remote_name) {
-                Ok(remote) => {
-                    if remote.url().is_some_and(|url| url != git_url) {
-                        remove_dir_all(&git_work_dir).unwrap();
-                        warn!("blog content git({}) url mismatch, reinitialized", &git_url);
-                        return;
-                    }
-                }
-                Err(e) => {
-                    remove_dir_all(&git_work_dir).unwrap();
-                    error!(
-                        "unexpected error on indexing blog content git({}), reinitialized: {}",
-                        &git_url, e
-                    );
-                }
-            }
-
-            match git::sync(&git_remote_name, &git_remote_branch, &repo) {
-                Ok(updates) => {
-                    if !updates.is_empty() {
-                        // incremental update
-                        info!(
-                            "blog content git({}) update detected: {:?}",
-                            &git_url, updates
-                        );
-
-                        let mut update_md: Vec<PathBuf> = vec![];
-
-                        for update in updates {
-                            if [Delta::Deleted, Delta::Untracked].contains(&update.status) {
-                                if let Some(path) = &update.old_path {
-                                    if drop_index(
-                                        &get_md_file_basename(
-                                            path.file_name().unwrap().to_str().unwrap(),
-                                        ),
-                                        db_connection,
-                                    )
-                                    .is_ok()
-                                    {
-                                        info!(
-                                            "blog content git({}) file `{}` is detected as deleted/untracked, index dropped",
-                                            &git_url,
-                                            &path.display()
-                                        );
-                                    } else {
-                                        error!(
-                                            "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index",
-                                            &git_url,
-                                            &path.display()
-                                        )
-                                    }
-                                }
-                            } else if let Some(path) = &update.new_path {
-                                if path.components().any(|comp| comp.as_os_str() == "posts")
-                                    && path.extension().is_some_and(|ext| ext == "md")
-                                {
-                                    update_md.push(path.to_path_buf());
-                                }
-                            }
-                        }
-
-                        index_files(&update_md, db_connection);
-                    }
-                }
-                Err(e) => {
-                    error!(
-                        "failed to sync blog content git({}) updates: {}",
-                        &git_url, e
-                    );
-                }
-            }
-        }
-        Err(_) => {
-            match Repository::clone(&git_url, &git_work_dir) {
-                Ok(_repo) => {
-                    // full repo index
-                    let files = search_md(git_work_dir.join("posts"));
-                    index_files(&files, db_connection);
-                }
-                Err(e) => {
-                    error!("failed to clone blog content git({}): {}", &git_url, e);
-                }
-            }
-        }
-    };
-}

backend/server/src/modules/blog/cron/check_update/utils.rs

@@ -1,88 +0,0 @@
-use std::path::PathBuf;
-
-use crate::{
-    constants::{
-        config::general::{default_value, var_name},
-        time::DEFAULT_DATE_FORMAT,
-    },
-    modules::blog::cron::check_update::index::index_file,
-};
-
-use chrono::{DateTime, FixedOffset, ParseResult};
-use dirs::home_dir;
-use futures::{executor::block_on, future::join_all};
-use sqlx::{Pool, Postgres};
-use tokio::{runtime::Runtime, spawn};
-use tracing::{error, warn};
-
-pub fn search_md(entry: PathBuf) -> Vec<PathBuf> {
-    if entry.ends_with(".git") {
-        return vec![];
-    }
-    if entry.is_dir() {
-        if let Ok(files) = entry.read_dir() {
-            files
-                .flat_map(|file| {
-                    let file_path = file.unwrap().path();
-                    search_md(file_path)
-                })
-                .collect()
-        } else {
-            vec![]
-        }
-    } else if entry.extension().is_some_and(|ext| ext == "md") {
-        vec![entry]
-    } else {
-        vec![]
-    }
-}
-
-pub fn index_files(files: &[PathBuf], db_conn: &Pool<Postgres>) {
-    let rt = Runtime::new().unwrap();
-    rt.block_on(async {
-        let files_clone = files.to_owned();
-        let handlers = files_clone
-            .into_iter()
-            .map(|file: PathBuf| {
-                let conn_clone = db_conn.clone();
-                spawn(async move {
-                    if let Err(e) = index_file(&file, &conn_clone).await {
-                        error!("failed to index markdown file `{}`: {}", file.display(), e);
-                    }
-                })
-            })
-            .collect::<Vec<_>>();
-
-        block_on(join_all(handlers));
-    });
-}
-
-pub fn expand_path(path: String) -> String {
-    if path.starts_with('~') {
-        home_dir()
-            .map(|dir| {
-                dir.join(path.strip_prefix("~/").unwrap())
-                    .into_os_string()
-                    .into_string()
-                    .unwrap()
-            })
-            .unwrap_or_else(|| {
-                warn!(
-                    "failed to parse env var `{}`, using default value `{}`",
-                    var_name::WORK_DIR,
-                    default_value::WORK_DIR
-                );
-                default_value::WORK_DIR.to_string()
-            })
-    } else {
-        path
-    }
-}
-
-pub fn parse_date(date_str: &str) -> ParseResult<DateTime<FixedOffset>> {
-    DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT)
-}
-
-pub fn get_md_file_basename(f_name: &str) -> String {
-    f_name.replace(".md", "")
-}

backend/server/src/modules/blog/cron/mod.rs

@@ -1 +0,0 @@
-// pub mod check_update;

backend/server/src/modules/blog/mod.rs

@@ -1,5 +1,5 @@
 mod api;
-mod cron;
+mod components;
 
 use crate::{
     modules::{
@@ -13,9 +13,12 @@ use crate::{
         },
         Module,
     },
-    scheduler::tasks::{async_basic::AsyncBasic, CronTask},
+    scheduler::tasks::CronTask,
 };
 
+use crate::modules::blog::components::check_update;
+use crate::scheduler::tasks::async_basic::AsyncBasic;
+use crate::scheduler::tasks::basic::BasicTask;
 use axum::{routing::get, Router};
 use sqlx::PgPool;
 
@@ -42,12 +45,12 @@ impl Module for Blog {
     }
 
     fn get_cron_tasks(&self) -> Vec<Box<dyn CronTask>> {
-        // let db_handler = self.db_handler.clone();
+        let db_handler = self.db_handler.clone();
         vec![
-            // BasicTask::new(move || check_update::task(&db_handler), "0/15 * * * * *")
-            //     .unwrap()
-            //     .to_task(),
-            AsyncBasic::new(|| Box::pin(async { println!("test") }), "* * * * * *")
+            AsyncBasic::new(check_update::task(db_handler.clone()), "* * * * * *")
+                .unwrap()
+                .to_task(),
+            BasicTask::new(|| println!("test lol"), "* * * * * *")
                 .unwrap()
                 .to_task(),
         ]

backend/server/src/modules/mod.rs

@@ -1,7 +1,6 @@
 pub mod blog;
 
-use std::env;
-use std::num::ParseIntError;
+use std::{env, num::ParseIntError, thread};
 
 use crate::{
     constants::config::server::{default_value, var_name},
@@ -10,6 +9,7 @@ use crate::{
 use macros::error_panic;
 
 use axum::Router;
+use futures::executor::block_on;
 use tokio::net::TcpListener;
 use tracing::{info, warn};
 
@@ -68,11 +68,15 @@ impl ModuleTree {
         };
 
         for module in &self.modules {
-            let _ = scheduler.insert_list(module.get_cron_tasks());
+            let _ = scheduler.insert_list(module.get_cron_tasks()).await;
             app = app.nest(module.get_mount_point(), module.get_server_router());
         }
 
-        scheduler.start().unwrap();
+        let _ = thread::spawn(move || {
+            let fut = scheduler.start();
+            block_on(fut).unwrap()
+        })
+        .join();
 
         axum::serve(listener, app).await.unwrap_or_else(|e| {
             error_panic!("failed to start axum: {}", e.to_string());