mirror of
https://github.com/Lumither/nocturne.git
synced 2026-10-17 10:26:21 +00:00
feat(cron::update) auto update git repo and update db index
This commit is contained in:
18 files changed
+882
-39
No files matched your search
@@ -1,24 +0,0 @@
|
||||
use sqlx::{Pool, Postgres};
|
||||
|
||||
pub fn task(db_connection: &Pool<Postgres>) {
|
||||
// get dir
|
||||
// let git_work_dir_path = &env::var("GIT_WORK_DIR").unwrap_or_else(|e| {
|
||||
// if let Ok(default_log_path) = env::var("WORK_DIR") {
|
||||
// if default_log_path.starts_with('~') {
|
||||
// let home_path = home_dir().unwrap_or_else(|| panic!("Cannot find home directory"));
|
||||
// home_path
|
||||
// .join(default_log_path.strip_prefix("~/").unwrap())
|
||||
// .into_os_string()
|
||||
// .into_string()
|
||||
// .expect(
|
||||
// "[fatal] failed to parse env `WORK_DIR`, try to use absolute path instead",
|
||||
// )
|
||||
// } else {
|
||||
// default_log_path
|
||||
// }
|
||||
// } else {
|
||||
// panic!("[fatal] failed to parse env `GIT_WORK_DIR`: {}", e);
|
||||
// }
|
||||
// });
|
||||
println!("check update");
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum PostIdxError {
|
||||
#[error("missing essential field `{}` for file `{}`", field, filename)]
|
||||
MissingField { field: String, filename: String },
|
||||
|
||||
#[error(
|
||||
"invalid field `{}` for file `{}`, <{}>: {}",
|
||||
field_name,
|
||||
filename,
|
||||
err_field,
|
||||
msg
|
||||
)]
|
||||
InvalidField {
|
||||
field_name: String,
|
||||
err_field: String,
|
||||
filename: String,
|
||||
msg: String,
|
||||
},
|
||||
|
||||
#[error(
|
||||
"database error on writing `{}` to table `{}`: <{}>: {}",
|
||||
data_desc,
|
||||
db_table,
|
||||
id,
|
||||
msg
|
||||
)]
|
||||
DBWriteFailure {
|
||||
data_desc: String,
|
||||
db_table: String,
|
||||
id: String,
|
||||
msg: String,
|
||||
},
|
||||
|
||||
#[error(
|
||||
"Database error on Reading `{}` from table `{}`: <{}>: {}",
|
||||
data_desc,
|
||||
db_table,
|
||||
id,
|
||||
msg
|
||||
)]
|
||||
DBReadFailure {
|
||||
data_desc: String,
|
||||
db_table: String,
|
||||
id: String,
|
||||
msg: String,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::blog::cron::check_update::utils::get_md_file_basename;
|
||||
use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date};
|
||||
use markdown::MdFile;
|
||||
|
||||
use sqlx::{query, PgPool, Pool, Postgres, Row};
|
||||
use tokio::runtime::Runtime;
|
||||
use tracing::{info, warn};
|
||||
|
||||
async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
|
||||
let post_id = post.file_id;
|
||||
let title = match post.meta.get("title") {
|
||||
None => {
|
||||
return Err(PostIdxError::MissingField {
|
||||
field: "title".to_string(),
|
||||
filename: post.filename.clone(),
|
||||
})
|
||||
}
|
||||
Some(title) => title.as_str().unwrap_or_else(|| {
|
||||
warn!("invalid title for `{}`", &post.filename);
|
||||
"invalid title"
|
||||
}),
|
||||
};
|
||||
let sub_title = post
|
||||
.meta
|
||||
.get("sub_title")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
let summary = post
|
||||
.meta
|
||||
.get("summary")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
let content = post.content.as_str();
|
||||
let category = match post.meta.get("category") {
|
||||
None => {
|
||||
return Err(PostIdxError::MissingField {
|
||||
field: "category".to_string(),
|
||||
filename: post.filename.clone(),
|
||||
})
|
||||
}
|
||||
Some(title) => title.as_str().unwrap_or_else(|| {
|
||||
warn!("invalid category for `{}`", &post.filename);
|
||||
"invalid category"
|
||||
}),
|
||||
};
|
||||
let last_update = {
|
||||
if let Some(last_update) = post.meta.get("last_update") {
|
||||
match parse_date(last_update.as_str().unwrap()) {
|
||||
Ok(date) => Some(date),
|
||||
Err(e) => {
|
||||
return Err(PostIdxError::InvalidField {
|
||||
field_name: "last_update".to_string(),
|
||||
err_field: last_update.to_string(),
|
||||
filename: post.filename.clone(),
|
||||
msg: e.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let first_update = match post.meta.get("date") {
|
||||
None => {
|
||||
return Err(PostIdxError::MissingField {
|
||||
field: "date".to_string(),
|
||||
filename: post.filename.clone(),
|
||||
})
|
||||
}
|
||||
Some(first_update) => match parse_date(first_update.as_str().unwrap()) {
|
||||
Ok(date) => date,
|
||||
Err(e) => {
|
||||
return Err(PostIdxError::InvalidField {
|
||||
field_name: "date".to_string(),
|
||||
err_field: first_update.to_string(),
|
||||
filename: post.filename.clone(),
|
||||
msg: e.to_string(),
|
||||
})
|
||||
}
|
||||
},
|
||||
};
|
||||
let filename = get_md_file_basename(&post.filename);
|
||||
|
||||
if let Err(e) = query(
|
||||
r##"
|
||||
INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (post_id) DO
|
||||
UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
summary = EXCLUDED.summary,
|
||||
content = EXCLUDED.content,
|
||||
last_update = EXCLUDED.last_update,
|
||||
first_update = EXCLUDED.first_update,
|
||||
sub_title = EXCLUDED.sub_title,
|
||||
category = EXCLUDED.category,
|
||||
filename = EXCLUDED.filename;
|
||||
"##,
|
||||
)
|
||||
.bind(post_id)
|
||||
.bind(title)
|
||||
.bind(summary)
|
||||
.bind(content)
|
||||
.bind(last_update)
|
||||
.bind(first_update)
|
||||
.bind(sub_title)
|
||||
.bind(category)
|
||||
.bind(filename)
|
||||
.execute(db_connection)
|
||||
.await
|
||||
{
|
||||
return Err(
|
||||
PostIdxError::DBWriteFailure {
|
||||
data_desc: "post base information".to_string(),
|
||||
db_table: "Post".to_string(),
|
||||
id: post_id.to_string(),
|
||||
msg: e.to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
|
||||
if let Some(tags_string) = post.meta.get("tags").to_owned() {
|
||||
let tags = match tags_string.as_array().map(|vec| {
|
||||
vec.iter()
|
||||
.map(|val| val.as_str().unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
}) {
|
||||
None => {
|
||||
return Err(PostIdxError::MissingField {
|
||||
field: "tags".to_string(),
|
||||
filename: post.filename.clone(),
|
||||
})
|
||||
}
|
||||
Some(val) => val,
|
||||
};
|
||||
|
||||
let post_id = post.file_id;
|
||||
|
||||
// remove old record
|
||||
if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1")
|
||||
.bind(post_id)
|
||||
.execute(db_connection)
|
||||
.await
|
||||
{
|
||||
return Err(PostIdxError::DBWriteFailure {
|
||||
data_desc: "post tag information".to_string(),
|
||||
db_table: "Tag".to_string(),
|
||||
id: post_id.to_string(),
|
||||
msg: e.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// add new record
|
||||
for tag in tags {
|
||||
match query("INSERT INTO Tag VALUES ($1, $2);")
|
||||
.bind(post_id)
|
||||
.bind(tag)
|
||||
.execute(db_connection)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
return Err(PostIdxError::DBWriteFailure {
|
||||
data_desc: "post tag information".to_string(),
|
||||
db_table: "Tag".to_string(),
|
||||
id: post_id.to_string(),
|
||||
msg: e.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn hash(db_connection: &PgPool, post: &MdFile) -> Result<bool, PostIdxError> {
|
||||
let base64_form = post.hash.clone();
|
||||
match query("SELECT hash FROM Hash WHERE post_id = $1")
|
||||
.bind(post.file_id)
|
||||
.fetch_optional(db_connection)
|
||||
.await
|
||||
{
|
||||
Ok(row) => {
|
||||
let db_record: String = match row {
|
||||
None => "".to_string(),
|
||||
Some(row) => row.try_get("hash").unwrap_or("".to_string()),
|
||||
};
|
||||
if base64_form == db_record {
|
||||
Ok(false)
|
||||
} else {
|
||||
if let Err(e) = query(
|
||||
r#"
|
||||
INSERT INTO Hash (post_id, hash) VALUES ($1, $2)
|
||||
ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#,
|
||||
)
|
||||
.bind(post.file_id)
|
||||
.bind(&base64_form)
|
||||
.execute(db_connection)
|
||||
.await
|
||||
{
|
||||
return Err(PostIdxError::DBWriteFailure {
|
||||
data_desc: "post hash".to_string(),
|
||||
db_table: "Hash".to_string(),
|
||||
id: post.file_id.to_string(),
|
||||
msg: e.to_string(),
|
||||
});
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
Err(e) => Err(PostIdxError::DBReadFailure {
|
||||
data_desc: "post hash".to_string(),
|
||||
db_table: "Hash".to_string(),
|
||||
id: post.file_id.to_string(),
|
||||
msg: e.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
|
||||
if let Err(e) = query(
|
||||
// todo: sanitization
|
||||
format!(
|
||||
r#"
|
||||
DO
|
||||
$$
|
||||
DECLARE
|
||||
_post_id UUID := '{}';
|
||||
_meta_cols JSONB := '{}';
|
||||
_exists BOOLEAN;
|
||||
_col_name TEXT;
|
||||
_col_value TEXT;
|
||||
_sql TEXT;
|
||||
_insert_columns TEXT := 'post_id';
|
||||
_insert_values TEXT := quote_literal(_post_id);
|
||||
_update_set TEXT := '';
|
||||
_blacklist TEXT[] := ARRAY['tags'];
|
||||
BEGIN
|
||||
SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists;
|
||||
|
||||
FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols)
|
||||
LOOP
|
||||
IF _col_name = ANY(_blacklist) THEN
|
||||
CONTINUE;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_name = 'meta'
|
||||
AND column_name = _col_name
|
||||
-- may change in the future
|
||||
AND table_schema = 'public') THEN
|
||||
_sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT';
|
||||
EXECUTE _sql;
|
||||
END IF;
|
||||
_insert_columns := _insert_columns || ', ' || quote_ident(_col_name);
|
||||
_insert_values := _insert_values || ', ' || quote_literal(_col_value);
|
||||
_update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value);
|
||||
END LOOP;
|
||||
|
||||
_update_set := substring(_update_set FROM 3);
|
||||
|
||||
IF _exists THEN
|
||||
_sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id);
|
||||
EXECUTE _sql;
|
||||
ELSE
|
||||
_sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')';
|
||||
EXECUTE _sql;
|
||||
END IF;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"#, post.file_id.to_string().as_str(), post.meta
|
||||
).as_str()
|
||||
)
|
||||
.execute(db_connection)
|
||||
.await {
|
||||
Err(PostIdxError::DBWriteFailure {
|
||||
data_desc: "post meta data".to_string(),
|
||||
db_table: "Meta".to_string(),
|
||||
id: post.file_id.to_string(),
|
||||
msg: e.to_string(),
|
||||
})
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn index_file(path: &Path, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
|
||||
let md_file = MdFile::from_file(path)?;
|
||||
|
||||
base(db_conn, &md_file).await?;
|
||||
tag(db_conn, &md_file).await?;
|
||||
meta(db_conn, &md_file).await?;
|
||||
hash(db_conn, &md_file).await?;
|
||||
|
||||
info!("indexed markdown file: {}", path.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn drop_index(file_basename: &str, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
|
||||
let rt = Runtime::new().unwrap();
|
||||
rt.block_on(async {
|
||||
if let Err(e) = query(
|
||||
r#"
|
||||
WITH deleted_post AS (
|
||||
DELETE FROM post
|
||||
WHERE id = (SELECT id FROM post WHERE filename = $1)
|
||||
RETURNING id
|
||||
)
|
||||
DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post);
|
||||
DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post);
|
||||
DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post);
|
||||
"#,
|
||||
)
|
||||
.bind(file_basename)
|
||||
.execute(db_conn)
|
||||
.await
|
||||
{
|
||||
Err(Box::from(
|
||||
format!("failed to drop index: {}", e).to_string(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
use std::env;
|
||||
use std::fs::remove_dir_all;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::{
|
||||
blog::cron::check_update::{
|
||||
index::drop_index,
|
||||
utils::{expand_path, get_md_file_basename, index_files, search_md},
|
||||
},
|
||||
utils::git,
|
||||
};
|
||||
use constants::config::general::{default_value, var_name};
|
||||
use macros::panic_with_log;
|
||||
|
||||
use git2::{Delta, Repository};
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tracing::{error, info, warn, Level};
|
||||
|
||||
mod error;
|
||||
mod index;
|
||||
mod utils;
|
||||
|
||||
pub fn task(db_connection: &Pool<Postgres>) {
|
||||
// init
|
||||
let git_work_dir = {
|
||||
let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
|
||||
expand_path(default_log_path)
|
||||
} else {
|
||||
let fallback_path = expand_path(default_value::WORK_DIR.to_string());
|
||||
warn!(
|
||||
"unset env var `{}`, using default path: {}/blog_git",
|
||||
var_name::WORK_DIR,
|
||||
fallback_path
|
||||
);
|
||||
fallback_path
|
||||
};
|
||||
Path::new(&path).join("blog_git")
|
||||
};
|
||||
|
||||
// clone & check update
|
||||
|
||||
let git_url = match env::var(var_name::BLOG_GIT_URL) {
|
||||
Ok(url) => url,
|
||||
Err(e) => {
|
||||
panic_with_log!(
|
||||
Level::ERROR,
|
||||
"failed to read `{}`: {}",
|
||||
var_name::BLOG_GIT_URL,
|
||||
e
|
||||
)
|
||||
}
|
||||
};
|
||||
let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME)
|
||||
.unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string());
|
||||
let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH)
|
||||
.unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string());
|
||||
|
||||
match Repository::open(&git_work_dir) {
|
||||
Ok(repo) => {
|
||||
// prevent git remote url mismatch (clean cache)
|
||||
match repo.find_remote(&git_remote_name) {
|
||||
Ok(remote) => {
|
||||
if remote.url().is_some_and(|url| url != git_url) {
|
||||
remove_dir_all(&git_work_dir).unwrap();
|
||||
warn!("blog content git({}) url mismatch, reinitialized", &git_url);
|
||||
return;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
remove_dir_all(&git_work_dir).unwrap();
|
||||
error!(
|
||||
"unexpected error on indexing blog content git({}), reinitialized: {}",
|
||||
&git_url, e
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
match git::sync(&git_remote_name, &git_remote_branch, &repo) {
|
||||
Ok(updates) => {
|
||||
if !updates.is_empty() {
|
||||
// incremental update
|
||||
info!(
|
||||
"blog content git({}) update detected: {:?}",
|
||||
&git_url, updates
|
||||
);
|
||||
|
||||
let mut update_md: Vec<PathBuf> = vec![];
|
||||
|
||||
for update in updates {
|
||||
if [Delta::Deleted, Delta::Untracked].contains(&update.status) {
|
||||
if let Some(path) = &update.old_path {
|
||||
if drop_index(
|
||||
&get_md_file_basename(
|
||||
path.file_name().unwrap().to_str().unwrap(),
|
||||
),
|
||||
db_connection,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
info!(
|
||||
"blog content git({}) file `{}` is detected as deleted/untracked, index dropped",
|
||||
&git_url,
|
||||
&path.display()
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
"blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index",
|
||||
&git_url,
|
||||
&path.display()
|
||||
)
|
||||
}
|
||||
}
|
||||
} else if let Some(path) = &update.new_path {
|
||||
if path.extension().is_some_and(|ext| ext == "md") {
|
||||
update_md.push(path.to_path_buf());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
index_files(&update_md, db_connection);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"failed to sync blog content git({}) updates: {}",
|
||||
&git_url, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
match Repository::clone(&git_url, &git_work_dir) {
|
||||
Ok(_repo) => {
|
||||
// full repo index
|
||||
let files = search_md(git_work_dir);
|
||||
index_files(&files, db_connection);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("failed to clone blog content git({}): {}", &git_url, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
use dirs::home_dir;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::blog::cron::check_update::index::index_file;
|
||||
use chrono::{DateTime, FixedOffset, ParseResult};
|
||||
use constants::config::general::{default_value, var_name};
|
||||
use constants::time::DEFAULT_DATE_FORMAT;
|
||||
|
||||
use futures::executor::block_on;
|
||||
use futures::future::join_all;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::runtime::Runtime;
|
||||
use tokio::spawn;
|
||||
use tracing::{error, warn};
|
||||
|
||||
pub fn search_md(entry: PathBuf) -> Vec<PathBuf> {
|
||||
if entry.ends_with(".git") {
|
||||
return vec![];
|
||||
}
|
||||
if entry.is_dir() {
|
||||
if let Ok(files) = entry.read_dir() {
|
||||
files
|
||||
.flat_map(|file| {
|
||||
let file_path = file.unwrap().path();
|
||||
search_md(file_path)
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
vec![]
|
||||
}
|
||||
} else if entry.extension().is_some_and(|ext| ext == "md") {
|
||||
vec![entry]
|
||||
} else {
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index_files(files: &[PathBuf], db_conn: &Pool<Postgres>) {
|
||||
let rt = Runtime::new().unwrap();
|
||||
rt.block_on(async {
|
||||
let files_clone = files.to_owned();
|
||||
let handlers = files_clone
|
||||
.into_iter()
|
||||
.map(|file: PathBuf| {
|
||||
let conn_clone = db_conn.clone();
|
||||
spawn(async move {
|
||||
if let Err(e) = index_file(&file, &conn_clone).await {
|
||||
error!("failed to index markdown file `{}`: {}", file.display(), e);
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
block_on(join_all(handlers));
|
||||
});
|
||||
}
|
||||
|
||||
pub fn expand_path(path: String) -> String {
|
||||
if path.starts_with('~') {
|
||||
home_dir()
|
||||
.map(|dir| {
|
||||
dir.join(path.strip_prefix("~/").unwrap())
|
||||
.into_os_string()
|
||||
.into_string()
|
||||
.unwrap()
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
warn!(
|
||||
"failed to parse env var `{}`, using default value `{}`",
|
||||
var_name::WORK_DIR,
|
||||
default_value::WORK_DIR
|
||||
);
|
||||
default_value::WORK_DIR.to_string()
|
||||
})
|
||||
} else {
|
||||
path
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_date(date_str: &str) -> ParseResult<DateTime<FixedOffset>> {
|
||||
DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT)
|
||||
}
|
||||
|
||||
pub fn get_md_file_basename(f_name: &str) -> String {
|
||||
f_name.replace(".md", "")
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds};
|
||||
use tracing::warn;
|
||||
|
||||
pub fn init() {
|
||||
// git2 option
|
||||
unsafe {
|
||||
if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) {
|
||||
warn!("failed to set git2 server connect timeout: {}", e);
|
||||
}
|
||||
if let Err(e) = set_server_timeout_in_milliseconds(60_000) {
|
||||
warn!("failed to set git2 server timeout: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
mod cron;
|
||||
mod database;
|
||||
pub mod logger;
|
||||
mod misc;
|
||||
mod schema;
|
||||
pub mod server;
|
||||
|
||||
@@ -24,6 +25,8 @@ pub fn load_env() {
|
||||
}
|
||||
|
||||
pub async fn start() {
|
||||
misc::init();
|
||||
|
||||
// connect database
|
||||
let db_pool = match database::init().await {
|
||||
Ok(pool) => {
|
||||
|
||||
@@ -7,11 +7,11 @@ CREATE TABLE IF NOT EXISTS Post
|
||||
post_id UUID PRIMARY KEY,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
category VARCHAR(255) NOT NULL,
|
||||
header_img TEXT,
|
||||
filename TEXT UNIQUE NOT NULL,
|
||||
summary TEXT,
|
||||
sub_title TEXT,
|
||||
content TEXT NOT NULL,
|
||||
last_update TIMESTAMPTZ NOT NULL,
|
||||
last_update TIMESTAMPTZ,
|
||||
first_update TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
"##;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
mod blog;
|
||||
mod init;
|
||||
mod scheduler;
|
||||
mod utils;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
/// git merge
|
||||
/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs
|
||||
/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837
|
||||
///
|
||||
use git2::{
|
||||
build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
use tracing::{error, warn};
|
||||
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
pub struct FileDelta {
|
||||
pub old_path: Option<PathBuf>,
|
||||
pub new_path: Option<PathBuf>,
|
||||
pub status: Delta,
|
||||
}
|
||||
|
||||
fn do_fetch<'a>(
|
||||
repo: &'a Repository,
|
||||
refs: &[&str],
|
||||
remote: &'a mut Remote,
|
||||
) -> Result<AnnotatedCommit<'a>, git2::Error> {
|
||||
let mut fetch_option = FetchOptions::new();
|
||||
|
||||
// Always fetch all tags.
|
||||
// Perform a download and also update tips
|
||||
fetch_option.download_tags(AutotagOption::All);
|
||||
remote.fetch(refs, Some(&mut fetch_option), None)?;
|
||||
|
||||
let fetch_head = repo.find_reference("FETCH_HEAD")?;
|
||||
repo.reference_to_annotated_commit(&fetch_head)
|
||||
}
|
||||
|
||||
fn fast_forward(
|
||||
repo: &Repository,
|
||||
lb: &mut Reference,
|
||||
rc: &AnnotatedCommit,
|
||||
) -> Result<(), git2::Error> {
|
||||
let name = match lb.name() {
|
||||
Some(s) => s.to_string(),
|
||||
None => String::from_utf8_lossy(lb.name_bytes()).to_string(),
|
||||
};
|
||||
let msg = format!("fast-forward({}) -> {}", name, rc.id());
|
||||
// println!("{}", msg);
|
||||
lb.set_target(rc.id(), &msg)?;
|
||||
repo.set_head(&name)?;
|
||||
repo.checkout_head(Some(
|
||||
build::CheckoutBuilder::default()
|
||||
// For some reason, the force is required to make the working directory actually get updated
|
||||
// I suspect we should be adding some logic to handle dirty working directory states,
|
||||
// but this is just an example so maybe not.
|
||||
.force(),
|
||||
))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normal_merge(
|
||||
repo: &Repository,
|
||||
local: &AnnotatedCommit,
|
||||
remote: &AnnotatedCommit,
|
||||
) -> Result<(), git2::Error> {
|
||||
let local_tree = repo.find_commit(local.id())?.tree()?;
|
||||
let remote_tree = repo.find_commit(remote.id())?.tree()?;
|
||||
let ancestor = repo
|
||||
.find_commit(repo.merge_base(local.id(), remote.id())?)?
|
||||
.tree()?;
|
||||
let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?;
|
||||
|
||||
if idx.has_conflicts() {
|
||||
warn!("merge conflicts detected");
|
||||
repo.checkout_index(Some(&mut idx), None)?;
|
||||
return Ok(());
|
||||
}
|
||||
let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?;
|
||||
// now create the merge commit
|
||||
let msg = format!("merge({}) -> {}", remote.id(), local.id());
|
||||
let sig = repo.signature()?;
|
||||
let local_commit = repo.find_commit(local.id())?;
|
||||
let remote_commit = repo.find_commit(remote.id())?;
|
||||
|
||||
// Do our merge commit and set the current branch head to that commit.
|
||||
let _merge_commit = repo.commit(
|
||||
Some("HEAD"),
|
||||
&sig,
|
||||
&sig,
|
||||
&msg,
|
||||
&result_tree,
|
||||
&[&local_commit, &remote_commit],
|
||||
)?;
|
||||
// Set working tree to match head.
|
||||
repo.checkout_head(None)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn do_merge<'a>(
|
||||
repo: &'a Repository,
|
||||
remote_branch: &str,
|
||||
fetch_commit: AnnotatedCommit<'a>,
|
||||
) -> Result<Vec<FileDelta>, git2::Error> {
|
||||
// 1. do a merge analysis
|
||||
let analysis = repo.merge_analysis(&[&fetch_commit])?;
|
||||
|
||||
let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?;
|
||||
|
||||
// 2. Do the appropriate merge
|
||||
if analysis.0.is_fast_forward() {
|
||||
// println!("Doing a fast-forward");
|
||||
// do a fast-forward
|
||||
let ref_name = format!("refs/heads/{}", remote_branch);
|
||||
match repo.find_reference(&ref_name) {
|
||||
Ok(mut r) => {
|
||||
fast_forward(repo, &mut r, &fetch_commit)?;
|
||||
}
|
||||
Err(_) => {
|
||||
// The branch doesn't exist, so set the reference to the
|
||||
// commit directly. Usually this is because you are pulling
|
||||
// into an empty repository.
|
||||
repo.reference(
|
||||
&ref_name,
|
||||
fetch_commit.id(),
|
||||
true,
|
||||
&format!("({}) -> {}", remote_branch, fetch_commit.id()),
|
||||
)?;
|
||||
repo.set_head(&ref_name)?;
|
||||
repo.checkout_head(Some(
|
||||
build::CheckoutBuilder::default()
|
||||
.allow_conflicts(true)
|
||||
.conflict_style_merge(true)
|
||||
.force(),
|
||||
))?;
|
||||
}
|
||||
};
|
||||
} else if analysis.0.is_normal() {
|
||||
// do a normal merge
|
||||
normal_merge(repo, &head_commit, &fetch_commit)?;
|
||||
}
|
||||
// else {
|
||||
// // already up-to-date
|
||||
// println!("Nothing to do...");
|
||||
// }
|
||||
|
||||
calculate_diff(repo, &head_commit, &fetch_commit)
|
||||
}
|
||||
|
||||
fn calculate_diff(
|
||||
repo: &Repository,
|
||||
local: &AnnotatedCommit,
|
||||
remote: &AnnotatedCommit,
|
||||
) -> Result<Vec<FileDelta>, git2::Error> {
|
||||
let local_tree = repo.find_commit(local.id())?.tree()?;
|
||||
let remote_tree = repo.find_commit(remote.id())?.tree()?;
|
||||
let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?;
|
||||
|
||||
let mut updated_files: Vec<FileDelta> = Vec::new();
|
||||
|
||||
if let Err(e) = diff.foreach(
|
||||
&mut |d, _| {
|
||||
// let delta = d.status();
|
||||
//
|
||||
// let n_path = d.new_file().path();
|
||||
// let o_path = d.old_file().path();
|
||||
//
|
||||
// if let Some(path) = n_path {
|
||||
// updated_files.insert(path.to_owned());
|
||||
// }
|
||||
// if let Some(path) = o_path {
|
||||
// updated_files.insert(path.to_owned());
|
||||
// }
|
||||
updated_files.push(FileDelta {
|
||||
old_path: d.old_file().path().map(PathBuf::from),
|
||||
new_path: d.new_file().path().map(PathBuf::from),
|
||||
status: d.status(),
|
||||
});
|
||||
true // Continue
|
||||
},
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
) {
|
||||
error!("error at calculating diff: {:?}", e);
|
||||
}
|
||||
|
||||
Ok(updated_files.into_iter().collect())
|
||||
}
|
||||
|
||||
pub fn sync(
|
||||
remote_name: &str,
|
||||
remote_branch: &str,
|
||||
repo: &Repository,
|
||||
) -> Result<Vec<FileDelta>, git2::Error> {
|
||||
let mut remote = repo.find_remote(remote_name)?;
|
||||
let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?;
|
||||
do_merge(repo, remote_branch, fetch_commit)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod git;
|
||||
Reference in new issue
Block a user