lumither/nocturne
1
0 Fork 0
mirror of https://github.com/Lumither/nocturne.git synced 2026-10-17 10:26:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(backend) rebuild start

Browse Source
This commit is contained in:
lumither committed 2025-02-17 15:27:41 -08:00
1 parent 8938c96409
commit acd0525513
45 files changed
+69 -2029
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/Cargo.toml
+1 -2
View File
@@ -1,8 +1,7 @@
[workspace] [workspace]
resolver = "2" resolver = "2"
members = [ members = [
"api_server", "server",
"constants",
"macros", "macros",
"markdown", "markdown",
] ]
backend/api_server/Cargo.toml
-25
View File
@@ -1,25 +0,0 @@
[package]
name = "api_server"
version = "0.1.0"
edition = "2021"
[dependencies]
macros = { path = "../macros" }
constants = { path = "../constants" }
markdown = { path = "../markdown" }
chrono = { version = "0.4.38", features = ["serde", "alloc"] }
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] }
tracing-appender = "0.2.3"
dotenv = "0.15.0"
dirs = "5.0.1"
tokio = { version = "1.40.0", features = ["full"] }
sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] }
axum = { version = "0.7.5", features = ["macros"] }
serde = { version = "1.0.199", features = ["derive"] }
serde_json = "1.0.128"
uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] }
futures = { version = "0.3.30", features = ["executor"] }
cron = "0.13.0"
git2 = "0.19.0"
thiserror = "2.0.3"
backend/api_server/src/blog/api/get_page_count.rs
-26
View File
@@ -1,26 +0,0 @@
use constants::blog::PAGE_SIZE;
use axum::extract::State;
use axum::Json;
use serde_json::{json, Value};
use sqlx::{query, PgPool, Row};
use tracing::error;
pub async fn get_page_count(
State(db_connection): State<PgPool>,
) -> Result<Json<Value>, Json<Value>> {
match query("select cast(ceil(count(*) / $1) as integer) as res from post")
.bind(PAGE_SIZE as f32)
.fetch_one(&db_connection)
.await
{
Ok(res) => {
let res: i32 = res.get("res");
Ok(Json::from(json!({"res": res})))
}
Err(e) => {
error!("Failed to get blog page count: {}", e.to_string());
Err(Json::from(json!({"error": e.to_string()})))
}
}
}
backend/api_server/src/blog/api/get_post.rs
-153
View File
@@ -1,153 +0,0 @@
use axum::{
extract::{Path, State},
http::StatusCode,
Json,
};
use serde_json::{json, Value};
use sqlx::{query, Error, PgPool, Row};
use std::str::FromStr;
use tracing::error;
use uuid::Uuid;
pub async fn get_post(
State(db_connection): State<PgPool>,
Path(post_id): Path<String>,
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let post_id = match Uuid::from_str(post_id.as_str()) {
Ok(id) => id,
Err(e) => {
error!(
"Failed to parse UUID <{}>: {}",
post_id.as_str(),
e.to_string()
);
return Err((
StatusCode::BAD_REQUEST,
Json::from(json!(("error", e.to_string()))),
));
}
};
let post = match query(
r#"
SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post
FROM post t
WHERE post_id = $1;
"#,
)
.bind(post_id)
.fetch_one(&db_connection)
.await
{
Ok(value) => value.get::<Value, _>("post"),
Err(e) => {
error!(
"Database error on Reading `base post information` from table `post`: <{}>: {}",
post_id.to_string(),
e.to_string()
);
return match e {
Error::RowNotFound => Err((
StatusCode::BAD_REQUEST,
Json::from(json!({"error": e.to_string()})),
)),
_ => Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
)),
};
}
};
let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;")
.bind(post_id)
.fetch_all(&db_connection)
.await
{
Ok(tags) => tags.iter().map(|tag| tag.get::<String, _>("tag")).collect(),
Err(e) => {
error!("Database query error :{}", e.to_string());
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
));
}
};
let meta = match query(
r#"
SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta
FROM meta t
WHERE post_id = $1;
"#,
)
.bind(post_id)
.fetch_one(&db_connection)
.await
{
Ok(value) => value.get::<Value, _>("meta"),
Err(e) => {
error!(
"Database error on Reading `{}` from table `{}`: <{}>: {}",
"post meta data",
"meta",
post_id.to_string(),
e.to_string()
);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
));
}
};
let adjacent = match query(r"
WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1),
prev AS (SELECT post_id, t.title, t.sub_title, t.header_img
FROM meta t
WHERE t.date < (SELECT date FROM target_post)
ORDER BY t.date DESC
LIMIT 1),
next AS (SELECT post_id, t.title, t.sub_title, t.header_img
FROM meta t
WHERE t.date > (SELECT date FROM target_post)
ORDER BY t.date
LIMIT 1)
SELECT json_strip_nulls(json_build_object(
'prev',
(SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img)
FROM prev),
'next',
(SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img)
FROM next)
)) AS result;
"
)
.bind(post_id)
.fetch_one(&db_connection)
.await
{
Ok(res) => {res.get::<Value, _>("result")}
Err(e) => {
error!(
"Database error on Reading `{}` from table `{}`: <{}>: {}",
"adjoint posts data",
"meta",
post_id.to_string(),
e.to_string()
);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
));
}
};
let mut res = post.as_object().unwrap().to_owned();
res.insert("tags".to_string(), tags.into());
res.insert("meta".to_string(), meta);
res.insert("adj".to_string(), adjacent);
Ok(Json::from(json!(res)))
}
backend/api_server/src/blog/api/get_post_list.rs
-130
View File
@@ -1,130 +0,0 @@
use std::error::Error;
use constants::blog::PAGE_SIZE;
use axum::{extract::Query, extract::State, http::StatusCode, Json};
use chrono::{DateTime, Utc};
use futures::future::join_all;
use serde::Deserialize;
use serde_json::{json, Map, Value};
use sqlx::{postgres::PgRow, query, PgPool, Row};
use tracing::error;
use uuid::Uuid;
#[derive(Deserialize)]
pub struct Params {
page: Option<u32>,
}
#[axum::debug_handler]
pub async fn get_post_list(
State(db_connection): State<PgPool>,
Query(params): Query<Params>,
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let search_param_page = params.page.unwrap_or(1);
if search_param_page < 1 {
return Err((
StatusCode::BAD_REQUEST,
Json::from(json!({"error": "`page` should be greater than 0"})),
));
}
let offset = PAGE_SIZE * (search_param_page - 1);
let post_list = sqlx::query("SELECT * FROM post ORDER BY first_update DESC LIMIT $1 OFFSET $2")
.bind(PAGE_SIZE as i32)
.bind(offset as i32)
.fetch_all(&db_connection)
.await;
let post_list = match post_list {
Ok(post_list) => post_list,
Err(e) => {
error!(
"Database query error on fetching post list: {}",
e.to_string()
);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
));
}
};
let mut res = Vec::new();
let posts: Vec<_> = post_list
.into_iter()
.map(|post| async { analyze_post(&db_connection, post).await })
.collect();
let posts = join_all(posts).await;
for post in posts {
match post {
Ok(p) => res.push(p),
Err(e) => {
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!(("error", e.to_string()))),
))
}
}
}
// res.insert("posts".to_string(), Value::from(posts));
Ok(Json::from(Value::from(res)))
}
async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result<Value, Box<dyn Error + Send>> {
let post_id: Uuid = post.get("post_id");
let tags = fetch_post_tags(db_connection, post_id).await;
let meta = fetch_post_meta(db_connection, post_id).await.unwrap();
let header_img = meta.get("header_img");
let last_update = meta.get("last_update");
Ok(json!({
"post_id": post_id.to_string(),
"title": post.get::<String, _>("title"),
"category": post.get::<String, _>("category"),
"header_img": header_img,
"sub_title": post.get::<String, _>("sub_title"),
"summary": post.get::<String, _>("summary"),
"last_update": last_update,
"first_update": post.get::<DateTime<Utc>, _>("first_update"),
"tags": tags,
}))
}
async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option<Value> {
match query(
r#"
SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta
FROM meta t
WHERE post_id = $1;
"#,
)
.bind(post_id)
.fetch_one(db_connection)
.await
{
Ok(value) => Some(value.get::<Value, _>("meta")),
Err(e) => {
error!(
"Database error on Reading `{}` from table `{}`: <{}>: {}",
"post meta data",
"meta",
post_id.to_string(),
e.to_string()
);
None
}
}
}
async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec<String> {
match query("SELECT tag FROM Tag WHERE post_id = $1")
.bind(post_id)
.fetch_all(db_connection)
.await
{
Ok(res) => res.iter().map(|tag| tag.get::<String, _>("tag")).collect(),
Err(_) => vec![],
}
}
backend/api_server/src/blog/api/mod.rs
-3
View File
@@ -1,3 +0,0 @@
pub mod get_page_count;
pub mod get_post;
pub mod get_post_list;
backend/api_server/src/blog/cron/check_update/error.rs
-49
View File
@@ -1,49 +0,0 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum PostIdxError {
#[error("missing essential field `{}` for file `{}`", field, filename)]
MissingField { field: String, filename: String },
#[error(
"invalid field `{}` for file `{}`, <{}>: {}",
field_name,
filename,
err_field,
msg
)]
InvalidField {
field_name: String,
err_field: String,
filename: String,
msg: String,
},
#[error(
"database error on writing `{}` to table `{}`: <{}>: {}",
data_desc,
db_table,
id,
msg
)]
DBWriteFailure {
data_desc: String,
db_table: String,
id: String,
msg: String,
},
#[error(
"Database error on Reading `{}` from table `{}`: <{}>: {}",
data_desc,
db_table,
id,
msg
)]
DBReadFailure {
data_desc: String,
db_table: String,
id: String,
msg: String,
},
}
backend/api_server/src/blog/cron/check_update/index.rs
-351
View File
@@ -1,351 +0,0 @@
use std::error::Error;
use std::path::Path;
use crate::blog::cron::check_update::utils::get_md_file_basename;
use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date};
use markdown::MdFile;
use sqlx::{query, PgPool, Pool, Postgres, Row};
use tokio::runtime::Runtime;
use tracing::{info, warn};
async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
let post_id = post.file_id;
let title = match post.meta.get("title") {
None => {
return Err(PostIdxError::MissingField {
field: "title".to_string(),
filename: post.filename.clone(),
})
}
Some(title) => title.as_str().unwrap_or_else(|| {
warn!("invalid title for `{}`", &post.filename);
"invalid title"
}),
};
let sub_title = post
.meta
.get("sub_title")
.and_then(|v| v.as_str())
.unwrap_or("");
let summary = post
.meta
.get("summary")
.and_then(|v| v.as_str())
.unwrap_or("");
let content = post.content.as_str();
let category = match post.meta.get("category") {
None => {
return Err(PostIdxError::MissingField {
field: "category".to_string(),
filename: post.filename.clone(),
})
}
Some(title) => title.as_str().unwrap_or_else(|| {
warn!("invalid category for `{}`", &post.filename);
"invalid category"
}),
};
let last_update = {
if let Some(last_update) = post.meta.get("last_update") {
match parse_date(last_update.as_str().unwrap()) {
Ok(date) => Some(date),
Err(e) => {
return Err(PostIdxError::InvalidField {
field_name: "last_update".to_string(),
err_field: last_update.to_string(),
filename: post.filename.clone(),
msg: e.to_string(),
})
}
}
} else {
None
}
};
let first_update = match post.meta.get("date") {
None => {
return Err(PostIdxError::MissingField {
field: "date".to_string(),
filename: post.filename.clone(),
})
}
Some(first_update) => match parse_date(first_update.as_str().unwrap()) {
Ok(date) => date,
Err(e) => {
return Err(PostIdxError::InvalidField {
field_name: "date".to_string(),
err_field: first_update.to_string(),
filename: post.filename.clone(),
msg: e.to_string(),
})
}
},
};
let filename = get_md_file_basename(&post.filename);
if let Err(e) = query(
r##"
INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (post_id) DO
UPDATE SET
title = EXCLUDED.title,
summary = EXCLUDED.summary,
content = EXCLUDED.content,
last_update = EXCLUDED.last_update,
first_update = EXCLUDED.first_update,
sub_title = EXCLUDED.sub_title,
category = EXCLUDED.category,
filename = EXCLUDED.filename;
"##,
)
.bind(post_id)
.bind(title)
.bind(summary)
.bind(content)
.bind(last_update)
.bind(first_update)
.bind(sub_title)
.bind(category)
.bind(filename)
.execute(db_connection)
.await
{
return Err(
PostIdxError::DBWriteFailure {
data_desc: "post base information".to_string(),
db_table: "Post".to_string(),
id: post_id.to_string(),
msg: e.to_string(),
}
);
}
Ok(())
}
async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
if let Some(tags_string) = post.meta.get("tags").to_owned() {
let tags = match tags_string.as_array().map(|vec| {
vec.iter()
.map(|val| val.as_str().unwrap())
.collect::<Vec<_>>()
}) {
None => {
return Err(PostIdxError::MissingField {
field: "tags".to_string(),
filename: post.filename.clone(),
})
}
Some(val) => val,
};
let post_id = post.file_id;
// remove old record
if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1")
.bind(post_id)
.execute(db_connection)
.await
{
return Err(PostIdxError::DBWriteFailure {
data_desc: "post tag information".to_string(),
db_table: "Tag".to_string(),
id: post_id.to_string(),
msg: e.to_string(),
});
}
// add new record
for tag in tags {
match query("INSERT INTO Tag VALUES ($1, $2);")
.bind(post_id)
.bind(tag)
.execute(db_connection)
.await
{
Ok(_) => {}
Err(e) => {
return Err(PostIdxError::DBWriteFailure {
data_desc: "post tag information".to_string(),
db_table: "Tag".to_string(),
id: post_id.to_string(),
msg: e.to_string(),
})
}
}
}
}
Ok(())
}
async fn hash(db_connection: &PgPool, post: &MdFile) -> Result<bool, PostIdxError> {
let base64_form = post.hash.clone();
match query("SELECT hash FROM Hash WHERE post_id = $1")
.bind(post.file_id)
.fetch_optional(db_connection)
.await
{
Ok(row) => {
let db_record: String = match row {
None => "".to_string(),
Some(row) => row.try_get("hash").unwrap_or("".to_string()),
};
if base64_form == db_record {
Ok(false)
} else {
if let Err(e) = query(
r#"
INSERT INTO Hash (post_id, hash) VALUES ($1, $2)
ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#,
)
.bind(post.file_id)
.bind(&base64_form)
.execute(db_connection)
.await
{
return Err(PostIdxError::DBWriteFailure {
data_desc: "post hash".to_string(),
db_table: "Hash".to_string(),
id: post.file_id.to_string(),
msg: e.to_string(),
});
}
Ok(true)
}
}
Err(e) => Err(PostIdxError::DBReadFailure {
data_desc: "post hash".to_string(),
db_table: "Hash".to_string(),
id: post.file_id.to_string(),
msg: e.to_string(),
}),
}
}
async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
if let Err(e) = query(
// todo: sanitization
format!(
r#"
DO
$$
DECLARE
_post_id UUID := '{}';
_meta_cols JSONB := '{}';
_exists BOOLEAN;
_col_name TEXT;
_col_value TEXT;
_sql TEXT;
_insert_columns TEXT := 'post_id';
_insert_values TEXT := quote_literal(_post_id);
_update_set TEXT := '';
_blacklist TEXT[] := ARRAY['tags'];
BEGIN
SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists;
FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols)
LOOP
IF _col_name = ANY(_blacklist) THEN
CONTINUE;
END IF;
IF NOT EXISTS (SELECT 1
FROM information_schema.columns
WHERE table_name = 'meta'
AND column_name = _col_name
-- may change in the future
AND table_schema = 'public') THEN
_sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT';
EXECUTE _sql;
END IF;
_insert_columns := _insert_columns || ', ' || quote_ident(_col_name);
_insert_values := _insert_values || ', ' || quote_literal(_col_value);
_update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value);
END LOOP;
_update_set := substring(_update_set FROM 3);
IF _exists THEN
_sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id);
EXECUTE _sql;
ELSE
_sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')';
EXECUTE _sql;
END IF;
END;
$$ LANGUAGE plpgsql;
"#, post.file_id.to_string().as_str(), post.meta
).as_str()
)
.execute(db_connection)
.await {
Err(PostIdxError::DBWriteFailure {
data_desc: "post meta data".to_string(),
db_table: "Meta".to_string(),
id: post.file_id.to_string(),
msg: e.to_string(),
})
} else {
Ok(())
}
}
pub async fn index_file(path: &Path, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
let md_file = MdFile::from_file(path)?;
async fn trail(db_conn: &Pool<Postgres>, md_file: &MdFile) -> Result<(), PostIdxError> {
base(db_conn, md_file).await?;
tag(db_conn, md_file).await?;
meta(db_conn, md_file).await?;
hash(db_conn, md_file).await.map(|_| ())
}
let mut cnt = 5;
while let Err(e) = trail(db_conn, &md_file).await {
match e {
PostIdxError::DBWriteFailure { .. } => {
cnt -= 1;
}
PostIdxError::DBReadFailure { .. } => {
cnt -= 1;
}
_ => return Err(e.into()),
}
if cnt <= 0 {
return Err(e.into());
}
}
info!("indexed markdown file: {}", path.display());
Ok(())
}
pub fn drop_index(file_basename: &str, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
let rt = Runtime::new().unwrap();
rt.block_on(async {
if let Err(e) = query(
r#"
WITH deleted_post AS (
DELETE FROM post
WHERE id = (SELECT id FROM post WHERE filename = $1)
RETURNING id
)
DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post);
DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post);
DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post);
"#,
)
.bind(file_basename)
.execute(db_conn)
.await
{
Err(Box::from(
format!("failed to drop index: {}", e).to_string(),
))
} else {
Ok(())
}
})
}
backend/api_server/src/blog/cron/check_update/mod.rs
-146
View File
@@ -1,146 +0,0 @@
use std::env;
use std::error::Error;
use std::fs::remove_dir_all;
use std::path::{Path, PathBuf};
use crate::{
blog::cron::check_update::{
index::drop_index,
utils::{expand_path, get_md_file_basename, index_files, search_md},
},
utils::git,
};
use constants::config::general::{default_value, var_name};
use macros::panic_with_log;
use git2::{Delta, Repository};
use sqlx::{Pool, Postgres};
use tracing::{error, info, warn, Level};
mod error;
mod index;
mod utils;
pub fn task(db_connection: &Pool<Postgres>) {
// init
let git_work_dir = {
let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
expand_path(default_log_path)
} else {
let fallback_path = expand_path(default_value::WORK_DIR.to_string());
warn!(
"unset env var `{}`, using default path: {}/blog_git",
var_name::WORK_DIR,
fallback_path
);
fallback_path
};
Path::new(&path).join("blog_git")
};
// clone & check update
let git_url = match env::var(var_name::BLOG_GIT_URL) {
Ok(url) => url,
Err(e) => {
panic_with_log!(
Level::ERROR,
"failed to read `{}`: {}",
var_name::BLOG_GIT_URL,
e
)
}
};
let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME)
.unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string());
let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH)
.unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string());
match Repository::open(&git_work_dir) {
Ok(repo) => {
// prevent git remote url mismatch (clean cache)
match repo.find_remote(&git_remote_name) {
Ok(remote) => {
if remote.url().is_some_and(|url| url != git_url) {
remove_dir_all(&git_work_dir).unwrap();
warn!("blog content git({}) url mismatch, reinitialized", &git_url);
return;
}
}
Err(e) => {
remove_dir_all(&git_work_dir).unwrap();
error!(
"unexpected error on indexing blog content git({}), reinitialized: {}",
&git_url, e
);
}
}
match git::sync(&git_remote_name, &git_remote_branch, &repo) {
Ok(updates) => {
if !updates.is_empty() {
// incremental update
info!(
"blog content git({}) update detected: {:?}",
&git_url, updates
);
let mut update_md: Vec<PathBuf> = vec![];
for update in updates {
if [Delta::Deleted, Delta::Untracked].contains(&update.status) {
if let Some(path) = &update.old_path {
match drop_index(
&get_md_file_basename(
path.file_name().unwrap().to_str().unwrap(),
),
db_connection,
) {
Ok(_) => info!(
"blog content git({}) file `{}` is detected as deleted/untracked, index dropped",
&git_url,
&path.display()
),
Err(e) => error!(
"blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index: {}",
&git_url,
&path.display(),
e
)
}
}
} else if let Some(path) = &update.new_path {
if path.components().any(|comp| comp.as_os_str() == "posts")
&& path.extension().is_some_and(|ext| ext == "md")
{
// update_md.push(path.to_path_buf());
update_md.push(dbg!(path.to_path_buf()));
}
}
}
index_files(&update_md, db_connection);
}
}
Err(e) => {
error!(
"failed to sync blog content git({}) updates: {}",
&git_url, e
);
}
}
}
Err(_) => {
match Repository::clone(&git_url, &git_work_dir) {
Ok(_repo) => {
// full repo index
let files = search_md(git_work_dir.join("posts"));
index_files(&files, db_connection);
}
Err(e) => {
error!("failed to clone blog content git({}): {}", &git_url, e);
}
}
}
};
}
backend/api_server/src/blog/cron/check_update/utils.rs
-86
View File
@@ -1,86 +0,0 @@
use dirs::home_dir;
use std::path::PathBuf;
use crate::blog::cron::check_update::index::index_file;
use chrono::{DateTime, FixedOffset, ParseResult};
use constants::config::general::{default_value, var_name};
use constants::time::DEFAULT_DATE_FORMAT;
use futures::executor::block_on;
use futures::future::join_all;
use sqlx::{Pool, Postgres};
use tokio::runtime::Runtime;
use tokio::spawn;
use tracing::{error, warn};
pub fn search_md(entry: PathBuf) -> Vec<PathBuf> {
if entry.ends_with(".git") {
return vec![];
}
if entry.is_dir() {
if let Ok(files) = entry.read_dir() {
files
.flat_map(|file| {
let file_path = file.unwrap().path();
search_md(file_path)
})
.collect()
} else {
vec![]
}
} else if entry.extension().is_some_and(|ext| ext == "md") {
vec![entry]
} else {
vec![]
}
}
pub fn index_files(files: &[PathBuf], db_conn: &Pool<Postgres>) {
let rt = Runtime::new().unwrap();
rt.block_on(async {
let files_clone = dbg!(files).to_owned();
let handlers = files_clone
.into_iter()
.map(|file: PathBuf| {
let conn_clone = db_conn.clone();
spawn(async move {
if let Err(e) = index_file(&file, &conn_clone).await {
error!("failed to index markdown file `{}`: {}", file.display(), e);
}
})
})
.collect::<Vec<_>>();
block_on(join_all(handlers));
});
}
pub fn expand_path(path: String) -> String {
if path.starts_with('~') {
home_dir()
.map(|dir| {
dir.join(path.strip_prefix("~/").unwrap())
.into_os_string()
.into_string()
.unwrap()
})
.unwrap_or_else(|| {
warn!(
"failed to parse env var `{}`, using default value `{}`",
var_name::WORK_DIR,
default_value::WORK_DIR
);
default_value::WORK_DIR.to_string()
})
} else {
path
}
}
pub fn parse_date(date_str: &str) -> ParseResult<DateTime<FixedOffset>> {
DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT)
}
pub fn get_md_file_basename(f_name: &str) -> String {
f_name.replace(".md", "")
}
backend/api_server/src/blog/cron/mod.rs
-15
View File
@@ -1,15 +0,0 @@
use crate::scheduler::tasks::basic::BasicTask;
use crate::scheduler::tasks::Task;
use sqlx::{Pool, Postgres};
pub mod check_update;
pub fn get_tasks(db_connection: Pool<Postgres>) -> Vec<Box<dyn Task>> {
let db_conn_clone = db_connection.clone();
vec![
// BasicTask::new(move || check_update::task(&db_conn_clone), "0/15 * * * * *")
BasicTask::new(move || check_update::task(&db_conn_clone), "* * * * * *")
.unwrap()
.to_task(),
]
}
backend/api_server/src/blog/mod.rs
-17
View File
@@ -1,17 +0,0 @@
mod api;
pub mod cron;
use crate::blog::api::{
get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list,
};
use axum::routing::get;
use axum::Router;
use sqlx::{Pool, Postgres};
pub fn get_router() -> Router<Pool<Postgres>> {
Router::new()
.route("/get_post_list", get(get_post_list))
.route("/get_page_count", get(get_page_count))
.route("/get_post/:post_id", get(get_post))
}
backend/api_server/src/init/cron.rs
-12
View File
@@ -1,12 +0,0 @@
use crate::blog;
use crate::scheduler::Scheduler;
use sqlx::{Pool, Postgres};
pub fn start(db_pool: Pool<Postgres>) {
let scheduler = Scheduler::new();
let _ = vec![blog::cron::get_tasks(db_pool.clone())]
.into_iter()
.map(|list| scheduler.insert_list(list))
.collect::<Vec<_>>();
scheduler.start().unwrap()
}
backend/api_server/src/init/database.rs
-41
View File
@@ -1,41 +0,0 @@
use std::env;
use std::error::Error;
use std::str::FromStr;
use constants::config::database::{default_value, var_name};
use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
use sqlx::{ConnectOptions, Pool, Postgres};
use tracing::info;
pub async fn init() -> Result<Pool<Postgres>, Box<dyn Error>> {
let db_connect_option = PgConnectOptions::from_str(&parse_db_uri())
.unwrap()
.disable_statement_logging();
Ok(PgPoolOptions::new()
.max_connections(1) // todo: temporal solution, to be fixed
.connect_with(db_connect_option)
.await?)
}
fn parse_db_uri() -> String {
env::var(var_name::URI).unwrap_or_else(|_| {
info!(
"`{}` env var is not set, \
using `{}`, `{}`, `{}`, `{}` and `{}`",
var_name::URI,
var_name::USERNAME,
var_name::PASSWORD,
var_name::HOST,
var_name::PORT,
var_name::DBNAME
);
let postgres_user = env::var(var_name::USERNAME).unwrap_or(default_value::USERNAME.into());
let postgres_password =
env::var(var_name::PASSWORD).unwrap_or(default_value::PASSWORD.into());
let db_host = env::var(var_name::HOST).unwrap_or(default_value::HOST.into());
let db_port = env::var(var_name::PORT).unwrap_or(default_value::PORT.into());
let postgres_db = env::var(var_name::DBNAME).unwrap_or(default_value::DBNAME.into());
let db_type = default_value::DATABASE_TYPE;
format!("{db_type}://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}")
})
}
backend/api_server/src/init/logger.rs
-106
View File
@@ -1,106 +0,0 @@
use std::path::Path;
use std::{env, io};
use chrono::Local;
use constants::config::general::{default_value, var_name};
use dirs::home_dir;
use tracing::info;
use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling};
use tracing_subscriber::{
filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter,
layer::SubscriberExt, Layer, Registry,
};
struct LocalTimer;
impl FormatTime for LocalTimer {
fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result {
write!(
w,
"{}",
Local::now().format(constants::time::DEFAULT_TIME_FORMAT)
)
}
}
type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static;
pub fn init() -> (WorkerGuard, WorkerGuard) {
let log_path = match env::var(var_name::LOG_DIR) {
Ok(path) => Path::new(&path).to_path_buf(),
Err(_) => {
let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
expand_home_path(default_log_path)
} else {
let fallback_path = expand_home_path(default_value::WORK_DIR.to_string());
println!(
"[warn] unset env var `{}` and `{}`, trying to log on fallback path: {}/log",
var_name::LOG_DIR,
var_name::WORK_DIR,
fallback_path
);
fallback_path
};
Path::new(&path).join("log")
}
};
let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log"));
let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log"));
let info_format = tracing_subscriber::fmt::format()
.with_level(true)
.with_target(true)
.with_source_location(true)
.with_ansi(false);
let err_format = tracing_subscriber::fmt::format()
.with_level(true)
.with_target(true)
.with_source_location(true)
.with_ansi(false);
let stdout_format = tracing_subscriber::fmt::format()
.with_level(true)
.with_target(true)
.with_source_location(true)
.with_timer(LocalTimer);
let subscribers = Registry::default()
.with(
fmt::Layer::default()
.with_writer(info_log_writer)
.event_format(info_format),
)
.with(
fmt::Layer::default()
.with_writer(err_log_writer)
.event_format(err_format)
.with_filter(LevelFilter::ERROR),
)
.with(
fmt::Layer::default()
.with_writer(io::stdout)
.with_ansi(true)
.event_format(stdout_format),
);
tracing::subscriber::set_global_default(subscribers)
.expect("[fatal] failed to setup logging system");
info!("logging to dir: {}", log_path.to_str().unwrap());
(info_guard, error_guard)
}
fn expand_home_path(path: String) -> String {
if path.starts_with('~') {
let home_path = home_dir()
.unwrap_or_else(|| panic!("[fatal] unable to locate home dir when parsing env `{}`, try to use absolute path instead", var_name::WORK_DIR));
home_path
.join(path.strip_prefix("~/").unwrap())
.into_os_string()
.into_string()
.unwrap()
} else {
path
}
}
backend/api_server/src/init/misc.rs
-14
View File
@@ -1,14 +0,0 @@
use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds};
use tracing::warn;
pub fn init() {
// git2 option
unsafe {
if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) {
warn!("failed to set git2 server connect timeout: {}", e);
}
if let Err(e) = set_server_timeout_in_milliseconds(60_000) {
warn!("failed to set git2 server timeout: {}", e);
}
}
}
backend/api_server/src/init/mod.rs
-58
View File
@@ -1,58 +0,0 @@
mod cron;
mod database;
pub mod logger;
mod misc;
mod schema;
pub mod server;
use macros::panic_with_log;
use std::{env, panic};
use tracing::{info, Level};
pub fn load_env() {
if let Some(env_file) = env::args().nth(1) {
match dotenv::from_filename(&env_file) {
Ok(_) => {
println!("[info] env loaded from {}, starting up", env_file);
}
Err(e) => {
panic!("[fatal] failed to read {}: {}", env_file, e);
}
}
} else {
println!("[info] no .env file referred, starting up")
}
}
pub async fn start() {
misc::init();
// connect database
let db_pool = match database::init().await {
Ok(pool) => {
info!("database connection pool initialized");
pool
}
Err(e) => {
panic_with_log!(
Level::ERROR,
"database connection pool failed to initialize: {}",
e
);
}
};
// schema init
schema::init(&db_pool).await;
// load cron task
cron::start(db_pool.clone());
// api serve
match server::start(db_pool).await {
Ok(_) => {}
Err(e) => {
panic_with_log!(Level::ERROR, "server panicked: {}", e)
}
}
}
backend/api_server/src/init/schema.rs
-49
View File
@@ -1,49 +0,0 @@
use sqlx::{query, Pool, Postgres};
use tracing::info;
const POST_BASE_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Post
(
post_id UUID PRIMARY KEY,
title VARCHAR(255) NOT NULL,
category VARCHAR(255) NOT NULL,
filename TEXT UNIQUE NOT NULL,
summary TEXT,
sub_title TEXT,
content TEXT NOT NULL,
last_update TIMESTAMPTZ,
first_update TIMESTAMPTZ NOT NULL
);
"##;
// base model, will add more cols at runtime
const POST_META_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Meta
(
post_id UUID PRIMARY KEY
);
"##;
const POST_HASH_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Hash
(
post_id UUID PRIMARY KEY,
hash VARCHAR(44)
);
"##;
const TAG_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Tag
(
post_id UUID NOT NULL,
tag VARCHAR(255) NOT NULL
);
"##;
pub async fn init(db_connection: &Pool<Postgres>) {
query(POST_BASE_MODEL).execute(db_connection).await.unwrap();
query(POST_META_MODEL).execute(db_connection).await.unwrap();
query(TAG_MODEL).execute(db_connection).await.unwrap();
query(POST_HASH_MODEL).execute(db_connection).await.unwrap();
info!("database schema initialized");
}
backend/api_server/src/init/server.rs
-52
View File
@@ -1,52 +0,0 @@
use std::env;
use std::error::Error;
use std::num::ParseIntError;
use std::process::exit;
use crate::blog;
use axum::Router;
use constants::config::server::{default_value, var_name};
use macros::panic_with_log;
use sqlx::{Pool, Postgres};
use tracing::{error, info, warn, Level};
pub async fn start(db_pool: Pool<Postgres>) -> Result<(), Box<dyn Error>> {
let app = Router::new()
.nest("/blog", blog::get_router())
.with_state(db_pool);
let port: u32 = match env::var(var_name::PORT) {
Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| {
warn!(
"failed to parse `{}`, using default port: {}",
var_name::PORT,
e.to_string()
);
default_value::PORT
}),
Err(_) => default_value::PORT,
};
let listener =
match tokio::net::TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await {
Ok(listener) => {
info!("server started on {}:{}", default_value::HOST, port);
listener
}
Err(e) => {
panic_with_log!(
Level::ERROR,
"failed to start server on {}:{}: {}",
default_value::HOST,
port,
e.to_string()
);
}
};
axum::serve(listener, app).await.unwrap_or_else(|e| {
error!("failed to start axum: {}", e.to_string());
exit(1);
});
Ok(())
}
backend/api_server/src/main.rs
-13
View File
@@ -1,13 +0,0 @@
mod blog;
mod init;
mod scheduler;
mod utils;
#[tokio::main]
async fn main() {
init::load_env();
let _guards = init::logger::init();
init::start().await;
}
backend/api_server/src/scheduler/mod.rs
-194
View File
@@ -1,194 +0,0 @@
use crate::{
scheduler::scheduler_error::SchedulerError,
scheduler::scheduler_error::SchedulerError::IdAlreadyRunning,
scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper,
scheduler::tasks::Task,
};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc, thread};
use tracing::log::error;
use uuid::Uuid;
pub mod scheduler_error;
mod sleeper;
mod task_func;
pub mod tasks;
struct TaskStatus {
sleeper: Option<Arc<SkippableSleeper>>,
is_alive: AtomicBool,
}
#[derive(Default)]
pub struct Scheduler {
tasks: Arc<RwLock<HashMap<Uuid, Arc<Box<dyn Task>>>>>,
running: Arc<RwLock<HashMap<Uuid, Arc<TaskStatus>>>>,
}
impl Scheduler {
/// Start all tasks in the scheduler
pub fn start(&self) -> Result<(), SchedulerError> {
for (k, _v) in self.tasks.read()?.iter() {
self.run_non_blocking(k)?
}
Ok(())
}
/// Terminate all tasks in the scheduler
pub fn halt() {}
fn run_task(
running: Arc<RwLock<HashMap<Uuid, Arc<TaskStatus>>>>,
id: Uuid,
task: Arc<Box<dyn Task>>,
) -> Result<(), SchedulerError> {
if running.read()?.get(&id).is_some() {
return Err(IdAlreadyRunning);
} else {
running.write()?.insert(
id,
TaskStatus {
sleeper: None,
is_alive: AtomicBool::new(true),
}
.into(),
);
}
loop {
if let Some(next_execution) = task.get_next_execution() {
// guarantee is alive
let task_status = {
let running_reading_guard = running.read()?;
running_reading_guard.get(&id).cloned().clone()
};
if let Some(task_status) = task_status {
if !&task_status.is_alive.load(Ordering::SeqCst) {
{
let mut running_lock_guard = running.write()?;
running_lock_guard.remove(&id);
}
return Ok(());
} else {
let sleeper = Arc::from(SkippableSleeper::from(next_execution));
{
let mut running_lock_guard = running.write()?;
running_lock_guard.insert(
id,
TaskStatus {
sleeper: Some(sleeper.clone()),
is_alive: AtomicBool::new(true),
}
.into(),
);
}
sleeper.start();
task.call();
}
}
} else {
error!("unexpected error ");
break;
}
}
Ok(())
}
/// Run a task with `id`
pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> {
let id = id.to_owned();
if let Some(task) = self.tasks.read()?.get(&id) {
Self::run_task(self.running.clone(), id, task.clone())
} else {
Err(IdNotFound)
}
}
pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> {
let id = id.to_owned();
if let Some(task) = self.tasks.read()?.get(&id) {
let running_clone = self.running.clone();
let task_clone = task.clone();
thread::spawn(move || Self::run_task(running_clone, id, task_clone));
Ok(())
} else {
Err(IdNotFound)
}
}
fn get_running(&self, id: &Uuid) -> Option<Arc<TaskStatus>> {
let ret = self.running.read().unwrap().get(id).cloned();
ret
}
/// Stop a task with `id`
pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> {
match self.get_running(id) {
None => Err(IdNotFound),
Some(status) => {
status.is_alive.store(false, Ordering::SeqCst);
if let Some(s) = &status.sleeper {
s.skip();
} else {
return Err(IdAlreadyRunning);
}
Ok(())
}
}
}
pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> {
match self.get_running(id) {
None => Err(IdNotFound),
Some(status) => {
if let Some(s) = &status.sleeper {
s.skip()
} else {
return Err(IdAlreadyRunning);
}
Ok(())
}
}
}
pub fn insert(&self, task: Box<dyn Task>) -> Result<Uuid, SchedulerError> {
let id = Uuid::new_v4();
self.tasks.write()?.insert(id, Arc::new(task));
Ok(id)
}
pub fn insert_list(&self, tasks: Vec<Box<dyn Task>>) -> Result<Vec<Uuid>, SchedulerError> {
tasks.into_iter().map(|t| self.insert(t)).collect()
}
pub fn new() -> Self {
Self::default()
}
}
#[cfg(test)]
mod tests {
use crate::scheduler::tasks::basic::BasicTask;
use crate::scheduler::Scheduler;
use std::error::Error;
use std::thread;
use std::time::Duration;
#[test]
fn test_run() -> Result<(), Box<dyn Error>> {
let scheduler = Scheduler::default();
let task = BasicTask::new(|| println!("test print"), "* * * * * *")?;
let id = scheduler.insert(Box::new(task))?;
for i in 1..=2 {
println!("Loop {} start", i);
println!("Run {}", i);
scheduler.run_non_blocking(&id)?;
thread::sleep(Duration::from_secs(4));
scheduler.stop(&id)?;
println!("Stop {}", i);
thread::sleep(Duration::from_secs(2));
println!("Loop {} finished", i);
}
Ok(())
}
}
backend/api_server/src/scheduler/scheduler_error.rs
-44
View File
@@ -1,44 +0,0 @@
use crate::scheduler::scheduler_error::SchedulerError::MutexPoison;
use std::error::Error;
use std::fmt::{Debug, Display, Formatter};
use std::sync::PoisonError;
pub enum SchedulerError {
IdNotFound,
IdAlreadyRunning,
MutexPoison(String),
}
impl<T> From<PoisonError<T>> for SchedulerError {
fn from(value: PoisonError<T>) -> Self {
MutexPoison(value.to_string())
}
}
impl Debug for SchedulerError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self)
}
}
impl Display for SchedulerError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
match self {
SchedulerError::IdNotFound => {
"SchedulerError::IdNotFound".to_string()
}
SchedulerError::IdAlreadyRunning => {
"SchedulerError::IdAlreadyRunning".to_string()
}
MutexPoison(str) => {
format!("SchedulerError::MutexPoison: {}", str)
}
}
)
}
}
impl Error for SchedulerError {}
backend/api_server/src/scheduler/sleeper.rs
-77
View File
@@ -1,77 +0,0 @@
use chrono::{DateTime, Utc};
use std::sync::{Arc, Condvar, Mutex};
use std::thread;
use std::time::Duration;
#[derive(Clone, Debug, Default)]
pub struct SkippableSleeper {
pub duration: Duration,
state: Arc<(Mutex<bool>, Condvar)>,
}
impl SkippableSleeper {
pub fn new(duration: Duration) -> Self {
SkippableSleeper {
duration,
state: Arc::new((Mutex::new(false), Condvar::new())),
}
}
// Blocks until the duration has passed or skip() is called
pub fn start(&self) {
let (lock, cvar) = &*self.state;
let mut received = lock.lock().unwrap();
let state_clone = Arc::clone(&self.state);
let duration_clone = self.duration;
thread::spawn(move || {
thread::sleep(duration_clone);
let (lock, cvar) = &*state_clone;
let mut received = lock.lock().unwrap();
*received = true;
cvar.notify_one();
});
while !*received {
received = cvar.wait(received).unwrap();
}
}
// Skip the blocking delay
pub fn skip(&self) {
let (lock, cvar) = &*self.state;
let mut received = lock.lock().unwrap();
*received = true;
cvar.notify_one();
}
}
impl From<DateTime<Utc>> for SkippableSleeper {
fn from(value: DateTime<Utc>) -> Self {
let from_utc_timestamp = value.timestamp() as u64;
let curr_utc_timestamp = Utc::now().timestamp() as u64;
SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp))
}
}
#[cfg(test)]
mod test {
use crate::scheduler::sleeper::SkippableSleeper;
use std::thread;
use std::time::Duration;
#[test]
fn usage() {
let sleeper = SkippableSleeper::new(Duration::new(10, 0));
thread::spawn({
let sleeper = sleeper.clone();
move || {
println!("Sleeping...");
sleeper.start();
println!("Woke up!");
}
});
println!("Start Sleeping...");
thread::sleep(Duration::from_secs(4));
println!("Skipping sleep...");
sleeper.skip();
}
}
backend/api_server/src/scheduler/task_func.rs
-3
View File
@@ -1,3 +0,0 @@
pub trait TaskFunc: Fn() + Send + Sync + 'static {}
impl<T> TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {}
backend/api_server/src/scheduler/tasks/basic.rs
-37
View File
@@ -1,37 +0,0 @@
use crate::scheduler::task_func::TaskFunc;
use crate::scheduler::tasks::Task;
use chrono::{DateTime, Utc};
use cron::Schedule;
use std::error::Error;
use std::str::FromStr;
pub struct BasicTask {
schedule: Schedule,
func: Box<dyn TaskFunc>,
}
impl BasicTask {
pub fn new<F>(func: F, cron_expr: &str) -> Result<Self, Box<dyn Error>>
where
F: TaskFunc,
{
Ok(BasicTask {
schedule: Schedule::from_str(cron_expr)?,
func: Box::new(func),
})
}
pub fn to_task(self) -> Box<dyn Task> {
Box::new(self)
}
}
impl Task for BasicTask {
fn get_next_execution(&self) -> Option<DateTime<Utc>> {
self.schedule.upcoming(Utc).next()
}
fn call(&self) {
(self.func)()
}
}
backend/api_server/src/scheduler/tasks/mod.rs
-8
View File
@@ -1,8 +0,0 @@
use chrono::{DateTime, Utc};
pub mod basic;
pub trait Task: Send + Sync {
fn get_next_execution(&self) -> Option<DateTime<Utc>>;
fn call(&self);
}
backend/api_server/src/utils/git.rs
-195
View File
@@ -1,195 +0,0 @@
/// git merge
/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs
/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837
///
use git2::{
build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository,
};
use std::path::PathBuf;
use tracing::{error, warn};
#[derive(Debug, Eq, PartialEq)]
pub struct FileDelta {
pub old_path: Option<PathBuf>,
pub new_path: Option<PathBuf>,
pub status: Delta,
}
fn do_fetch<'a>(
repo: &'a Repository,
refs: &[&str],
remote: &'a mut Remote,
) -> Result<AnnotatedCommit<'a>, git2::Error> {
let mut fetch_option = FetchOptions::new();
// Always fetch all tags.
// Perform a download and also update tips
fetch_option.download_tags(AutotagOption::All);
remote.fetch(refs, Some(&mut fetch_option), None)?;
let fetch_head = repo.find_reference("FETCH_HEAD")?;
repo.reference_to_annotated_commit(&fetch_head)
}
fn fast_forward(
repo: &Repository,
lb: &mut Reference,
rc: &AnnotatedCommit,
) -> Result<(), git2::Error> {
let name = match lb.name() {
Some(s) => s.to_string(),
None => String::from_utf8_lossy(lb.name_bytes()).to_string(),
};
let msg = format!("fast-forward({}) -> {}", name, rc.id());
// println!("{}", msg);
lb.set_target(rc.id(), &msg)?;
repo.set_head(&name)?;
repo.checkout_head(Some(
build::CheckoutBuilder::default()
// For some reason, the force is required to make the working directory actually get updated
// I suspect we should be adding some logic to handle dirty working directory states,
// but this is just an example so maybe not.
.force(),
))?;
Ok(())
}
fn normal_merge(
repo: &Repository,
local: &AnnotatedCommit,
remote: &AnnotatedCommit,
) -> Result<(), git2::Error> {
let local_tree = repo.find_commit(local.id())?.tree()?;
let remote_tree = repo.find_commit(remote.id())?.tree()?;
let ancestor = repo
.find_commit(repo.merge_base(local.id(), remote.id())?)?
.tree()?;
let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?;
if idx.has_conflicts() {
warn!("merge conflicts detected");
repo.checkout_index(Some(&mut idx), None)?;
return Ok(());
}
let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?;
// now create the merge commit
let msg = format!("merge({}) -> {}", remote.id(), local.id());
let sig = repo.signature()?;
let local_commit = repo.find_commit(local.id())?;
let remote_commit = repo.find_commit(remote.id())?;
// Do our merge commit and set the current branch head to that commit.
let _merge_commit = repo.commit(
Some("HEAD"),
&sig,
&sig,
&msg,
&result_tree,
&[&local_commit, &remote_commit],
)?;
// Set working tree to match head.
repo.checkout_head(None)?;
Ok(())
}
fn do_merge<'a>(
repo: &'a Repository,
remote_branch: &str,
fetch_commit: AnnotatedCommit<'a>,
) -> Result<Vec<FileDelta>, git2::Error> {
// 1. do a merge analysis
let analysis = repo.merge_analysis(&[&fetch_commit])?;
let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?;
// 2. Do the appropriate merge
if analysis.0.is_fast_forward() {
// println!("Doing a fast-forward");
// do a fast-forward
let ref_name = format!("refs/heads/{}", remote_branch);
match repo.find_reference(&ref_name) {
Ok(mut r) => {
fast_forward(repo, &mut r, &fetch_commit)?;
}
Err(_) => {
// The branch doesn't exist, so set the reference to the
// commit directly. Usually this is because you are pulling
// into an empty repository.
repo.reference(
&ref_name,
fetch_commit.id(),
true,
&format!("({}) -> {}", remote_branch, fetch_commit.id()),
)?;
repo.set_head(&ref_name)?;
repo.checkout_head(Some(
build::CheckoutBuilder::default()
.allow_conflicts(true)
.conflict_style_merge(true)
.force(),
))?;
}
};
} else if analysis.0.is_normal() {
// do a normal merge
normal_merge(repo, &head_commit, &fetch_commit)?;
}
// else {
// // already up-to-date
// println!("Nothing to do...");
// }
calculate_diff(repo, &head_commit, &fetch_commit)
}
fn calculate_diff(
repo: &Repository,
local: &AnnotatedCommit,
remote: &AnnotatedCommit,
) -> Result<Vec<FileDelta>, git2::Error> {
let local_tree = repo.find_commit(local.id())?.tree()?;
let remote_tree = repo.find_commit(remote.id())?.tree()?;
let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?;
let mut updated_files: Vec<FileDelta> = Vec::new();
if let Err(e) = diff.foreach(
&mut |d, _| {
// let delta = d.status();
//
// let n_path = d.new_file().path();
// let o_path = d.old_file().path();
//
// if let Some(path) = n_path {
// updated_files.insert(path.to_owned());
// }
// if let Some(path) = o_path {
// updated_files.insert(path.to_owned());
// }
updated_files.push(FileDelta {
old_path: d.old_file().path().map(PathBuf::from),
new_path: d.new_file().path().map(PathBuf::from),
status: d.status(),
});
true // Continue
},
None,
None,
None,
) {
error!("error at calculating diff: {:?}", e);
}
Ok(updated_files.into_iter().collect())
}
pub fn sync(
remote_name: &str,
remote_branch: &str,
repo: &Repository,
) -> Result<Vec<FileDelta>, git2::Error> {
let mut remote = repo.find_remote(remote_name)?;
let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?;
do_merge(repo, remote_branch, fetch_commit)
}
backend/api_server/src/utils/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod git;
backend/constants/Cargo.toml
-6
View File
@@ -1,6 +0,0 @@
[package]
name = "constants"
version = "0.1.0"
edition = "2021"
[dependencies]
backend/macros/src/log.rs
+20 -1
View File
@@ -6,7 +6,7 @@
/// use macros::panic_with_log; /// use macros::panic_with_log;
/// use tracing::Level; /// use tracing::Level;
/// ///
/// panic_with_log!(Level::ERROR, "Test panicked: {}", 42); /// panic_with_log!(Level::ERROR, "test panicked: {}", 42);
/// ``` /// ```
#[macro_export] #[macro_export]
macro_rules! panic_with_log { macro_rules! panic_with_log {
@@ -18,6 +18,25 @@ macro_rules! panic_with_log {
}}; }};
} }
/// log the message with `Level::ERROR` and exit(1)
///
/// # Example
/// ```rust
///
/// use macros::error_panic;
///
/// error_panic!("test panicked: {}", 42);
/// ```
#[macro_export]
macro_rules! error_panic {
($msg:literal $(, $($arg:tt)*)?) => {{
use std::process;
use tracing::{event, Level};
event!(Level::ERROR, $msg $(, $($arg)*)?);
process::exit(1);
}};
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test] #[test]
backend/server/schema.sql
+32
View File
@@ -0,0 +1,32 @@
CREATE TABLE tag
(
id SERIAL PRIMARY KEY,
name VARCHAR(20) UNIQUE NOT NULL,
description TEXT
);
CREATE TABLE category
(
id SERIAL PRIMARY KEY,
name VARCHAR(20) UNIQUE NOT NULL,
description TEXT
);
CREATE TABLE post
(
id SERIAL PRIMARY KEY,
identifier VARCHAR(50) UNIQUE NOT NULL,
title VARCHAR(255) NOT NULL,
subtitle VARCHAR(255) NOT NULL,
published_date DATE NOT NULL,
last_update DATE,
content TEXT NOT NULL,
category SERIAL REFERENCES category (id) NOT NULL
);
CREATE TABLE post_tag
(
post SERIAL REFERENCES post (id),
tag SERIAL REFERENCES tag (id)
)
backend/constants/src/blog.rs → backend/server/src/constants/blog.rs
View File
File renamed without changes.
backend/constants/src/config.rs → backend/server/src/constants/config.rs
View File
File renamed without changes.
backend/constants/src/lib.rs → backend/server/src/constants/mod.rs
View File
File renamed without changes.
backend/constants/src/time.rs → backend/server/src/constants/time.rs
View File
File renamed without changes.
backend/server/src/main.rs
+7
View File
@@ -0,0 +1,7 @@
mod constants;
const BUILD_ID: &str = env!("BUILD_ID");
fn main() {
println!("{}", BUILD_ID);
}
loadbalancer/.gitignore
+3
View File
@@ -0,0 +1,3 @@
target/
Cargo.lock
openresty/conf/nocturne.conf → nocturne.conf
+6 -13
View File
@@ -1,24 +1,17 @@
# server {
# listen 80;
# server_name lumither.com;
# return 301 https://$host$request_uri;
# }
server { server {
listen 80; listen 80;
# listen 443 ssl; listen 443 ssl;
# server_name lumither.com; server_name lumither.com;
# ssl_certificate certs/cert.pem; # ssl_certificate certs/cert.pem;
# ssl_certificate_key certs/key.pem; # ssl_certificate_key certs/key.pem;
# ssl_session_cache shared:SSL:1m; ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 5m; ssl_session_timeout 5m;
# ssl_ciphers HIGH:!aNULL:!MD5; ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on; ssl_prefer_server_ciphers on;
location / { location / {
default_type text/html; default_type text/html;
openresty/deploy.sh
-4
View File
@@ -1,4 +0,0 @@
#!/bin/bash
rm -rf /opt/openresty/nginx/lua
cp -r ./lua /opt/openresty/nginx
openresty/lua/hello_world.lua
-1
View File
@@ -1 +0,0 @@
ngx.say("hello world")
scripts/dev_server.sh
-14
View File
@@ -1,14 +0,0 @@
#!/bin/bash
# refresh env
/bin/bash ./flush_env.sh
# start frontend
cd frontend || exit
screen -S nocturne_frontend -d -m bun dev
cd ..
# start backend
cd backend || exit
screen -S nocturne_backend -d -m cargo r
cd ..
scripts/flush_env.sh
-6
View File
@@ -1,6 +0,0 @@
#!/bin/env bash
cp .env ./backend
cp .env ./frontend
cp .env.local ./backend
cp .env.local ./frontend
scripts/refresh.sh
-42
View File
@@ -1,42 +0,0 @@
#!/bin/bash
source .env
if [ -z "$BLOG_ENDPOINT" ]; then
echo "BLOG_ENDPOINT is not set in the .env file"
exit 1
fi
if [ -z "$GIT_WORK_DIR" ]; then
echo "GIT_WORK_DIR is not set in the .env file"
exit 1
fi
if [ -z "$BACKEND_PORT" ]; then
echo "BACKEND_PORT is not set in the .env file"
exit 1
fi
check_for_updates() {
cd "$GIT_WORK_DIR" || exit
res=$(git pull)
if echo "$res" | grep -q "Already up to date"; then
echo "Repo is up to date."
else
echo "Repo has updated; calling db update"
curl --request POST -sL \
--url "localhost:$BACKEND_PORT"
#todo: to be continued
fi
}
if [ -d "$GIT_WORK_DIR" ]; then
check_for_updates
else
git clone "$BLOG_ENDPOINT" "$GIT_WORK_DIR"
if [ $? -eq 0 ]; then
check_for_updates
else
echo "Failed to clone the repo"
fi
fi
scripts/setup.sh
-15
View File
@@ -1,15 +0,0 @@
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
. "$DIR/.env"
sudo groupadd nocturne
USERNAME="$(whoami)"
echo "$USERNAME"
sudo usermod -aG nocturne "$USERNAME"
sudo mkdir -p "$LOG_ROOT_DIR/backend" "$LOG_ROOT_DIR/frontend"
sudo chown -R :nocturne "$LOG_ROOT_DIR"
sudo chmod -R 775 "$LOG_ROOT_DIR"
scripts/start_server.sh
-20
View File
@@ -1,20 +0,0 @@
#!/bin/bash
# refresh env
/bin/bash ./flush_env.sh
# start frontend
(
cd frontend || exit
bun i
bun next build
screen -S nocturne_frontend -d -m bun start
cd ..
) &
# start backend
(
cd backend || exit
screen -S nocturne_backend -d -m cargo r -r
cd ..
) & wait