lumither/nocturne
1
0 Fork 0
mirror of https://github.com/Lumither/nocturne.git synced 2026-10-17 10:26:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

PBS from Lumither/refactor

Browse Source 
Periodic Branch Synchronization
This commit is contained in:
lumither authored and GitHub committed 2025-04-13 04:02:54 -07:00
2 parents 6da6231c92 6a6af722c3
commit 3f02916e8f
138 files changed
+2133 -1785
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+3
View File
@@ -8,3 +8,6 @@
# exclude post repo # exclude post repo
/blog_posts_git /blog_posts_git
# rustc ICE dump
rustc-ice-*.txt
Makefile
+2 -9
View File
@@ -3,14 +3,7 @@ ifeq ($(ARCH), arm64)
ARCH := aarch64 ARCH := aarch64
endif endif
all: release .PHONY: up
release:
docker compose -f prod.compose.yaml build --build-arg ARCH=$(ARCH)
dev:
docker-compose -f dev.compose.yaml up --build
clean: clean:
docker-compose -f prod.compose.yaml -f dev.compose.yaml rm -fsv echo "todo"
README.md
+6 -32
View File
@@ -1,36 +1,10 @@
# `Nocturne: personal website & CMS` # `Nocturne: Personal Website`
This is the code repo of my personal website, and [here](https://github.com/Lumither/blog-posts/) is the link of the This is the code repo of my [personal website](https://lumither.com),
content repo. and [here](https://github.com/Lumither/blog-posts/) is the link to the
content (blog posts etc.) repo.
Powered by Next.js and Rust, built with passion and love Powered by Next.js and Rust, built with passion and love
<p align="center"> > [!NOTE]
<img src="https://avatars.githubusercontent.com/u/46409277?v=4" alt="GitHub Avatar"/> > This is an unstable branch, checkout `release` for the (guaranteed) working version
</p>
*will add more here in the future
## Nav:
<todo>
## Todos:
### Backend
- [ ] Add: log system
- [ ] Add: search api
- [ ] Add: tag filter (api)
- [ ] Abandon: UUID mechanism
- [ ] Fix: db connection issue
### Frontend
- [ ] Page: Home
- [ ] Page: About
- [ ] Page: Tags
- [ ] Page: Archive
- [ ] Page: Search
- [ ] Add: Right sidebar
backend/Cargo.toml
+2 -3
View File
@@ -1,8 +1,7 @@
[workspace] [workspace]
resolver = "2" resolver = "3"
members = [ members = [
"api_server", "server",
"constants",
"macros", "macros",
"markdown", "markdown",
] ]
backend/api_server/Cargo.toml
-25
View File
@@ -1,25 +0,0 @@
[package]
name = "api_server"
version = "0.1.0"
edition = "2021"
[dependencies]
macros = { path = "../macros" }
constants = { path = "../constants" }
markdown = { path = "../markdown" }
chrono = { version = "0.4.38", features = ["serde", "alloc"] }
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] }
tracing-appender = "0.2.3"
dotenv = "0.15.0"
dirs = "5.0.1"
tokio = { version = "1.40.0", features = ["full"] }
sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] }
axum = { version = "0.7.5", features = ["macros"] }
serde = { version = "1.0.199", features = ["derive"] }
serde_json = "1.0.128"
uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] }
futures = { version = "0.3.30", features = ["executor"] }
cron = "0.13.0"
git2 = "0.19.0"
thiserror = "2.0.3"
backend/api_server/src/blog/api/get_page_count.rs
-26
View File
@@ -1,26 +0,0 @@
use constants::blog::PAGE_SIZE;
use axum::extract::State;
use axum::Json;
use serde_json::{json, Value};
use sqlx::{query, PgPool, Row};
use tracing::error;
pub async fn get_page_count(
State(db_connection): State<PgPool>,
) -> Result<Json<Value>, Json<Value>> {
match query("select cast(ceil(count(*) / $1) as integer) as res from post")
.bind(PAGE_SIZE as f32)
.fetch_one(&db_connection)
.await
{
Ok(res) => {
let res: i32 = res.get("res");
Ok(Json::from(json!({"res": res})))
}
Err(e) => {
error!("Failed to get blog page count: {}", e.to_string());
Err(Json::from(json!({"error": e.to_string()})))
}
}
}
backend/api_server/src/blog/api/get_post.rs
-153
View File
@@ -1,153 +0,0 @@
use axum::{
extract::{Path, State},
http::StatusCode,
Json,
};
use serde_json::{json, Value};
use sqlx::{query, Error, PgPool, Row};
use std::str::FromStr;
use tracing::error;
use uuid::Uuid;
pub async fn get_post(
State(db_connection): State<PgPool>,
Path(post_id): Path<String>,
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let post_id = match Uuid::from_str(post_id.as_str()) {
Ok(id) => id,
Err(e) => {
error!(
"Failed to parse UUID <{}>: {}",
post_id.as_str(),
e.to_string()
);
return Err((
StatusCode::BAD_REQUEST,
Json::from(json!(("error", e.to_string()))),
));
}
};
let post = match query(
r#"
SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post
FROM post t
WHERE post_id = $1;
"#,
)
.bind(post_id)
.fetch_one(&db_connection)
.await
{
Ok(value) => value.get::<Value, _>("post"),
Err(e) => {
error!(
"Database error on Reading `base post information` from table `post`: <{}>: {}",
post_id.to_string(),
e.to_string()
);
return match e {
Error::RowNotFound => Err((
StatusCode::BAD_REQUEST,
Json::from(json!({"error": e.to_string()})),
)),
_ => Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
)),
};
}
};
let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;")
.bind(post_id)
.fetch_all(&db_connection)
.await
{
Ok(tags) => tags.iter().map(|tag| tag.get::<String, _>("tag")).collect(),
Err(e) => {
error!("Database query error :{}", e.to_string());
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
));
}
};
let meta = match query(
r#"
SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta
FROM meta t
WHERE post_id = $1;
"#,
)
.bind(post_id)
.fetch_one(&db_connection)
.await
{
Ok(value) => value.get::<Value, _>("meta"),
Err(e) => {
error!(
"Database error on Reading `{}` from table `{}`: <{}>: {}",
"post meta data",
"meta",
post_id.to_string(),
e.to_string()
);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
));
}
};
let adjacent = match query(r"
WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1),
prev AS (SELECT post_id, t.title, t.sub_title, t.header_img
FROM meta t
WHERE t.date < (SELECT date FROM target_post)
ORDER BY t.date DESC
LIMIT 1),
next AS (SELECT post_id, t.title, t.sub_title, t.header_img
FROM meta t
WHERE t.date > (SELECT date FROM target_post)
ORDER BY t.date
LIMIT 1)
SELECT json_strip_nulls(json_build_object(
'prev',
(SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img)
FROM prev),
'next',
(SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img)
FROM next)
)) AS result;
"
)
.bind(post_id)
.fetch_one(&db_connection)
.await
{
Ok(res) => {res.get::<Value, _>("result")}
Err(e) => {
error!(
"Database error on Reading `{}` from table `{}`: <{}>: {}",
"adjoint posts data",
"meta",
post_id.to_string(),
e.to_string()
);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!({"error": e.to_string()})),
));
}
};
let mut res = post.as_object().unwrap().to_owned();
res.insert("tags".to_string(), tags.into());
res.insert("meta".to_string(), meta);
res.insert("adj".to_string(), adjacent);
Ok(Json::from(json!(res)))
}
backend/api_server/src/blog/api/mod.rs
-3
View File
@@ -1,3 +0,0 @@
pub mod get_page_count;
pub mod get_post;
pub mod get_post_list;
backend/api_server/src/blog/cron/check_update/error.rs
-49
View File
@@ -1,49 +0,0 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum PostIdxError {
#[error("missing essential field `{}` for file `{}`", field, filename)]
MissingField { field: String, filename: String },
#[error(
"invalid field `{}` for file `{}`, <{}>: {}",
field_name,
filename,
err_field,
msg
)]
InvalidField {
field_name: String,
err_field: String,
filename: String,
msg: String,
},
#[error(
"database error on writing `{}` to table `{}`: <{}>: {}",
data_desc,
db_table,
id,
msg
)]
DBWriteFailure {
data_desc: String,
db_table: String,
id: String,
msg: String,
},
#[error(
"Database error on Reading `{}` from table `{}`: <{}>: {}",
data_desc,
db_table,
id,
msg
)]
DBReadFailure {
data_desc: String,
db_table: String,
id: String,
msg: String,
},
}
backend/api_server/src/blog/cron/check_update/index.rs
-351
View File
@@ -1,351 +0,0 @@
use std::error::Error;
use std::path::Path;
use crate::blog::cron::check_update::utils::get_md_file_basename;
use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date};
use markdown::MdFile;
use sqlx::{query, PgPool, Pool, Postgres, Row};
use tokio::runtime::Runtime;
use tracing::{info, warn};
async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
let post_id = post.file_id;
let title = match post.meta.get("title") {
None => {
return Err(PostIdxError::MissingField {
field: "title".to_string(),
filename: post.filename.clone(),
})
}
Some(title) => title.as_str().unwrap_or_else(|| {
warn!("invalid title for `{}`", &post.filename);
"invalid title"
}),
};
let sub_title = post
.meta
.get("sub_title")
.and_then(|v| v.as_str())
.unwrap_or("");
let summary = post
.meta
.get("summary")
.and_then(|v| v.as_str())
.unwrap_or("");
let content = post.content.as_str();
let category = match post.meta.get("category") {
None => {
return Err(PostIdxError::MissingField {
field: "category".to_string(),
filename: post.filename.clone(),
})
}
Some(title) => title.as_str().unwrap_or_else(|| {
warn!("invalid category for `{}`", &post.filename);
"invalid category"
}),
};
let last_update = {
if let Some(last_update) = post.meta.get("last_update") {
match parse_date(last_update.as_str().unwrap()) {
Ok(date) => Some(date),
Err(e) => {
return Err(PostIdxError::InvalidField {
field_name: "last_update".to_string(),
err_field: last_update.to_string(),
filename: post.filename.clone(),
msg: e.to_string(),
})
}
}
} else {
None
}
};
let first_update = match post.meta.get("date") {
None => {
return Err(PostIdxError::MissingField {
field: "date".to_string(),
filename: post.filename.clone(),
})
}
Some(first_update) => match parse_date(first_update.as_str().unwrap()) {
Ok(date) => date,
Err(e) => {
return Err(PostIdxError::InvalidField {
field_name: "date".to_string(),
err_field: first_update.to_string(),
filename: post.filename.clone(),
msg: e.to_string(),
})
}
},
};
let filename = get_md_file_basename(&post.filename);
if let Err(e) = query(
r##"
INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (post_id) DO
UPDATE SET
title = EXCLUDED.title,
summary = EXCLUDED.summary,
content = EXCLUDED.content,
last_update = EXCLUDED.last_update,
first_update = EXCLUDED.first_update,
sub_title = EXCLUDED.sub_title,
category = EXCLUDED.category,
filename = EXCLUDED.filename;
"##,
)
.bind(post_id)
.bind(title)
.bind(summary)
.bind(content)
.bind(last_update)
.bind(first_update)
.bind(sub_title)
.bind(category)
.bind(filename)
.execute(db_connection)
.await
{
return Err(
PostIdxError::DBWriteFailure {
data_desc: "post base information".to_string(),
db_table: "Post".to_string(),
id: post_id.to_string(),
msg: e.to_string(),
}
);
}
Ok(())
}
async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
if let Some(tags_string) = post.meta.get("tags").to_owned() {
let tags = match tags_string.as_array().map(|vec| {
vec.iter()
.map(|val| val.as_str().unwrap())
.collect::<Vec<_>>()
}) {
None => {
return Err(PostIdxError::MissingField {
field: "tags".to_string(),
filename: post.filename.clone(),
})
}
Some(val) => val,
};
let post_id = post.file_id;
// remove old record
if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1")
.bind(post_id)
.execute(db_connection)
.await
{
return Err(PostIdxError::DBWriteFailure {
data_desc: "post tag information".to_string(),
db_table: "Tag".to_string(),
id: post_id.to_string(),
msg: e.to_string(),
});
}
// add new record
for tag in tags {
match query("INSERT INTO Tag VALUES ($1, $2);")
.bind(post_id)
.bind(tag)
.execute(db_connection)
.await
{
Ok(_) => {}
Err(e) => {
return Err(PostIdxError::DBWriteFailure {
data_desc: "post tag information".to_string(),
db_table: "Tag".to_string(),
id: post_id.to_string(),
msg: e.to_string(),
})
}
}
}
}
Ok(())
}
async fn hash(db_connection: &PgPool, post: &MdFile) -> Result<bool, PostIdxError> {
let base64_form = post.hash.clone();
match query("SELECT hash FROM Hash WHERE post_id = $1")
.bind(post.file_id)
.fetch_optional(db_connection)
.await
{
Ok(row) => {
let db_record: String = match row {
None => "".to_string(),
Some(row) => row.try_get("hash").unwrap_or("".to_string()),
};
if base64_form == db_record {
Ok(false)
} else {
if let Err(e) = query(
r#"
INSERT INTO Hash (post_id, hash) VALUES ($1, $2)
ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#,
)
.bind(post.file_id)
.bind(&base64_form)
.execute(db_connection)
.await
{
return Err(PostIdxError::DBWriteFailure {
data_desc: "post hash".to_string(),
db_table: "Hash".to_string(),
id: post.file_id.to_string(),
msg: e.to_string(),
});
}
Ok(true)
}
}
Err(e) => Err(PostIdxError::DBReadFailure {
data_desc: "post hash".to_string(),
db_table: "Hash".to_string(),
id: post.file_id.to_string(),
msg: e.to_string(),
}),
}
}
async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> {
if let Err(e) = query(
// todo: sanitization
format!(
r#"
DO
$$
DECLARE
_post_id UUID := '{}';
_meta_cols JSONB := '{}';
_exists BOOLEAN;
_col_name TEXT;
_col_value TEXT;
_sql TEXT;
_insert_columns TEXT := 'post_id';
_insert_values TEXT := quote_literal(_post_id);
_update_set TEXT := '';
_blacklist TEXT[] := ARRAY['tags'];
BEGIN
SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists;
FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols)
LOOP
IF _col_name = ANY(_blacklist) THEN
CONTINUE;
END IF;
IF NOT EXISTS (SELECT 1
FROM information_schema.columns
WHERE table_name = 'meta'
AND column_name = _col_name
-- may change in the future
AND table_schema = 'public') THEN
_sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT';
EXECUTE _sql;
END IF;
_insert_columns := _insert_columns || ', ' || quote_ident(_col_name);
_insert_values := _insert_values || ', ' || quote_literal(_col_value);
_update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value);
END LOOP;
_update_set := substring(_update_set FROM 3);
IF _exists THEN
_sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id);
EXECUTE _sql;
ELSE
_sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')';
EXECUTE _sql;
END IF;
END;
$$ LANGUAGE plpgsql;
"#, post.file_id.to_string().as_str(), post.meta
).as_str()
)
.execute(db_connection)
.await {
Err(PostIdxError::DBWriteFailure {
data_desc: "post meta data".to_string(),
db_table: "Meta".to_string(),
id: post.file_id.to_string(),
msg: e.to_string(),
})
} else {
Ok(())
}
}
pub async fn index_file(path: &Path, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
let md_file = MdFile::from_file(path)?;
async fn trail(db_conn: &Pool<Postgres>, md_file: &MdFile) -> Result<(), PostIdxError> {
base(db_conn, md_file).await?;
tag(db_conn, md_file).await?;
meta(db_conn, md_file).await?;
hash(db_conn, md_file).await.map(|_| ())
}
let mut cnt = 5;
while let Err(e) = trail(db_conn, &md_file).await {
match e {
PostIdxError::DBWriteFailure { .. } => {
cnt -= 1;
}
PostIdxError::DBReadFailure { .. } => {
cnt -= 1;
}
_ => return Err(e.into()),
}
if cnt <= 0 {
return Err(e.into());
}
}
info!("indexed markdown file: {}", path.display());
Ok(())
}
pub fn drop_index(file_basename: &str, db_conn: &Pool<Postgres>) -> Result<(), Box<dyn Error>> {
let rt = Runtime::new().unwrap();
rt.block_on(async {
if let Err(e) = query(
r#"
WITH deleted_post AS (
DELETE FROM post
WHERE id = (SELECT id FROM post WHERE filename = $1)
RETURNING id
)
DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post);
DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post);
DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post);
"#,
)
.bind(file_basename)
.execute(db_conn)
.await
{
Err(Box::from(
format!("failed to drop index: {}", e).to_string(),
))
} else {
Ok(())
}
})
}
backend/api_server/src/blog/cron/check_update/mod.rs
-146
View File
@@ -1,146 +0,0 @@
use std::env;
use std::fs::remove_dir_all;
use std::path::{Path, PathBuf};
use crate::{
blog::cron::check_update::{
index::drop_index,
utils::{expand_path, get_md_file_basename, index_files, search_md},
},
utils::git,
};
use constants::config::general::{default_value, var_name};
use macros::panic_with_log;
use git2::{Delta, Repository};
use sqlx::{Pool, Postgres};
use tracing::{error, info, warn, Level};
mod error;
mod index;
mod utils;
pub fn task(db_connection: &Pool<Postgres>) {
// init
let git_work_dir = {
let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
expand_path(default_log_path)
} else {
let fallback_path = expand_path(default_value::WORK_DIR.to_string());
warn!(
"unset env var `{}`, using default path: {}/blog_git",
var_name::WORK_DIR,
fallback_path
);
fallback_path
};
Path::new(&path).join("blog_git")
};
// clone & check update
let git_url = match env::var(var_name::BLOG_GIT_URL) {
Ok(url) => url,
Err(e) => {
panic_with_log!(
Level::ERROR,
"failed to read `{}`: {}",
var_name::BLOG_GIT_URL,
e
)
}
};
let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME)
.unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string());
let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH)
.unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string());
match Repository::open(&git_work_dir) {
Ok(repo) => {
// prevent git remote url mismatch (clean cache)
match repo.find_remote(&git_remote_name) {
Ok(remote) => {
if remote.url().is_some_and(|url| url != git_url) {
remove_dir_all(&git_work_dir).unwrap();
warn!("blog content git({}) url mismatch, reinitialized", &git_url);
return;
}
}
Err(e) => {
remove_dir_all(&git_work_dir).unwrap();
error!(
"unexpected error on indexing blog content git({}), reinitialized: {}",
&git_url, e
);
}
}
match git::sync(&git_remote_name, &git_remote_branch, &repo) {
Ok(updates) => {
if !updates.is_empty() {
// incremental update
info!(
"blog content git({}) update detected: {:?}",
&git_url, updates
);
let mut update_md: Vec<PathBuf> = vec![];
for update in updates {
if [Delta::Deleted, Delta::Untracked].contains(&update.status) {
if let Some(path) = &update.old_path {
if drop_index(
&get_md_file_basename(
path.file_name().unwrap().to_str().unwrap(),
),
db_connection,
)
.is_ok()
{
info!(
"blog content git({}) file `{}` is detected as deleted/untracked, index dropped",
&git_url,
&path.display()
);
} else {
error!(
"blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index",
&git_url,
&path.display()
)
}
}
} else if let Some(path) = &update.new_path {
if path.components().any(|comp| comp.as_os_str() == "posts")
&& path.extension().is_some_and(|ext| ext == "md")
{
update_md.push(path.to_path_buf());
}
}
}
index_files(&update_md, db_connection);
}
}
Err(e) => {
error!(
"failed to sync blog content git({}) updates: {}",
&git_url, e
);
}
}
}
Err(_) => {
match Repository::clone(&git_url, &git_work_dir) {
Ok(_repo) => {
// full repo index
let files = search_md(git_work_dir.join("posts"));
index_files(&files, db_connection);
}
Err(e) => {
error!("failed to clone blog content git({}): {}", &git_url, e);
}
}
}
};
}
backend/api_server/src/blog/cron/check_update/utils.rs
-86
View File
@@ -1,86 +0,0 @@
use dirs::home_dir;
use std::path::PathBuf;
use crate::blog::cron::check_update::index::index_file;
use chrono::{DateTime, FixedOffset, ParseResult};
use constants::config::general::{default_value, var_name};
use constants::time::DEFAULT_DATE_FORMAT;
use futures::executor::block_on;
use futures::future::join_all;
use sqlx::{Pool, Postgres};
use tokio::runtime::Runtime;
use tokio::spawn;
use tracing::{error, warn};
pub fn search_md(entry: PathBuf) -> Vec<PathBuf> {
if entry.ends_with(".git") {
return vec![];
}
if entry.is_dir() {
if let Ok(files) = entry.read_dir() {
files
.flat_map(|file| {
let file_path = file.unwrap().path();
search_md(file_path)
})
.collect()
} else {
vec![]
}
} else if entry.extension().is_some_and(|ext| ext == "md") {
vec![entry]
} else {
vec![]
}
}
pub fn index_files(files: &[PathBuf], db_conn: &Pool<Postgres>) {
let rt = Runtime::new().unwrap();
rt.block_on(async {
let files_clone = files.to_owned();
let handlers = files_clone
.into_iter()
.map(|file: PathBuf| {
let conn_clone = db_conn.clone();
spawn(async move {
if let Err(e) = index_file(&file, &conn_clone).await {
error!("failed to index markdown file `{}`: {}", file.display(), e);
}
})
})
.collect::<Vec<_>>();
block_on(join_all(handlers));
});
}
pub fn expand_path(path: String) -> String {
if path.starts_with('~') {
home_dir()
.map(|dir| {
dir.join(path.strip_prefix("~/").unwrap())
.into_os_string()
.into_string()
.unwrap()
})
.unwrap_or_else(|| {
warn!(
"failed to parse env var `{}`, using default value `{}`",
var_name::WORK_DIR,
default_value::WORK_DIR
);
default_value::WORK_DIR.to_string()
})
} else {
path
}
}
pub fn parse_date(date_str: &str) -> ParseResult<DateTime<FixedOffset>> {
DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT)
}
pub fn get_md_file_basename(f_name: &str) -> String {
f_name.replace(".md", "")
}
backend/api_server/src/blog/cron/mod.rs
-14
View File
@@ -1,14 +0,0 @@
use crate::scheduler::tasks::basic::BasicTask;
use crate::scheduler::tasks::Task;
use sqlx::{Pool, Postgres};
pub mod check_update;
pub fn get_tasks(db_connection: Pool<Postgres>) -> Vec<Box<dyn Task>> {
let db_conn_clone = db_connection.clone();
vec![
BasicTask::new(move || check_update::task(&db_conn_clone), "0/15 * * * * *")
.unwrap()
.to_task(),
]
}
backend/api_server/src/blog/mod.rs
-17
View File
@@ -1,17 +0,0 @@
mod api;
pub mod cron;
use crate::blog::api::{
get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list,
};
use axum::routing::get;
use axum::Router;
use sqlx::{Pool, Postgres};
pub fn get_router() -> Router<Pool<Postgres>> {
Router::new()
.route("/get_post_list", get(get_post_list))
.route("/get_page_count", get(get_page_count))
.route("/get_post/:post_id", get(get_post))
}
backend/api_server/src/init/cron.rs
-12
View File
@@ -1,12 +0,0 @@
use crate::blog;
use crate::scheduler::Scheduler;
use sqlx::{Pool, Postgres};
pub fn start(db_pool: Pool<Postgres>) {
let scheduler = Scheduler::new();
let _ = vec![blog::cron::get_tasks(db_pool.clone())]
.into_iter()
.map(|list| scheduler.insert_list(list))
.collect::<Vec<_>>();
scheduler.start().unwrap()
}
backend/api_server/src/init/schema.rs
-49
View File
@@ -1,49 +0,0 @@
use sqlx::{query, Pool, Postgres};
use tracing::info;
const POST_BASE_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Post
(
post_id UUID PRIMARY KEY,
title VARCHAR(255) NOT NULL,
category VARCHAR(255) NOT NULL,
filename TEXT UNIQUE NOT NULL,
summary TEXT,
sub_title TEXT,
content TEXT NOT NULL,
last_update TIMESTAMPTZ,
first_update TIMESTAMPTZ NOT NULL
);
"##;
// base model, will add more cols at runtime
const POST_META_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Meta
(
post_id UUID PRIMARY KEY
);
"##;
const POST_HASH_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Hash
(
post_id UUID PRIMARY KEY,
hash VARCHAR(44)
);
"##;
const TAG_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Tag
(
post_id UUID NOT NULL,
tag VARCHAR(255) NOT NULL
);
"##;
pub async fn init(db_connection: &Pool<Postgres>) {
query(POST_BASE_MODEL).execute(db_connection).await.unwrap();
query(POST_META_MODEL).execute(db_connection).await.unwrap();
query(TAG_MODEL).execute(db_connection).await.unwrap();
query(POST_HASH_MODEL).execute(db_connection).await.unwrap();
info!("database schema initialized");
}
backend/api_server/src/init/server.rs
-52
View File
@@ -1,52 +0,0 @@
use std::env;
use std::error::Error;
use std::num::ParseIntError;
use std::process::exit;
use crate::blog;
use axum::Router;
use constants::config::server::{default_value, var_name};
use macros::panic_with_log;
use sqlx::{Pool, Postgres};
use tracing::{error, info, warn, Level};
pub async fn start(db_pool: Pool<Postgres>) -> Result<(), Box<dyn Error>> {
let app = Router::new()
.nest("/blog", blog::get_router())
.with_state(db_pool);
let port: u32 = match env::var(var_name::PORT) {
Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| {
warn!(
"failed to parse `{}`, using default port: {}",
var_name::PORT,
e.to_string()
);
default_value::PORT
}),
Err(_) => default_value::PORT,
};
let listener =
match tokio::net::TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await {
Ok(listener) => {
info!("server started on {}:{}", default_value::HOST, port);
listener
}
Err(e) => {
panic_with_log!(
Level::ERROR,
"failed to start server on {}:{}: {}",
default_value::HOST,
port,
e.to_string()
);
}
};
axum::serve(listener, app).await.unwrap_or_else(|e| {
error!("failed to start axum: {}", e.to_string());
exit(1);
});
Ok(())
}
backend/api_server/src/main.rs
-13
View File
@@ -1,13 +0,0 @@
mod blog;
mod init;
mod scheduler;
mod utils;
#[tokio::main]
async fn main() {
init::load_env();
let _guards = init::logger::init();
init::start().await;
}
backend/api_server/src/scheduler/mod.rs
-194
View File
@@ -1,194 +0,0 @@
use crate::{
scheduler::scheduler_error::SchedulerError,
scheduler::scheduler_error::SchedulerError::IdAlreadyRunning,
scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper,
scheduler::tasks::Task,
};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc, thread};
use tracing::log::error;
use uuid::Uuid;
pub mod scheduler_error;
mod sleeper;
mod task_func;
pub mod tasks;
struct TaskStatus {
sleeper: Option<Arc<SkippableSleeper>>,
is_alive: AtomicBool,
}
#[derive(Default)]
pub struct Scheduler {
tasks: Arc<RwLock<HashMap<Uuid, Arc<Box<dyn Task>>>>>,
running: Arc<RwLock<HashMap<Uuid, Arc<TaskStatus>>>>,
}
impl Scheduler {
/// Start all tasks in the scheduler
pub fn start(&self) -> Result<(), SchedulerError> {
for (k, _v) in self.tasks.read()?.iter() {
self.run_non_blocking(k)?
}
Ok(())
}
/// Terminate all tasks in the scheduler
pub fn halt() {}
fn run_task(
running: Arc<RwLock<HashMap<Uuid, Arc<TaskStatus>>>>,
id: Uuid,
task: Arc<Box<dyn Task>>,
) -> Result<(), SchedulerError> {
if running.read()?.get(&id).is_some() {
return Err(IdAlreadyRunning);
} else {
running.write()?.insert(
id,
TaskStatus {
sleeper: None,
is_alive: AtomicBool::new(true),
}
.into(),
);
}
loop {
if let Some(next_execution) = task.get_next_execution() {
// guarantee is alive
let task_status = {
let running_reading_guard = running.read()?;
running_reading_guard.get(&id).cloned().clone()
};
if let Some(task_status) = task_status {
if !&task_status.is_alive.load(Ordering::SeqCst) {
{
let mut running_lock_guard = running.write()?;
running_lock_guard.remove(&id);
}
return Ok(());
} else {
let sleeper = Arc::from(SkippableSleeper::from(next_execution));
{
let mut running_lock_guard = running.write()?;
running_lock_guard.insert(
id,
TaskStatus {
sleeper: Some(sleeper.clone()),
is_alive: AtomicBool::new(true),
}
.into(),
);
}
sleeper.start();
task.call();
}
}
} else {
error!("unexpected error ");
break;
}
}
Ok(())
}
/// Run a task with `id`
pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> {
let id = id.to_owned();
if let Some(task) = self.tasks.read()?.get(&id) {
Self::run_task(self.running.clone(), id, task.clone())
} else {
Err(IdNotFound)
}
}
pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> {
let id = id.to_owned();
if let Some(task) = self.tasks.read()?.get(&id) {
let running_clone = self.running.clone();
let task_clone = task.clone();
thread::spawn(move || Self::run_task(running_clone, id, task_clone));
Ok(())
} else {
Err(IdNotFound)
}
}
fn get_running(&self, id: &Uuid) -> Option<Arc<TaskStatus>> {
let ret = self.running.read().unwrap().get(id).cloned();
ret
}
/// Stop a task with `id`
pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> {
match self.get_running(id) {
None => Err(IdNotFound),
Some(status) => {
status.is_alive.store(false, Ordering::SeqCst);
if let Some(s) = &status.sleeper {
s.skip();
} else {
return Err(IdAlreadyRunning);
}
Ok(())
}
}
}
pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> {
match self.get_running(id) {
None => Err(IdNotFound),
Some(status) => {
if let Some(s) = &status.sleeper {
s.skip()
} else {
return Err(IdAlreadyRunning);
}
Ok(())
}
}
}
pub fn insert(&self, task: Box<dyn Task>) -> Result<Uuid, SchedulerError> {
let id = Uuid::new_v4();
self.tasks.write()?.insert(id, Arc::new(task));
Ok(id)
}
pub fn insert_list(&self, tasks: Vec<Box<dyn Task>>) -> Result<Vec<Uuid>, SchedulerError> {
tasks.into_iter().map(|t| self.insert(t)).collect()
}
pub fn new() -> Self {
Self::default()
}
}
#[cfg(test)]
mod tests {
use crate::scheduler::tasks::basic::BasicTask;
use crate::scheduler::Scheduler;
use std::error::Error;
use std::thread;
use std::time::Duration;
#[test]
fn test_run() -> Result<(), Box<dyn Error>> {
let scheduler = Scheduler::default();
let task = BasicTask::new(|| println!("test print"), "* * * * * *")?;
let id = scheduler.insert(Box::new(task))?;
for i in 1..=2 {
println!("Loop {} start", i);
println!("Run {}", i);
scheduler.run_non_blocking(&id)?;
thread::sleep(Duration::from_secs(4));
scheduler.stop(&id)?;
println!("Stop {}", i);
thread::sleep(Duration::from_secs(2));
println!("Loop {} finished", i);
}
Ok(())
}
}
backend/api_server/src/scheduler/sleeper.rs
-77
View File
@@ -1,77 +0,0 @@
use chrono::{DateTime, Utc};
use std::sync::{Arc, Condvar, Mutex};
use std::thread;
use std::time::Duration;
#[derive(Clone, Debug, Default)]
pub struct SkippableSleeper {
pub duration: Duration,
state: Arc<(Mutex<bool>, Condvar)>,
}
impl SkippableSleeper {
pub fn new(duration: Duration) -> Self {
SkippableSleeper {
duration,
state: Arc::new((Mutex::new(false), Condvar::new())),
}
}
// Blocks until the duration has passed or skip() is called
pub fn start(&self) {
let (lock, cvar) = &*self.state;
let mut received = lock.lock().unwrap();
let state_clone = Arc::clone(&self.state);
let duration_clone = self.duration;
thread::spawn(move || {
thread::sleep(duration_clone);
let (lock, cvar) = &*state_clone;
let mut received = lock.lock().unwrap();
*received = true;
cvar.notify_one();
});
while !*received {
received = cvar.wait(received).unwrap();
}
}
// Skip the blocking delay
pub fn skip(&self) {
let (lock, cvar) = &*self.state;
let mut received = lock.lock().unwrap();
*received = true;
cvar.notify_one();
}
}
impl From<DateTime<Utc>> for SkippableSleeper {
fn from(value: DateTime<Utc>) -> Self {
let from_utc_timestamp = value.timestamp() as u64;
let curr_utc_timestamp = Utc::now().timestamp() as u64;
SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp))
}
}
#[cfg(test)]
mod test {
use crate::scheduler::sleeper::SkippableSleeper;
use std::thread;
use std::time::Duration;
#[test]
fn usage() {
let sleeper = SkippableSleeper::new(Duration::new(10, 0));
thread::spawn({
let sleeper = sleeper.clone();
move || {
println!("Sleeping...");
sleeper.start();
println!("Woke up!");
}
});
println!("Start Sleeping...");
thread::sleep(Duration::from_secs(4));
println!("Skipping sleep...");
sleeper.skip();
}
}
backend/api_server/src/scheduler/task_func.rs
-3
View File
@@ -1,3 +0,0 @@
pub trait TaskFunc: Fn() + Send + Sync + 'static {}
impl<T> TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {}
backend/api_server/src/scheduler/tasks/mod.rs
-8
View File
@@ -1,8 +0,0 @@
use chrono::{DateTime, Utc};
pub mod basic;
pub trait Task: Send + Sync {
fn get_next_execution(&self) -> Option<DateTime<Utc>>;
fn call(&self);
}
backend/macros/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package] [package]
name = "macros" name = "macros"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2024"
[dependencies] [dependencies]
tracing = "0.1.40" tracing = "0.1.40"
backend/macros/src/dev.rs
+6
View File
@@ -0,0 +1,6 @@
#[macro_export]
macro_rules! dev_consume {
($($var:ident),+) => {
$(let _ = $var;)*
};
}
backend/macros/src/include_str.rs
+24
View File
@@ -0,0 +1,24 @@
/// Includes a UTF-8 encoded file as a string
///
/// The file is located relative to `CARGO_MANIFEST_DIR`.
///
/// # Example
/// ```rust
/// use macros::include_str_from_crate_root;
///
/// let str: &str = include_str_from_crate_root!("/query/scheme.sql");
///
// todo: impl
// const PATH_PREFIX: &str = "/src";
// let str: &str = include_str_from_crate_root!("{}/query/scheme.sql", PATH_PREFIX);
/// ```
#[macro_export]
macro_rules! include_str_from_crate_root {
// ($path:literal $(, $($arg:tt)*)?) => {
// ($path:expr, $($args:tt)*) => {
// include_str!(concat!(env!("CARGO_MANIFEST_DIR"), format_args!($path, $($args)*)))
// };
($path:expr) => {
include_str!(concat!(env!("CARGO_MANIFEST_DIR"), $path))
};
}
backend/macros/src/lib.rs
+2
View File
@@ -1 +1,3 @@
mod dev;
mod include_str;
mod log; mod log;
backend/macros/src/log.rs
+21 -2
View File
@@ -6,7 +6,7 @@
/// use macros::panic_with_log; /// use macros::panic_with_log;
/// use tracing::Level; /// use tracing::Level;
/// ///
/// panic_with_log!(Level::ERROR, "Test panicked: {}", 42); /// panic_with_log!(Level::ERROR, "test panicked: {}", 42);
/// ``` /// ```
#[macro_export] #[macro_export]
macro_rules! panic_with_log { macro_rules! panic_with_log {
@@ -18,6 +18,25 @@ macro_rules! panic_with_log {
}}; }};
} }
/// log the message with `Level::ERROR` and exit(1)
///
/// # Example
/// ```rust
///
/// use macros::error_panic;
///
/// error_panic!("test panicked: {}", 42);
/// ```
#[macro_export]
macro_rules! error_panic {
($msg:literal $(, $($arg:tt)*)?) => {{
use std::process;
use tracing::{event, Level};
event!(Level::ERROR, $msg $(, $($arg)*)?);
process::exit(1);
}};
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test] #[test]
@@ -25,7 +44,7 @@ mod tests {
use std::io; use std::io;
use tracing::Level; use tracing::Level;
use tracing_subscriber::prelude::*; use tracing_subscriber::prelude::*;
use tracing_subscriber::{fmt, Registry}; use tracing_subscriber::{Registry, fmt};
let format = tracing_subscriber::fmt::format() let format = tracing_subscriber::fmt::format()
.with_level(true) .with_level(true)
backend/markdown/Cargo.toml
+2 -2
View File
@@ -1,14 +1,14 @@
[package] [package]
name = "markdown" name = "markdown"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2024"
[dependencies] [dependencies]
uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] }
serde = { version = "1.0.209", features = ["derive"] } serde = { version = "1.0.209", features = ["derive"] }
serde_json = "1.0.128" serde_json = "1.0.128"
regex = "1.10.4" regex = "1.10.4"
yaml-rust2 = "0.9.0" yaml-rust2 = "0.10.0"
sha2 = "0.10.8" sha2 = "0.10.8"
base64 = "0.22.1" base64 = "0.22.1"
thiserror = "2.0.3" thiserror = "2.0.3"
backend/markdown/src/error.rs
-6
View File
@@ -2,12 +2,6 @@ use thiserror::Error;
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum Error { pub enum Error {
#[error("missing front-matter field: id")]
MissingID,
#[error("invalid file id: {}", msg)]
InvalidID { msg: String },
#[error("invalid front-matter: {}", msg)] #[error("invalid front-matter: {}", msg)]
InvalidFrontMatter { msg: String }, InvalidFrontMatter { msg: String },
backend/markdown/src/lib.rs
+19 -30
View File
@@ -1,23 +1,20 @@
mod error; pub mod error;
mod utils; mod utils;
use std::fs; use std::{fs, path::Path, str::FromStr};
use std::path::Path;
use std::str::FromStr;
use crate::error::Error; use crate::{
use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; error::Error,
utils::front_matter::{parse_front_matter, split_md_front_matter},
};
use base64::{engine::general_purpose, Engine}; use base64::{Engine, engine::general_purpose};
use serde::Serialize; use serde::Serialize;
use serde_json::Value; use serde_json::Value;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use uuid::Uuid;
#[derive(Default, Debug, Serialize)] #[derive(Default, Debug, Serialize, Clone)]
pub struct MdFile { pub struct MdFile {
pub file_id: Uuid,
pub filename: String,
pub meta: Value, pub meta: Value,
pub content: String, pub content: String,
pub hash: String, pub hash: String,
@@ -29,30 +26,22 @@ impl MdFile {
Ok(file) => file, Ok(file) => file,
Err(e) => return Err(Error::FSError { msg: e.to_string() }), Err(e) => return Err(Error::FSError { msg: e.to_string() }),
}; };
let raw_file_bytes = raw_file.as_bytes().to_vec(); Self::from_str(&raw_file)
let (front_matter_str, content) = split_md_front_matter(raw_file); }
}
impl FromStr for MdFile {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let bytes = s.as_bytes().to_vec();
let (front_matter_str, content) = split_md_front_matter(s);
let front_matter = match parse_front_matter(&front_matter_str) { let front_matter = match parse_front_matter(&front_matter_str) {
Ok(front_matter) => front_matter, Ok(front_matter) => front_matter,
Err(e) => return Err(Error::InvalidFrontMatter { msg: e.to_string() }), Err(e) => return Err(Error::InvalidFrontMatter { msg: e.to_string() }),
}; };
let hash = calculate_hash_base64(&raw_file_bytes); let hash = calculate_hash_base64(&bytes);
let file_id = match front_matter["id"].as_str() {
Some(id) => match Uuid::from_str(id) {
Ok(id) => id,
Err(e) => return Err(Error::InvalidID { msg: e.to_string() }),
},
None => return Err(Error::MissingID),
};
let file_name = Path::new(file_path)
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_string();
Ok(MdFile { Ok(MdFile {
file_id,
filename: file_name,
meta: front_matter, meta: front_matter,
content, content,
hash, hash,
backend/markdown/src/utils/front_matter.rs
+9 -8
View File
@@ -1,17 +1,19 @@
use std::error::Error;
use crate::utils::yaml::yaml_to_serde; use crate::utils::yaml::yaml_to_serde;
use regex::Regex; use regex::Regex;
use serde_json::Value; use serde_json::Value;
use std::error::Error;
use yaml_rust2::YamlLoader; use yaml_rust2::YamlLoader;
pub fn split_md_front_matter(raw_file: String) -> (String, String) { pub fn split_md_front_matter(raw_file: &str) -> (String, String) {
let re = Regex::new(r"(?s)^(---\n.*?\n---\n)(.*)").unwrap(); let re = Regex::new(r"(?s)^(---\n.*?\n---\n)(.*)").unwrap();
if let Some(captures) = re.captures(raw_file.as_str()) { if let Some(captures) = re.captures(raw_file) {
let front_matter = captures.get(1).map_or("", |m| m.as_str()).to_string(); let front_matter = captures.get(1).map_or("", |m| m.as_str()).to_string();
let main_content = captures.get(2).map_or("", |m| m.as_str()).to_string(); let main_content = captures.get(2).map_or("", |m| m.as_str()).to_string();
(front_matter, main_content) (front_matter, main_content)
} else { } else {
(String::new(), raw_file) (String::new(), raw_file.to_string())
} }
} }
@@ -43,8 +45,7 @@ tags: [ "test1", "test2" ]
test body test body
"## "##,
.to_string(),
); );
assert_eq!( assert_eq!(
front_matter, front_matter,
@@ -64,7 +65,7 @@ tags: [ "test1", "test2" ]
#[test] #[test]
fn test_front_matter_parser() { fn test_front_matter_parser() {
dbg!(parse_front_matter( parse_front_matter(
r##"--- r##"---
title: test_title title: test_title
date: 0000-00-00+0000 date: 0000-00-00+0000
@@ -75,7 +76,7 @@ summary: summary
tags: [ "test1", "test2" ] tags: [ "test1", "test2" ]
--- ---
"##, "##,
)) )
.expect("Failed to parse front matter"); .expect("Failed to parse front matter");
} }
} }
backend/server/Cargo.toml
+29
View File
@@ -0,0 +1,29 @@
[package]
name = "server"
version = "0.1.0"
edition = "2024"
[dependencies]
macros = { path = "../macros" }
markdown = { path = "../markdown" }
axum = { version = "0.8.1", features = ["macros"] }
tracing = { version = "0.1.41", features = ["log"] }
git2 = "0.20.0"
chrono = { version = "0.4.39", features = ["serde"] }
cron = "0.15.0"
sqlx = { version = "0.8.3", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] }
serde_json = "1.0.133"
serde = { version = "1.0.215", features = ["derive"] }
tracing-subscriber = "0.3.18"
tracing-appender = "0.2.3"
dirs = "6.0.0"
thiserror = "2.0.11"
tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros", "sync"] }
dotenv = "0.15.0"
uuid = { version = "1.13.2", features = ["v4"] }
futures = "0.3.31"
[build-dependencies]
git2 = "0.20.0"
chrono = "0.4.39"
backend/server/build.rs
+16
View File
@@ -0,0 +1,16 @@
use chrono::Utc;
use git2::Repository;
fn main() {
let repo = Repository::open("../..").expect("failed to open repository");
let commit = repo
.head()
.expect("failed to get HEAD")
.peel_to_commit()
.expect("failed to get HEAD commit");
let hash = commit.id().to_string();
let utc_timestamp = Utc::now().date_naive().format("%Y%m%d").to_string();
let pkg_version = env!("CARGO_PKG_VERSION");
let build_id = format!("{}:{}@{}", pkg_version, &hash[..8], utc_timestamp);
println!("cargo:rustc-env=BUILD_ID={}", build_id);
}
backend/constants/src/blog.rs → backend/server/src/constants/blog.rs
View File
File renamed without changes.
backend/constants/src/config.rs → backend/server/src/constants/config.rs
+2
View File
@@ -1,3 +1,5 @@
// todo: rebuild with static OnceCell
// Database related // Database related
pub mod database { pub mod database {
pub mod var_name { pub mod var_name {
backend/constants/src/lib.rs → backend/server/src/constants/mod.rs
View File
File renamed without changes.
backend/constants/src/time.rs → backend/server/src/constants/time.rs
+2 -1
View File
@@ -1,2 +1,3 @@
pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z"; pub const DEFAULT_TIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%z";
pub const DEFAULT_DATE_FORMAT: &str = "%Y-%m-%d%zT%H%M%S";
pub const DEFAULT_NAIVE_DATE_FORMAT: &str = "%Y-%m-%d";
backend/api_server/src/init/database.rs → backend/server/src/init/database.rs
+3 -2
View File
@@ -2,7 +2,8 @@ use std::env;
use std::error::Error; use std::error::Error;
use std::str::FromStr; use std::str::FromStr;
use constants::config::database::{default_value, var_name}; use crate::constants::config::database::{default_value, var_name};
use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
use sqlx::{ConnectOptions, Pool, Postgres}; use sqlx::{ConnectOptions, Pool, Postgres};
use tracing::info; use tracing::info;
@@ -12,7 +13,7 @@ pub async fn init() -> Result<Pool<Postgres>, Box<dyn Error>> {
.unwrap() .unwrap()
.disable_statement_logging(); .disable_statement_logging();
Ok(PgPoolOptions::new() Ok(PgPoolOptions::new()
.max_connections(1) // todo: temporal solution, to be fixed .max_connections(1) // todo: temporary solution, to be fixed
.connect_with(db_connect_option) .connect_with(db_connect_option)
.await?) .await?)
} }
backend/api_server/src/init/logger.rs → backend/server/src/init/logger.rs
+5 -4
View File
@@ -1,14 +1,15 @@
use std::path::Path; use std::path::Path;
use std::{env, io}; use std::{env, io};
use crate::constants::config::general::{default_value, var_name};
use chrono::Local; use chrono::Local;
use constants::config::general::{default_value, var_name};
use dirs::home_dir; use dirs::home_dir;
use tracing::info; use tracing::info;
use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling};
use tracing_subscriber::{ use tracing_subscriber::{
filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, Layer, Registry, filter::LevelFilter, fmt, fmt::MakeWriter, fmt::format::Writer,
layer::SubscriberExt, Layer, Registry, fmt::time::FormatTime, layer::SubscriberExt,
}; };
struct LocalTimer; struct LocalTimer;
@@ -18,7 +19,7 @@ impl FormatTime for LocalTimer {
write!( write!(
w, w,
"{}", "{}",
Local::now().format(constants::time::DEFAULT_TIME_FORMAT) Local::now().format(crate::constants::time::DEFAULT_TIME_FORMAT)
) )
} }
} }
backend/api_server/src/init/misc.rs → backend/server/src/init/misc.rs
View File
File renamed without changes.
backend/api_server/src/init/mod.rs → backend/server/src/init/mod.rs
+14 -22
View File
@@ -1,13 +1,13 @@
mod cron;
mod database; mod database;
pub mod logger; pub mod logger;
mod misc; mod misc;
mod schema;
pub mod server;
use macros::panic_with_log;
use std::{env, panic}; use std::{env, panic};
use tracing::{info, Level};
use crate::modules::{ModuleTree, blog::Blog};
use macros::error_panic;
use tracing::info;
pub fn load_env() { pub fn load_env() {
if let Some(env_file) = env::args().nth(1) { if let Some(env_file) = env::args().nth(1) {
@@ -34,25 +34,17 @@ pub async fn start() {
pool pool
} }
Err(e) => { Err(e) => {
panic_with_log!( error_panic!("database connection pool failed to initialize: {}", e);
Level::ERROR,
"database connection pool failed to initialize: {}",
e
);
} }
}; };
// schema init // // schema init
schema::init(&db_pool).await; // schema::init(&db_pool).await;
// load cron task ModuleTree::new()
cron::start(db_pool.clone()); .add_module(Blog {
db_handler: db_pool,
// api serve })
match server::start(db_pool).await { .blocking_serve()
Ok(_) => {} .await
Err(e) => {
panic_with_log!(Level::ERROR, "server panicked: {}", e)
}
}
} }
backend/server/src/main.rs
+22
View File
@@ -0,0 +1,22 @@
#[allow(dead_code)]
mod constants;
mod scheduler;
mod init;
mod modules;
mod utils;
use crate::init::{logger, start};
const BUILD_ID: &str = env!("BUILD_ID");
#[tokio::main]
async fn main() {
println!("{}", BUILD_ID);
init::load_env();
let _guards = logger::init();
start().await;
}
backend/server/src/modules/blog/api/get_page_count/mod.rs
+25
View File
@@ -0,0 +1,25 @@
use crate::constants::blog::PAGE_SIZE;
use axum::{Json, extract::State};
use serde_json::{Value, json};
use sqlx::{PgPool, Row, query};
use tracing::error;
const GET_PAGE_COUNT_QUERY: &str = include_str!("post_page_count.sql");
pub async fn handler(State(db_connection): State<PgPool>) -> Result<Json<Value>, Json<Value>> {
match query(GET_PAGE_COUNT_QUERY)
.bind(PAGE_SIZE as i64)
.fetch_one(&db_connection)
.await
{
Ok(res) => {
let res: i32 = res.get("count");
Ok(Json::from(json!({"res": res})))
}
Err(e) => {
error!("failed to get blog page count: {}", e.to_string());
Err(Json::from(json!({"error": e.to_string()})))
}
}
}
backend/server/src/modules/blog/api/get_page_count/post_page_count.sql
+2
View File
@@ -0,0 +1,2 @@
SELECT cast(ceil(count(*) / cast($1 AS FLOAT)) AS INTEGER) AS count
FROM posts;
backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql
+1
View File
@@ -0,0 +1 @@
SELECT
frontend/app/public/css/article.css → backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql
View File
File renamed without changes.
backend/server/src/modules/blog/api/get_post/mod.rs
+35
View File
@@ -0,0 +1,35 @@
use macros::dev_consume;
use axum::{
Json,
extract::{Path, State},
response::IntoResponse,
};
use serde_json::json;
use sqlx::PgPool;
use uuid::Uuid;
const GET_POST_BY_UUID: &str = include_str!("get_post_by_uuid.sql");
const GET_POST_BY_STR_ID: &str = include_str!("get_post_by_str_id.sql");
pub async fn handler(
State(db_connection): State<PgPool>,
Path(identifier): Path<String>,
) -> impl IntoResponse {
if let Ok(id) = identifier.parse::<Uuid>() {
select_post_by_uuid(&db_connection, id).await
} else {
select_post_by_str_id(&db_connection, &identifier).await
};
Json::from(json!({"res": "ok"})).into_response()
}
async fn select_post_by_uuid(db: &PgPool, id: Uuid) {
dev_consume!(db, id);
todo!()
}
async fn select_post_by_str_id(db: &PgPool, id: &str) {
dev_consume!(db, id);
todo!()
}
backend/api_server/src/blog/api/get_post_list.rs → backend/server/src/modules/blog/api/get_post_list.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::error::Error; use std::error::Error;
use constants::blog::PAGE_SIZE; use crate::constants::blog::PAGE_SIZE;
use axum::{extract::Query, extract::State, http::StatusCode, Json}; use axum::{extract::Query, extract::State, http::StatusCode, Json};
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
backend/server/src/modules/blog/api/mod.rs
+6
View File
@@ -0,0 +1,6 @@
pub mod get_page_count;
// todo: remove exception tag
#[allow(dead_code)]
pub mod get_post;
// pub mod get_post_list;
backend/server/src/modules/blog/components/check_update/apply.rs
+208
View File
@@ -0,0 +1,208 @@
use crate::modules::blog::components::check_update::{
changes::{Change, Create, Delete, Move, Update},
error::Error,
utils::{parse_naive_date_str, parse_post_identifier},
};
use futures::{
future::{join_all, BoxFuture},
FutureExt,
};
use sqlx::{query, PgPool, Postgres, Row, Transaction};
use tracing::error;
const DELETE_POST_BY_ID: &str = include_str!("sql/delete_post_by_id.sql");
const GET_TAG_ID: &str = include_str!("sql/get_tag_id.sql");
const GET_CATEGORY_ID: &str = include_str!("sql/get_category_id.sql");
const CREATE_RETURN_TAG_ID: &str = include_str!("sql/create_return_tag_id.sql");
const CREATE_RETURN_CATEGORY_ID: &str = include_str!("sql/create_return_category_id.sql");
const CREATE_POST: &str = include_str!("sql/create_post.sql");
pub async fn apply_deltas(db_conn: &PgPool, changes: Vec<Change>) {
let handlers = changes
.into_iter()
.map(|change| match change {
Change::Delete(del) => handle_delete(db_conn, del).boxed(),
Change::Move(mv) => handle_move(db_conn, mv).boxed(),
Change::Update(update) => handle_update(db_conn, update).boxed(),
Change::Create(create) => handle_create(db_conn, create).boxed(),
})
.collect::<Vec<BoxFuture<'_, ()>>>();
join_all(handlers).await;
}
async fn handle_delete(db: &PgPool, del: Delete) {
if let Err(e) = query(DELETE_POST_BY_ID).bind(del.uuid).execute(db).await {
error!(
"failed to delete post: {}, {}",
del.uuid.to_string(),
e.to_string()
)
}
}
async fn handle_move(db: &PgPool, mv: Move) {
handle_delete(
db,
Delete {
uuid: mv.uuid,
path: mv.from,
},
)
.await;
handle_create(
db,
Create {
uuid: mv.uuid,
path: mv.to,
payload: mv.payload,
},
)
.await;
}
async fn handle_update(db: &PgPool, update: Update) {
handle_delete(
db,
Delete {
uuid: update.uuid,
path: update.path.clone(),
},
)
.await;
handle_create(
db,
Create {
uuid: update.uuid,
path: update.path,
payload: update.payload,
},
)
.await;
}
async fn handle_create(db: &PgPool, create: Create) {
let mut tx = db.begin().await.unwrap();
let identifier = parse_post_identifier(&create.path).unwrap();
let tags_json = create.payload.meta["tags"].clone();
let tags: Vec<_> = if let Some(tags) = tags_json.as_array() {
tags.iter().filter_map(|tag| tag.as_str()).collect()
} else {
vec![]
};
let mut tag_ids = vec![];
for tag in tags {
match get_or_create_tag_id(&mut tx, tag).await {
Ok(tid) => tag_ids.push(tid),
Err(e) => {
error!("failed to fetch or create tag ({}): {}", tag, e);
return;
}
}
}
let category_json = create.payload.meta["category"].clone();
let category = category_json.as_str().unwrap_or("Uncategorized");
let category_id = match get_or_create_category_id(&mut tx, category).await {
Ok(id) => id,
Err(e) => {
error!(
"failed to fetch category id for post {}: {}",
&identifier, e
);
return;
}
};
let meta = create.payload.meta;
let post_id = create.uuid;
let post_identifier = &identifier;
let post_title = meta
.get("title")
.map(|t| t.as_str().unwrap())
.unwrap_or("Untitled");
let post_subtitle = meta
.get("subtitle")
.map(|t| t.as_str().unwrap())
.unwrap_or("");
let post_date_created = if let Some(Some(date)) = meta
.get("date")
.map(|d| parse_naive_date_str(d.as_str().unwrap_or("")))
{
date
} else {
error!("failed to parse `date` for post: {}", post_identifier);
return;
};
let post_date_updated = meta
.get("update")
.map(|d| parse_naive_date_str(d.as_str().unwrap_or("")))
.unwrap_or(None);
let post_status = meta.get("status").map(|v| v.as_str()).unwrap_or(None);
let post_content = create.payload.content;
if let Err(e) = query(CREATE_POST)
.bind(post_id)
.bind(post_identifier)
.bind(post_title)
.bind(post_subtitle)
.bind(post_date_created)
.bind(post_date_updated)
.bind(post_status)
.bind(post_content)
.bind(category_id)
.bind(tag_ids)
.execute(&mut *tx)
.await
{
error!("failed to insert post {}: {}", &identifier, e);
return;
}
tx.commit().await.unwrap();
}
async fn get_or_create_tag_id(
db: &mut Transaction<'_, Postgres>,
tag_name: &str,
) -> Result<i32, Error> {
get_create_id(db, tag_name, GET_TAG_ID, CREATE_RETURN_TAG_ID).await
}
async fn get_or_create_category_id(
db: &mut Transaction<'_, Postgres>,
category_name: &str,
) -> Result<i32, Error> {
get_create_id(
db,
category_name,
GET_CATEGORY_ID,
CREATE_RETURN_CATEGORY_ID,
)
.await
}
async fn get_create_id(
db: &mut Transaction<'_, Postgres>,
field_name: &str,
fetch_query: &str,
create_fetch_query: &str,
) -> Result<i32, Error> {
if let Some(id) = query(fetch_query)
.bind(field_name)
.fetch_optional(&mut **db)
.await?
{
Ok(id.get("id"))
} else {
let id = query(create_fetch_query)
.bind(field_name)
.fetch_one(&mut **db)
.await?
.get("id");
Ok(id)
}
}
backend/server/src/modules/blog/components/check_update/changes.rs
+68
View File
@@ -0,0 +1,68 @@
use std::path::PathBuf;
use markdown::MdFile;
use uuid::Uuid;
#[allow(dead_code)]
#[derive(Debug)]
pub enum Change {
Delete(Delete),
Move(Move),
Update(Update),
Create(Create),
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct Delete {
pub uuid: Uuid,
pub path: PathBuf,
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct Move {
pub uuid: Uuid,
pub from: PathBuf,
pub to: PathBuf,
pub payload: MdFile,
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct Update {
pub uuid: Uuid,
pub path: PathBuf,
pub payload: MdFile,
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct Create {
pub uuid: Uuid,
pub path: PathBuf,
pub payload: MdFile,
}
#[derive(Debug, Clone, Default)]
pub struct CreateDelete2UpdateSlot {
pub create: Option<Create>,
pub delete: Option<Delete>,
}
impl CreateDelete2UpdateSlot {
pub fn to_change(&self) -> Option<Change> {
match (&self.create, &self.delete) {
(Some(create), Some(delete)) => Some(Change::Move(Move {
uuid: create.uuid,
from: delete.path.clone(),
to: create.path.clone(),
payload: create.payload.clone(),
})),
(Some(create), None) => Some(Change::Create(create.clone())),
(None, Some(delete)) => Some(Change::Delete(delete.clone())),
(None, None) => None,
}
}
}
backend/server/src/modules/blog/components/check_update/error.rs
+18
View File
@@ -0,0 +1,18 @@
use markdown::error::Error as MarkdownError;
use thiserror::Error;
#[derive(Error, Debug)]
pub enum Error {
#[error("markdown file error")]
MdFile(#[from] MarkdownError),
#[error("db error")]
Db(#[from] sqlx::Error),
#[error("git error")]
Git2(#[from] git2::Error),
#[error("git remote url mismatch")]
GitUrlMismatch,
}
backend/server/src/modules/blog/components/check_update/mod.rs
+103
View File
@@ -0,0 +1,103 @@
use std::{
env,
path::{Path, PathBuf},
sync::Arc,
};
use crate::{
constants::config::general::{default_value, var_name},
modules::blog::components::check_update::{
apply::apply_deltas, pull::fetch_deltas, utils::expand_path,
},
scheduler::task_func::AsyncTaskFunc,
};
use macros::panic_with_log;
use sqlx::PgPool;
use tracing::{log::error, trace, warn, Level};
// mod error;
// mod index;
mod apply;
mod changes;
mod error;
mod pull;
mod utils;
#[derive(Debug)]
struct Args {
db_connection: PgPool,
git_work_dir: PathBuf,
git_url: String,
git_remote_name: String,
git_remote_branch: String,
}
pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc {
// init
let git_work_dir = {
let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) {
expand_path(default_log_path)
} else {
let fallback_path = expand_path(default_value::WORK_DIR.to_string());
warn!(
"unset env var `{}`, using default path: {}/blog_git",
var_name::WORK_DIR,
fallback_path
);
fallback_path
};
Path::new(&path).join("blog_git")
};
// clone & check update
let git_url = match env::var(var_name::BLOG_GIT_URL) {
Ok(url) => url,
Err(e) => {
panic_with_log!(
Level::ERROR,
"failed to read `{}`: {}",
var_name::BLOG_GIT_URL,
e
)
}
};
let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME)
.unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string());
let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH)
.unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string());
let arc_args = Arc::new(Args {
db_connection,
git_work_dir,
git_url,
git_remote_name,
git_remote_branch,
});
move || Box::pin(workflow(arc_args.clone()))
}
async fn workflow(args: Arc<Args>) {
let deltas = match fetch_deltas(
&args.git_url,
&args.git_remote_name,
&args.git_remote_branch,
&args.git_work_dir,
)
.await
{
Ok(updates) => updates,
Err(e) => {
error!("delta parsing error: {}", e);
return;
}
};
if deltas.is_empty() {
trace!("blog git already up-to-date");
return;
}
apply_deltas(&args.db_connection, deltas).await;
}
backend/server/src/modules/blog/components/check_update/pull.rs
+171
View File
@@ -0,0 +1,171 @@
use std::{collections::HashMap, fs::remove_dir_all, path::PathBuf};
use crate::{
modules::blog::components::check_update::{
changes::{Change, Create, CreateDelete2UpdateSlot, Delete, Update},
error::Error,
},
utils::{git, git::FileDelta},
};
use markdown::MdFile;
use git2::{Delta, Repository};
use tracing::{error, warn};
use uuid::Uuid;
pub async fn fetch_deltas(
git_url: &str,
git_remote_name: &str,
git_remote_branch: &str,
git_work_dir: &PathBuf,
) -> Result<Vec<Change>, Error> {
let repo = match Repository::open(git_work_dir) {
Ok(repo) => repo,
Err(_) => match Repository::clone(git_url, git_work_dir) {
Ok(repo) => repo,
Err(e) => {
error!("failed to clone blog content git({}): {}", &git_url, e);
return Err(e.into());
}
},
};
let mut remote = match repo.find_remote(git_remote_name) {
Ok(remote) => remote,
Err(e) => {
remove_dir_all(git_work_dir).unwrap();
warn!(
"cannot find git remote ({}), reinitializing: {}",
git_remote_name, e
);
return Err(e.into());
}
};
if remote.url().is_some_and(|url| url != git_url) {
remove_dir_all(git_work_dir).unwrap();
warn!(
"git remote url mismatch (expected: {}, actual: {}), reinitializing",
git_url,
remote.url().unwrap()
);
return Err(Error::GitUrlMismatch);
}
let fetched_commit = git::do_fetch(&repo, &[git_remote_branch], &mut remote)?;
let commit_deltas = git::calculate_diff(
&repo,
&repo.reference_to_annotated_commit(&repo.head()?)?,
&fetched_commit,
)?;
// dbg!(&commit_deltas);
let mut changes: Vec<Change> = vec![];
let mut non_delete_deltas: Vec<FileDelta> = vec![];
// process deleted deltas before merging commit to prevent "file not found" os error
for delta in commit_deltas.into_iter() {
// todo: limit scanning scope to post only OR extract merging logic to external component
if delta.status == Delta::Deleted {
let file_path = &git_work_dir.join(
delta
.new_path
.expect("cannot parse modified post `new_path`"),
);
if let Some(id) = extract_post_id(&MdFile::from_file(file_path)?) {
changes.push(Change::Delete(Delete {
uuid: id,
path: file_path.clone(),
}))
} else {
error!("failed to parse uuid for file {}", file_path.display());
// todo: add to unresolved error list
}
} else {
non_delete_deltas.push(delta);
}
}
git::do_merge(&repo, git_remote_branch, fetched_commit)?;
// process adds and modifies deltas
for delta in non_delete_deltas.into_iter() {
let file_path = &git_work_dir.join(
delta
.new_path
.expect("cannot parse modified post `new_path`"),
);
match delta.status {
Delta::Added => {
let md_file = MdFile::from_file(file_path)?;
if let Some(id) = extract_post_id(&md_file) {
changes.push(Change::Create(Create {
uuid: id,
path: file_path.clone(),
payload: md_file,
}))
} else {
error!("failed to parse uuid for file {}", file_path.display());
// todo: add to unresolved error list
}
}
Delta::Modified => {
let md_file = MdFile::from_file(file_path)?;
if let Some(id) = extract_post_id(&md_file) {
changes.push(Change::Update(Update {
uuid: id,
path: file_path.clone(),
payload: md_file,
}))
} else {
error!("failed to parse uuid for file {}", file_path.display());
// todo: add to unresolved error list
}
}
Delta::Deleted => continue,
_ => {
error!("unsupported delta type on file {}", file_path.display());
continue;
}
}
}
// merge adds and deletes deltas to moves
let changes = dbg!(merge_changes(changes));
Ok(changes)
}
fn extract_post_id(post: &MdFile) -> Option<Uuid> {
match post.meta["uuid"].as_str() {
Some(id) => Uuid::parse_str(id).ok(),
None => None,
}
}
fn merge_changes(changes: Vec<Change>) -> Vec<Change> {
let mut slots: HashMap<Uuid, CreateDelete2UpdateSlot> = HashMap::new();
let mut merged: Vec<Change> = Vec::new();
for change in changes {
match change {
Change::Create(create) => {
let id = create.uuid;
slots.entry(id).or_default().create = Some(create);
}
Change::Delete(delete) => {
let id = delete.uuid;
slots.entry(id).or_default().delete = Some(delete);
}
_ => merged.push(change),
}
}
for slot in slots.values() {
if let Some(change) = slot.to_change() {
merged.push(change);
}
}
merged
}
backend/server/src/modules/blog/components/check_update/sql/create_post.sql
+7
View File
@@ -0,0 +1,7 @@
INSERT INTO posts
(id, identifier, title, subtitle, date_created, date_updated, status, content, category)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);
INSERT INTO post_tag (post, tag)
SELECT $1, tag
FROM UNNEST($10) AS t(tag);
backend/server/src/modules/blog/components/check_update/sql/create_return_category_id.sql
+3
View File
@@ -0,0 +1,3 @@
INSERT INTO categories(name)
VALUES ($1)
RETURNING id;
backend/server/src/modules/blog/components/check_update/sql/create_return_tag_id.sql
+3
View File
@@ -0,0 +1,3 @@
INSERT INTO tags(name)
VALUES ($1)
RETURNING id;
backend/server/src/modules/blog/components/check_update/sql/delete_post_by_id.sql
+3
View File
@@ -0,0 +1,3 @@
DELETE
FROM posts
WHERE id = $1;
backend/server/src/modules/blog/components/check_update/sql/get_category_id.sql
+3
View File
@@ -0,0 +1,3 @@
SELECT id
FROM categories
WHERE name = $1;
backend/server/src/modules/blog/components/check_update/sql/get_tag_id.sql
+3
View File
@@ -0,0 +1,3 @@
SELECT id
FROM tags
WHERE name = $1;
backend/server/src/modules/blog/components/check_update/utils.rs
+58
View File
@@ -0,0 +1,58 @@
use std::path::Path;
use crate::constants::{
config::general::{default_value, var_name},
time::DEFAULT_NAIVE_DATE_FORMAT,
};
use chrono::NaiveDate;
use dirs::home_dir;
use tracing::warn;
pub fn expand_path(path: String) -> String {
if path.starts_with('~') {
home_dir()
.map(|dir| {
dir.join(path.strip_prefix("~/").unwrap())
.into_os_string()
.into_string()
.unwrap()
})
.unwrap_or_else(|| {
warn!(
"failed to parse env var `{}`, using default value `{}`",
var_name::WORK_DIR,
default_value::WORK_DIR
);
default_value::WORK_DIR.to_string()
})
} else {
path
}
}
pub fn parse_post_identifier(path: &Path) -> Option<String> {
if let Some(comp) = path.components().nth_back(1) {
comp.as_os_str().to_str().map(|s| s.to_string())
} else {
None
}
}
pub fn parse_naive_date_str(date_str: &str) -> Option<NaiveDate> {
NaiveDate::parse_from_str(date_str, DEFAULT_NAIVE_DATE_FORMAT).ok()
}
#[cfg(test)]
mod tests {
use crate::modules::blog::components::check_update::utils::parse_naive_date_str;
use chrono::NaiveDate;
#[test]
fn test_parse_naive_date() {
assert_eq!(
NaiveDate::from_ymd_opt(2015, 1, 1),
parse_naive_date_str("2015-1-01")
);
}
}
backend/server/src/modules/blog/components/consistency_guard/mod.rs
+1
View File
@@ -0,0 +1 @@
backend/server/src/modules/blog/components/mod.rs
+3
View File
@@ -0,0 +1,3 @@
pub mod check_update;
mod consistency_guard;
mod rag;
backend/server/src/modules/blog/components/rag/mod.rs
+1
View File
@@ -0,0 +1 @@
backend/server/src/modules/blog/mod.rs
+55
View File
@@ -0,0 +1,55 @@
mod api;
mod components;
use crate::modules::blog::components::check_update;
use crate::scheduler::tasks::async_basic::AsyncBasic;
use crate::{
modules::{
Module,
blog::{
api::{
get_page_count,
get_post,
// get_post_list::get_post_list,
},
// cron::check_update,
},
},
scheduler::tasks::CronTask,
};
use axum::{Router, routing::get};
use sqlx::PgPool;
const MOUNT_POINT: &str = "/blog";
pub struct Blog {
pub db_handler: PgPool,
}
impl Module for Blog {
fn get_server_router(&self) -> Router {
let db_handler = self.db_handler.clone();
// todo: move resource init to new func
// todo: remove debug comment
Router::new()
// .route("/get_post_list", get(get_post_list))
.route("/get_page_count", get(get_page_count::handler))
.route("/get_post/{identifier}", get(get_post::handler))
.with_state(db_handler)
}
fn get_mount_point(&self) -> &'static str {
MOUNT_POINT
}
fn get_cron_tasks(&self) -> Vec<(&str, Box<dyn CronTask>)> {
let db_handler = self.db_handler.clone();
vec![(
"Blog Check Update",
AsyncBasic::new(check_update::task(db_handler.clone()), "*/5 * * * * *")
.unwrap()
.to_task(),
)]
}
}
backend/server/src/modules/blog/schema.sql
+65
View File
@@ -0,0 +1,65 @@
CREATE TABLE tags
(
id SERIAL PRIMARY KEY,
name VARCHAR(20) UNIQUE NOT NULL,
description TEXT DEFAULT ''
);
CREATE TABLE categories
(
id SERIAL PRIMARY KEY,
name VARCHAR(20) UNIQUE NOT NULL,
description TEXT DEFAULT ''
);
CREATE TABLE post_status
(
id SERIAL PRIMARY KEY,
name VARCHAR(20) UNIQUE NOT NULL
);
INSERT INTO post_status (name)
VALUES ('draft'),
('published'),
('planned'),
('hidden');
CREATE TABLE posts
(
id UUID PRIMARY KEY,
identifier VARCHAR(100) UNIQUE NOT NULL,
title VARCHAR(255) NOT NULL,
subtitle VARCHAR(255) NOT NULL,
date_created DATE NOT NULL,
date_updated DATE,
status INTEGER REFERENCES post_status (id) DEFAULT 1 NOT NULL,
content TEXT NOT NULL,
category INTEGER REFERENCES categories (id) NOT NULL
);
CREATE TABLE post_tag
(
post UUID REFERENCES posts (id) ON DELETE CASCADE,
tag INTEGER REFERENCES tags (id) ON DELETE CASCADE,
CONSTRAINT post_tag_pk PRIMARY KEY (post, tag)
);
CREATE TABLE metadata
(
pid UUID REFERENCES posts (id) ON DELETE CASCADE NOT NULL,
meta_key VARCHAR(50) NOT NULL,
meta_value TEXT NOT NULL,
CONSTRAINT metadata_pk PRIMARY KEY (pid, meta_key)
);
-- index
CREATE INDEX idx_tags_name ON tags (name);
CREATE INDEX idx_categories_name ON categories (name);
CREATE INDEX idx_posts_identifier ON posts (identifier);
CREATE INDEX idx_posts_status ON posts (status);
CREATE INDEX idx_posts_category ON posts (category);
CREATE INDEX idx_post_tag_post ON post_tag (post);
CREATE INDEX idx_post_tag_tag ON post_tag (tag);
CREATE INDEX idx_metadata_pid ON metadata (pid);
backend/server/src/modules/mod.rs
+82
View File
@@ -0,0 +1,82 @@
use std::{env, num::ParseIntError};
use crate::{
constants::config::server::{default_value, var_name},
scheduler::{Scheduler, tasks::CronTask},
};
use macros::error_panic;
use axum::Router;
use tokio::net::TcpListener;
use tracing::{info, warn};
pub mod blog;
// todo: merge `get_server_router` & `get_mount_point`, make Option
pub trait Module {
fn get_server_router(&self) -> Router;
fn get_mount_point(&self) -> &'static str;
fn get_cron_tasks(&self) -> Vec<(&str, Box<dyn CronTask>)>;
}
pub struct ModuleTree {
pub modules: Vec<Box<dyn Module>>,
}
impl ModuleTree {
pub fn new() -> Self {
ModuleTree { modules: vec![] }
}
pub fn add_module(&mut self, module: impl Module + 'static) -> &mut Self {
self.modules.push(Box::new(module));
self
}
pub async fn blocking_serve(&self) {
let mut scheduler = Scheduler::new();
let mut app = Router::new();
let port: u32 = match env::var(var_name::PORT) {
Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| {
warn!(
"failed to parse `{}`, using default port: {}",
var_name::PORT,
e.to_string()
);
default_value::PORT
}),
Err(_) => default_value::PORT,
};
let listener = match TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await {
Ok(listener) => {
info!("server starting on {}:{}", default_value::HOST, port);
listener
}
Err(e) => {
error_panic!(
"failed to start server on {}:{}: {}",
default_value::HOST,
port,
e.to_string()
);
}
};
for module in &self.modules {
let _ = scheduler.insert_list(module.get_cron_tasks());
app = app.nest(module.get_mount_point(), module.get_server_router());
}
tokio::spawn(async move {
scheduler.start().await;
});
axum::serve(listener, app).await.unwrap_or_else(|e| {
error_panic!("failed to start axum: {}", e.to_string());
});
}
}
backend/api_server/src/scheduler/scheduler_error.rs → backend/server/src/scheduler/error.rs
+9 -4
View File
@@ -1,8 +1,13 @@
use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; use std::{
use std::error::Error; error::Error,
use std::fmt::{Debug, Display, Formatter}; fmt::{Debug, Display, Formatter},
use std::sync::PoisonError; sync::PoisonError,
};
use crate::scheduler::error::SchedulerError::MutexPoison;
// todo: refactor with thiserror
#[allow(dead_code)]
pub enum SchedulerError { pub enum SchedulerError {
IdNotFound, IdNotFound,
IdAlreadyRunning, IdAlreadyRunning,
backend/server/src/scheduler/mod.rs
+54
View File
@@ -0,0 +1,54 @@
use std::{collections::HashMap, sync::Arc};
use crate::scheduler::{error::SchedulerError, tasks::CronTask, utils::execute};
use futures::future::join_all;
use tracing::warn;
pub mod error;
pub mod task_func;
pub mod tasks;
mod utils;
pub type TaskPool = HashMap<String, Arc<Box<dyn CronTask>>>;
#[derive(Default)]
pub struct Scheduler {
task_pool: TaskPool,
}
impl Scheduler {
/// Start all tasks in the scheduler
pub async fn start(&self) {
let fut = self
.task_pool
.clone()
.into_iter()
.map(|(name, task)| execute(name, task))
.collect::<Vec<_>>();
// self.runner.run_all().await
join_all(fut).await;
}
pub fn insert(&mut self, task: (&str, Box<dyn CronTask>)) -> Result<&Self, SchedulerError> {
let (k, v) = task;
if self.task_pool.insert(k.into(), v.into()).is_some() {
warn!("task list duplicated key detected: {}", k);
}
Ok(self)
}
pub fn insert_list(
&mut self,
tasks: Vec<(&str, Box<dyn CronTask>)>,
) -> Result<&Self, SchedulerError> {
for task in tasks {
self.insert(task)?;
}
Ok(self)
}
pub fn new() -> Self {
Self::default()
}
}
backend/server/src/scheduler/task_func.rs
+15
View File
@@ -0,0 +1,15 @@
use std::pin::Pin;
pub trait TaskFunc: Fn() + Send + Sync + 'static {}
impl<T> TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {}
pub trait AsyncTaskFunc:
Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static
{
}
impl<T> AsyncTaskFunc for T where
T: Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + Clone + 'static
{
}
backend/server/src/scheduler/tasks/async_basic.rs
+37
View File
@@ -0,0 +1,37 @@
use std::{error::Error, pin::Pin, str::FromStr};
use crate::{scheduler::task_func::AsyncTaskFunc, scheduler::tasks::CronTask};
use chrono::{DateTime, Utc};
use cron::Schedule;
pub struct AsyncBasic {
schedule: Schedule,
func: Box<dyn AsyncTaskFunc>,
}
impl AsyncBasic {
pub fn new<F>(func: F, cron_expr: &str) -> Result<Self, Box<dyn Error>>
where
F: AsyncTaskFunc,
{
Ok(AsyncBasic {
schedule: Schedule::from_str(cron_expr)?,
func: Box::new(func),
})
}
pub fn to_task(self) -> Box<dyn CronTask> {
Box::new(self)
}
}
impl CronTask for AsyncBasic {
fn get_next_execution(&self) -> Option<DateTime<Utc>> {
self.schedule.upcoming(Utc).next()
}
fn call(&self) -> Pin<Box<(dyn Future<Output = ()> + Send)>> {
Box::pin((self.func)())
}
}
backend/api_server/src/scheduler/tasks/basic.rs → backend/server/src/scheduler/tasks/basic.rs
+8 -8
View File
@@ -1,9 +1,9 @@
use crate::scheduler::task_func::TaskFunc; use std::{error::Error, pin::Pin, str::FromStr};
use crate::scheduler::tasks::Task;
use crate::scheduler::{task_func::TaskFunc, tasks::CronTask};
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use cron::Schedule; use cron::Schedule;
use std::error::Error;
use std::str::FromStr;
pub struct BasicTask { pub struct BasicTask {
schedule: Schedule, schedule: Schedule,
@@ -21,17 +21,17 @@ impl BasicTask {
}) })
} }
pub fn to_task(self) -> Box<dyn Task> { pub fn to_task(self) -> Box<dyn CronTask> {
Box::new(self) Box::new(self)
} }
} }
impl Task for BasicTask { impl CronTask for BasicTask {
fn get_next_execution(&self) -> Option<DateTime<Utc>> { fn get_next_execution(&self) -> Option<DateTime<Utc>> {
self.schedule.upcoming(Utc).next() self.schedule.upcoming(Utc).next()
} }
fn call(&self) { fn call(&self) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {
(self.func)() Box::pin(async { (self.func)() })
} }
} }
backend/server/src/scheduler/tasks/mod.rs
+14
View File
@@ -0,0 +1,14 @@
use std::pin::Pin;
use chrono::{DateTime, Utc};
#[allow(dead_code)]
pub mod async_basic;
#[allow(dead_code)]
pub mod basic;
pub trait CronTask: Send + Sync {
fn get_next_execution(&self) -> Option<DateTime<Utc>>;
fn call(&self) -> Pin<Box<dyn Future<Output = ()> + Send + '_>>;
}
backend/server/src/scheduler/utils.rs
+32
View File
@@ -0,0 +1,32 @@
use std::{sync::Arc, time::Duration};
use crate::scheduler::{error::SchedulerError, tasks::CronTask};
use chrono::{DateTime, Utc};
use tokio::time::{Sleep, sleep};
pub fn duration_from_now(future: DateTime<Utc>) -> Duration {
future
.signed_duration_since(Utc::now())
.to_std()
.unwrap_or(Duration::ZERO)
}
pub fn sleeper_from_now(future: DateTime<Utc>) -> Sleep {
sleep(duration_from_now(future))
}
#[allow(unused_variables)]
pub async fn execute(name: String, func: Arc<Box<dyn CronTask>>) -> Result<(), SchedulerError> {
// todo: stats
while let Some(next_execution) = func.get_next_execution() {
let sleeper = sleeper_from_now(next_execution);
tokio::pin!(sleeper);
tokio::select! {
_ = sleeper => {
func.call().await
}
}
}
Ok(())
}
backend/api_server/src/utils/git.rs → backend/server/src/utils/git.rs
+18 -47
View File
@@ -1,14 +1,13 @@
/// git merge /// git operations
/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs /// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs
/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 /// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837
/// ///
use git2::{ use git2::{
build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, build,
}; };
use std::path::PathBuf; use std::path::PathBuf;
use tracing::{error, warn}; use tracing::{error, warn};
#[derive(Debug, Eq, PartialEq)] #[derive(Debug, Eq, PartialEq)]
pub struct FileDelta { pub struct FileDelta {
pub old_path: Option<PathBuf>, pub old_path: Option<PathBuf>,
@@ -16,15 +15,13 @@ pub struct FileDelta {
pub status: Delta, pub status: Delta,
} }
fn do_fetch<'a>( pub fn do_fetch<'a>(
repo: &'a Repository, repo: &'a Repository,
refs: &[&str], refs: &[&str],
remote: &'a mut Remote, remote: &'a mut Remote,
) -> Result<AnnotatedCommit<'a>, git2::Error> { ) -> Result<AnnotatedCommit<'a>, git2::Error> {
let mut fetch_option = FetchOptions::new(); let mut fetch_option = FetchOptions::new();
// Always fetch all tags.
// Perform a download and also update tips
fetch_option.download_tags(AutotagOption::All); fetch_option.download_tags(AutotagOption::All);
remote.fetch(refs, Some(&mut fetch_option), None)?; remote.fetch(refs, Some(&mut fetch_option), None)?;
@@ -42,16 +39,9 @@ fn fast_forward(
None => String::from_utf8_lossy(lb.name_bytes()).to_string(), None => String::from_utf8_lossy(lb.name_bytes()).to_string(),
}; };
let msg = format!("fast-forward({}) -> {}", name, rc.id()); let msg = format!("fast-forward({}) -> {}", name, rc.id());
// println!("{}", msg);
lb.set_target(rc.id(), &msg)?; lb.set_target(rc.id(), &msg)?;
repo.set_head(&name)?; repo.set_head(&name)?;
repo.checkout_head(Some( repo.checkout_head(Some(build::CheckoutBuilder::default().force()))?;
build::CheckoutBuilder::default()
// For some reason, the force is required to make the working directory actually get updated
// I suspect we should be adding some logic to handle dirty working directory states,
// but this is just an example so maybe not.
.force(),
))?;
Ok(()) Ok(())
} }
@@ -73,13 +63,12 @@ fn normal_merge(
return Ok(()); return Ok(());
} }
let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?;
// now create the merge commit
let msg = format!("merge({}) -> {}", remote.id(), local.id()); let msg = format!("merge({}) -> {}", remote.id(), local.id());
let sig = repo.signature()?; let sig = repo.signature()?;
let local_commit = repo.find_commit(local.id())?; let local_commit = repo.find_commit(local.id())?;
let remote_commit = repo.find_commit(remote.id())?; let remote_commit = repo.find_commit(remote.id())?;
// Do our merge commit and set the current branch head to that commit.
let _merge_commit = repo.commit( let _merge_commit = repo.commit(
Some("HEAD"), Some("HEAD"),
&sig, &sig,
@@ -88,34 +77,25 @@ fn normal_merge(
&result_tree, &result_tree,
&[&local_commit, &remote_commit], &[&local_commit, &remote_commit],
)?; )?;
// Set working tree to match head.
repo.checkout_head(None)?; repo.checkout_head(None)?;
Ok(()) Ok(())
} }
fn do_merge<'a>( pub fn do_merge<'a>(
repo: &'a Repository, repo: &'a Repository,
remote_branch: &str, remote_branch: &str,
fetch_commit: AnnotatedCommit<'a>, fetch_commit: AnnotatedCommit<'a>,
) -> Result<Vec<FileDelta>, git2::Error> { ) -> Result<(), git2::Error> {
// 1. do a merge analysis
let analysis = repo.merge_analysis(&[&fetch_commit])?; let analysis = repo.merge_analysis(&[&fetch_commit])?;
let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?;
// 2. Do the appropriate merge
if analysis.0.is_fast_forward() { if analysis.0.is_fast_forward() {
// println!("Doing a fast-forward");
// do a fast-forward
let ref_name = format!("refs/heads/{}", remote_branch); let ref_name = format!("refs/heads/{}", remote_branch);
match repo.find_reference(&ref_name) { match repo.find_reference(&ref_name) {
Ok(mut r) => { Ok(mut r) => {
fast_forward(repo, &mut r, &fetch_commit)?; fast_forward(repo, &mut r, &fetch_commit)?;
} }
Err(_) => { Err(_) => {
// The branch doesn't exist, so set the reference to the
// commit directly. Usually this is because you are pulling
// into an empty repository.
repo.reference( repo.reference(
&ref_name, &ref_name,
fetch_commit.id(), fetch_commit.id(),
@@ -132,18 +112,13 @@ fn do_merge<'a>(
} }
}; };
} else if analysis.0.is_normal() { } else if analysis.0.is_normal() {
// do a normal merge
normal_merge(repo, &head_commit, &fetch_commit)?; normal_merge(repo, &head_commit, &fetch_commit)?;
} }
// else {
// // already up-to-date
// println!("Nothing to do...");
// }
calculate_diff(repo, &head_commit, &fetch_commit) Ok(())
} }
fn calculate_diff( pub fn calculate_diff(
repo: &Repository, repo: &Repository,
local: &AnnotatedCommit, local: &AnnotatedCommit,
remote: &AnnotatedCommit, remote: &AnnotatedCommit,
@@ -156,23 +131,12 @@ fn calculate_diff(
if let Err(e) = diff.foreach( if let Err(e) = diff.foreach(
&mut |d, _| { &mut |d, _| {
// let delta = d.status();
//
// let n_path = d.new_file().path();
// let o_path = d.old_file().path();
//
// if let Some(path) = n_path {
// updated_files.insert(path.to_owned());
// }
// if let Some(path) = o_path {
// updated_files.insert(path.to_owned());
// }
updated_files.push(FileDelta { updated_files.push(FileDelta {
old_path: d.old_file().path().map(PathBuf::from), old_path: d.old_file().path().map(PathBuf::from),
new_path: d.new_file().path().map(PathBuf::from), new_path: d.new_file().path().map(PathBuf::from),
status: d.status(), status: d.status(),
}); });
true // Continue true
}, },
None, None,
None, None,
@@ -184,6 +148,7 @@ fn calculate_diff(
Ok(updated_files.into_iter().collect()) Ok(updated_files.into_iter().collect())
} }
#[allow(dead_code)]
pub fn sync( pub fn sync(
remote_name: &str, remote_name: &str,
remote_branch: &str, remote_branch: &str,
@@ -191,5 +156,11 @@ pub fn sync(
) -> Result<Vec<FileDelta>, git2::Error> { ) -> Result<Vec<FileDelta>, git2::Error> {
let mut remote = repo.find_remote(remote_name)?; let mut remote = repo.find_remote(remote_name)?;
let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?;
do_merge(repo, remote_branch, fetch_commit) let delta = calculate_diff(
repo,
&repo.reference_to_annotated_commit(&repo.head()?)?,
&fetch_commit,
);
do_merge(repo, remote_branch, fetch_commit)?;
delta
} }
backend/api_server/src/utils/mod.rs → backend/server/src/utils/mod.rs
View File
File renamed without changes.
client/.gitignore
+11
View File
@@ -0,0 +1,11 @@
.venv/
.idea/
.DS_store
__pycache__
.cache
content/*
client/README.md
+30
View File
@@ -0,0 +1,30 @@
## Module Structure
```shell
.
├── README.md
├── cli
├── content
│ ├── pages
│ │ └── about
│ │ └── <static_page>.md
│ └── posts
│ └── <year>
│ └── <identifier>
│ ├── assets
│ │ └── <assets>
│ ├── translate_<lang>.md
│ └── index.md
├── requirements.txt
├── templates
│ ├── <template>.py
│ └── ...
└── utils
└── ...
```
### About `posts`
- Will render to `https://<DOMAIN>/blog/post/<identifier>`.
- Files under `assets/` (where images etc. located) should be pushed to OSS.
- For identifiers, use prefix `-` to indicate private (hidden) visibility. EX: `-hello-world`.
client/cli
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
import argparse
import rich
from templates.post import BlogPost
from utils.cache import init_cache
from utils.index import build_post_idx
def main():
# if args.new:
# new_post()
init_cache()
build_post_idx()
bp = BlogPost()
bp.init()
parser = argparse.ArgumentParser()
parser.add_argument("-n", "--new", help="Create a post with build-in template", type=str)
args = parser.parse_args(namespace=rich)
# todo: cli
if __name__ == '__main__':
main()
client/requirements.txt
+4
View File
@@ -0,0 +1,4 @@
rich~=13.9.4
prompt_toolkit~=3.0.50
pypinyin~=0.53.0
python-frontmatter~=1.1.0
client/templates/post.py
+87
View File
@@ -0,0 +1,87 @@
import uuid
from datetime import date
from datetime import datetime as dt
from typing import Dict, Optional
from templates.utils import Template, create_all_files
from utils.autocompletion import input_autocompletion, fuzzy_pinyin_matcher
from utils.cache import get_tags_cache, get_categories_cache
from utils.lang import eprint, wprint
POST_TEMPLATE = '''
---
title: {{ title }}
subtitle: {{ subtitle }}
published_date: {{ published_date }}
last_updated: {{ last_update }}
category: {{ category }}
tags: {{ tags }}
uuid: {{ uuid }}
---
WIP
'''
def get_title():
while True:
i = input("title > ")
if i.strip() == "":
eprint("empty title")
continue
else:
return i.strip()
def get_subtitle():
return input("subtitle > ")
def get_published_date():
while True:
d = input("published_date (YYYY-MM-DD) > ")
if d == "":
wprint("using today's date")
return dt.today().strftime("%Y-%m-%d")
else:
try:
_validate = dt.strptime(d, "%Y-%m-%d")
return d
except ValueError:
eprint("invalid date")
continue
def get_tags():
# todo: multi tags support
return input_autocompletion("tags > ", get_tags_cache(), fuzzy_pinyin_matcher)
def get_category():
while True:
i = input_autocompletion("category > ", get_categories_cache(), fuzzy_pinyin_matcher)
if i.strip() == "":
eprint("empty category")
continue
else:
return i.strip()
class BlogPost(Template):
WORK_PATH: str = "content/posts/{year}/{identifier}/"
ITEMS: Dict[str, Optional[str]] = {"index.md": POST_TEMPLATE, "assets/": None}
def init(self):
year = date.today().year
index_md = POST_TEMPLATE.format(
title=get_title(),
subtitle=get_subtitle(),
published_date=get_published_date(),
last_update=dt.today().strftime("%Y-%m-%d"),
tags=get_tags(),
category=get_category(),
uuid=uuid.uuid4()
)
create_all_files(self.WORK_PATH.format(year=year, identifier="test"), self.ITEMS)
client/templates/utils.py
+17
View File
@@ -0,0 +1,17 @@
import os
from abc import ABC, abstractmethod
from typing import Dict, Optional
class Template(ABC):
@abstractmethod
def init(self):
pass
def create_all_files(relative_work_path: str, items: Dict[str, Optional[str]]):
absolute_path = os.path.abspath(relative_work_path)
for item in items:
if item.endswith("/"):
print(absolute_path + item)
# todo: create files
client/utils/ansi.py
+31
View File
@@ -0,0 +1,31 @@
class Ansi:
HEADER = '\033[95m'
BLUE = '\033[94m'
GREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
def disable(self) -> None:
self.HEADER = ''
self.BLUE = ''
self.GREEN = ''
self.WARNING = ''
self.FAIL = ''
self.BOLD = ''
self.UNDERLINE = ''
self.END = ''
@staticmethod
def blend(color: list[str], text: str) -> str:
return f"{"".join(color)}{text}{Ansi.END}"
@staticmethod
def bold(text: str) -> str:
return f"{Ansi.BOLD}{text}{Ansi.END}"
@staticmethod
def underline(text: str) -> str:
return f"{Ansi.UNDERLINE}{text}{Ansi.END}"
client/utils/autocompletion.py
+46
View File
@@ -0,0 +1,46 @@
from collections.abc import Callable
from typing import List, Iterable
import prompt_toolkit
from prompt_toolkit.completion import Completer, CompleteEvent, Completion
from prompt_toolkit.document import Document
from pypinyin import lazy_pinyin, Style
class AutoCompleter(Completer):
def __init__(self, candidates: List[str], matching_algorithm: Callable[[str, str], bool]):
self.candidates = candidates
self.matching_algorithm = matching_algorithm
def get_completions(self, document: Document, complete_event: CompleteEvent) -> Iterable[Completion]:
curr_word = document.text_before_cursor
for c in self.candidates:
if self.matching_algorithm(curr_word, c):
yield Completion(c, start_position=-len(curr_word))
def exact_matcher(word: str, candidate: str, case_sensitive: bool = False) -> bool:
if case_sensitive:
return word == candidate
else:
return word.lower() == candidate.lower()
def fuzzy_matcher(word: str, candidate: str) -> bool:
return word.lower() in candidate.lower()
def fuzzy_pinyin_matcher(word: str, candidate: str) -> bool:
try:
word_matcher = ''.join(lazy_pinyin(word, style=Style.NORMAL))
candidate_matcher = ''.join(lazy_pinyin(candidate, style=Style.NORMAL))
return candidate_matcher.startswith(word_matcher)
except e:
return fuzzy_matcher(word, candidate)
def input_autocompletion(prompt: str, candidates: List[str],
matching_algorithm: Callable[[str, str], bool] = exact_matcher) -> str:
completer = AutoCompleter(candidates, matching_algorithm)
return prompt_toolkit.prompt(prompt, completer=completer, complete_in_thread=True)
client/utils/cache.py
+49
View File
@@ -0,0 +1,49 @@
import json
import os
from typing import TextIO
CACHE_DIR = ".cache"
ITEMS = {"register": "register.json"}
def overwrite_json_file(file: TextIO, json_data: dict):
file.seek(0)
file.write(json.dumps(json_data, indent=4, ensure_ascii=False))
file.truncate()
def init_cache():
if not os.path.exists(CACHE_DIR):
os.makedirs(CACHE_DIR)
for item in ITEMS.values():
item_path = os.path.join(CACHE_DIR, item)
if not os.path.exists(item_path):
with open(item_path, 'w') as cache_file:
match item:
case "register.json":
overwrite_json_file(cache_file, {})
break
def write_tags_cache(tags: list[str]):
with open(os.path.join(CACHE_DIR, ITEMS["register"]), 'r+') as tags_cache_file:
cache = json.load(tags_cache_file)
cache["tags"] = tags
overwrite_json_file(tags_cache_file, cache)
def write_category_cache(categories: list[str]):
with open(os.path.join(CACHE_DIR, ITEMS["register"]), 'r+') as categories_cache_file:
cache = json.load(categories_cache_file)
cache["categories"] = categories
overwrite_json_file(categories_cache_file, cache)
def get_tags_cache() -> list[str]:
with open(os.path.join(CACHE_DIR, ITEMS["register"]), 'r') as tags_cache_file:
return json.load(tags_cache_file)["tags"]
def get_categories_cache() -> list[str]:
with open(os.path.join(CACHE_DIR, ITEMS["register"]), 'r') as categories_cache_file:
return json.load(categories_cache_file)["categories"]
client/utils/index.py
+27
View File
@@ -0,0 +1,27 @@
import os
import frontmatter
from utils.cache import write_tags_cache, write_category_cache
from utils.lang import list_exclude
EXCLUDE = [".DS_Store"]
POSTS_DIR = "content/posts"
def build_post_idx():
tags = set()
categories = set()
for year in list_exclude(os.listdir(POSTS_DIR), EXCLUDE):
for post_dir in list_exclude(os.listdir(os.path.join(POSTS_DIR, year)), EXCLUDE):
target_file = os.path.join(POSTS_DIR, year, post_dir, "index.md")
with open(target_file, "r") as f:
post = frontmatter.loads(f.read())
try:
for tag in post.metadata["tags"]:
tags.add(tag)
categories.add(post.metadata["category"])
except KeyError:
print(f"failed to parse {target_file}")
write_tags_cache(list(tags))
write_category_cache(list(categories))
client/utils/lang.py
+13
View File
@@ -0,0 +1,13 @@
from utils.ansi import Ansi
def list_exclude(source: list, exclude: list):
return filter(lambda item: item not in exclude, source)
def eprint(msg: str):
print(Ansi.blend([Ansi.FAIL], "error: ") + msg)
def wprint(msg: str):
print(Ansi.blend([Ansi.WARNING], "warn: ") + msg)
frontend/app/about/layout.tsx → frontend/app/(pages)/about/layout.tsx
+6
View File
@@ -1,4 +1,10 @@
import React from 'react'; import React from 'react';
import { Metadata } from 'next';
import { SITE_CONFIG } from '@/src/constants';
export const metadata: Metadata = {
title: `About - ${ SITE_CONFIG.name }`
};
function Layout({ function Layout({
children children
frontend/app/about/me/page.tsx → frontend/app/(pages)/about/me/page.tsx
+1 -1
View File
@@ -1,5 +1,5 @@
import React from 'react'; import React from 'react';
import MarkdownRenderer from '@/app/public/MarkdownRenderer'; import MarkdownRenderer from '@/app/(pages)/public/MarkdownRenderer';
import { fetchText } from '@/app/fetch'; import { fetchText } from '@/app/fetch';
const Me = async () => { const Me = async () => {
frontend/app/about/page.tsx → frontend/app/(pages)/about/page.tsx
View File
File renamed without changes.
frontend/app/about/website/page.tsx → frontend/app/(pages)/about/website/page.tsx
+1 -1
View File
@@ -1,5 +1,5 @@
import React from 'react'; import React from 'react';
import MarkdownRenderer from '@/app/public/MarkdownRenderer'; import MarkdownRenderer from '@/app/(pages)/public/MarkdownRenderer';
import { fetchText } from '@/app/fetch'; import { fetchText } from '@/app/fetch';
const Website = async () => { const Website = async () => {
frontend/app/blog/BlogPostCard.tsx → frontend/app/(pages)/blog/BlogPostCard.tsx
View File
File renamed without changes.
frontend/app/blog/Error.tsx → frontend/app/(pages)/blog/Error.tsx
View File
File renamed without changes.
frontend/app/blog/Pagination.tsx → frontend/app/(pages)/blog/Pagination.tsx
+1 -1
View File
@@ -3,7 +3,7 @@
import React from 'react'; import React from 'react';
import { Pagination as NextUIPagination } from '@nextui-org/react'; import { Pagination as NextUIPagination } from '@nextui-org/react';
import { useRouter, useSearchParams } from 'next/navigation'; import { useRouter, useSearchParams } from 'next/navigation';
import { useScreenSizeTrigger } from '@/app/public/screenSizeTrigger'; import { useScreenSizeTrigger } from '@/app/(pages)/public/screenSizeTrigger';
const Pagination = ({ total }: { const Pagination = ({ total }: {
total: number, total: number,
frontend/app/blog/PostList.tsx → frontend/app/(pages)/blog/PostList.tsx
+6 -5
View File
@@ -1,11 +1,12 @@
'use server'; 'use server';
import React from 'react'; import React from 'react';
import { fetchNocturneApi } from '@/app/blog/api'; import { fetchNocturneApi } from '@/app/(pages)/blog/api';
import { MotionDiv } from '@/app/public/MotionDiv'; import BlogPostCard from '@/app/(pages)/blog/BlogPostCard';
import BlogPostCard from '@/app/blog/BlogPostCard';
import { Card, CardBody } from '@nextui-org/card'; import { Card, CardBody } from '@nextui-org/card';
import * as motion from 'motion/react-client';
const PostList = async ({ page }: { page: number }) => { const PostList = async ({ page }: { page: number }) => {
try { try {
const res = await fetchNocturneApi(`/blog/get_post_list?page=${ page }`); const res = await fetchNocturneApi(`/blog/get_post_list?page=${ page }`);
@@ -16,13 +17,13 @@ const PostList = async ({ page }: { page: number }) => {
<ul className={ 'flex flex-col items-center w-full space-y-7' }> <ul className={ 'flex flex-col items-center w-full space-y-7' }>
{ posts.map((post: JSON, key: number) => ( { posts.map((post: JSON, key: number) => (
<li key={ key } className={ 'w-full' }> <li key={ key } className={ 'w-full' }>
<MotionDiv <motion.div
initial={ { y: 20, opacity: 0 } } initial={ { y: 20, opacity: 0 } }
animate={ { y: 0, opacity: 1 } } animate={ { y: 0, opacity: 1 } }
transition={ { ease: 'easeInOut', duration: 0.5, delay: key * .2 + .25 } } transition={ { ease: 'easeInOut', duration: 0.5, delay: key * .2 + .25 } }
> >
<BlogPostCard post={ post } /> <BlogPostCard post={ post } />
</MotionDiv> </motion.div>
</li>) </li>)
) } ) }
</ul> </ul>
frontend/app/blog/api.ts → frontend/app/(pages)/blog/api.ts
View File
File renamed without changes.
frontend/app/blog/layout.tsx → frontend/app/(pages)/blog/layout.tsx
+6
View File
@@ -1,4 +1,10 @@
import React from 'react'; import React from 'react';
import { Metadata } from 'next';
import { SITE_CONFIG } from '@/src/constants';
export const metadata: Metadata = {
title: `Blog - ${ SITE_CONFIG.name }`
};
function Layout({ function Layout({
children children
frontend/app/blog/page.tsx → frontend/app/(pages)/blog/page.tsx
+3 -3
View File
@@ -1,6 +1,6 @@
import PostList from '@/app/blog/PostList'; import PostList from '@/app/(pages)/blog/PostList';
import Pagination from '@/app/blog/Pagination'; import Pagination from '@/app/(pages)/blog/Pagination';
import { fetchNocturneApi } from '@/app/blog/api'; import { fetchNocturneApi } from '@/app/(pages)/blog/api';
async function Blog({ async function Blog({
searchParams searchParams
frontend/app/blog/post/[id]/ButtonGoBack.tsx → frontend/app/(pages)/blog/post/[id]/ButtonGoBack.tsx
View File
File renamed without changes.
frontend/app/blog/post/[id]/Footer.tsx → frontend/app/(pages)/blog/post/[id]/Footer.tsx
+21 -13
View File
@@ -1,24 +1,32 @@
import React from 'react'; import React from 'react';
import MinifyPostCard from '@/app/blog/post/[id]/MinifyPostCard'; import MinifyPostCard from '@/app/(pages)/blog/post/[id]/MinifyPostCard';
import Link from 'next/link'; import Link from 'next/link';
import { FaCreativeCommons } from 'react-icons/fa6'; import { FaCreativeCommons } from 'react-icons/fa6';
type Props = { type Props = {
post: any // post: any
post_id: string;
adjacent_posts: any
perm_link: string | undefined;
author: string | undefined;
author_link: string | undefined;
first_update: string;
last_update: string | undefined;
cc: string | undefined;
title: string;
} }
const Footer = (props: Props) => { const Footer = (props: Props) => {
const { post } = props; const { adjacent_posts, perm_link, post_id, first_update, last_update, title } = props;
const adjacent_posts = post['adj']; const prev_post = adjacent_posts.prev;
const prev_post = adjacent_posts['prev']; const next_post = adjacent_posts.next;
const next_post = adjacent_posts['next']; const link = perm_link ?? `https://lumither.com/blog/post/${ post_id }`;
const link = post['meta']['link'] ?? `https://lumither.com/blog/post/${ post['meta']['id'] }`; const author = props.author ?? 'Tao';
const author = post['meta']['author'] ?? 'Tao'; const author_link = props.author_link ?? '/about/me';
const author_link = post['meta']['author_link'] ?? '/about'; const post_date = new Date(first_update);
const post_date = new Date(post['first_update']); const update_date: Date | undefined = last_update ? new Date(last_update) : undefined;
const update_date: Date | undefined = post['last_update'] ? new Date(post['last_update']) : undefined; const cc = props.cc ?? 'by-nc-sa';
const cc = post['meta']['cc'] ?? 'by-nc-sa';
return ( return (
@@ -35,7 +43,7 @@ const Footer = (props: Props) => {
<div className={ 'flex flex-col space-y-4 relative' }> <div className={ 'flex flex-col space-y-4 relative' }>
<div className={ 'z-[2]' }> <div className={ 'z-[2]' }>
<p className={ 'text-xl font-bold' }>{ post['meta']['title'] }</p> <p className={ 'text-xl font-bold' }>{ title }</p>
<Link href={ link }> <Link href={ link }>
<p className={ 'text-sm underline decoration-indigo-500 dark:decoration-sky-500' }>{ link }</p> <p className={ 'text-sm underline decoration-indigo-500 dark:decoration-sky-500' }>{ link }</p>
</Link> </Link>
Loaded 100 of 138 files, more files were not shown because too many files have changed in this diff. Show more