From 8938c964094910777afe2c8c91b0d5beb6b8e205 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 5 Feb 2025 13:57:20 -0800 Subject: [PATCH 01/32] feat(frontend) add `Activities.tsx` --- README.md | 40 +++-------- .../src/blog/cron/check_update/mod.rs | 22 +++--- .../src/blog/cron/check_update/utils.rs | 2 +- backend/api_server/src/blog/cron/mod.rs | 3 +- frontend/app/{ => (pages)}/about/layout.tsx | 6 ++ frontend/app/{ => (pages)}/about/me/page.tsx | 2 +- frontend/app/{ => (pages)}/about/page.tsx | 0 .../app/{ => (pages)}/about/website/page.tsx | 2 +- .../app/{ => (pages)}/blog/BlogPostCard.tsx | 0 frontend/app/{ => (pages)}/blog/Error.tsx | 0 .../app/{ => (pages)}/blog/Pagination.tsx | 2 +- frontend/app/{ => (pages)}/blog/PostList.tsx | 11 +-- frontend/app/{ => (pages)}/blog/api.ts | 0 frontend/app/{ => (pages)}/blog/layout.tsx | 6 ++ frontend/app/{ => (pages)}/blog/page.tsx | 6 +- .../blog/post/[id]/ButtonGoBack.tsx | 0 .../{ => (pages)}/blog/post/[id]/Footer.tsx | 34 +++++---- .../{ => (pages)}/blog/post/[id]/Headers.tsx | 46 ++++++------ .../blog/post/[id]/MinifyPostCard.tsx | 8 +-- .../app/(pages)/blog/post/[id]/PostCard.tsx | 55 +++++++++++++++ frontend/app/(pages)/blog/post/[id]/page.tsx | 70 +++++++++++++++++++ frontend/app/{ => (pages)}/friends/layout.tsx | 6 ++ frontend/app/{ => (pages)}/friends/page.tsx | 0 frontend/app/(pages)/layout.tsx | 30 ++++++++ .../{ => (pages)}/public/MarkdownRenderer.tsx | 0 .../app/{ => (pages)}/public/css/article.css | 0 .../{ => (pages)}/public/screenSizeTrigger.ts | 0 .../app/{ => (pages)}/public/type/Result.ts | 0 frontend/app/{ => (pages)}/search/page.tsx | 0 frontend/app/Activities.tsx | 43 ++++++++++++ frontend/app/blog/post/[id]/PostCard.tsx | 53 -------------- frontend/app/blog/post/[id]/page.tsx | 29 -------- frontend/app/entries.tsx | 56 +++++++++++++++ frontend/app/layout.tsx | 19 ++--- frontend/app/page.tsx | 47 ++++++++++++- frontend/app/public/MotionDiv.tsx | 5 -- frontend/components/Navbar.tsx | 57 +-------------- frontend/components/ThemeSwitcher.tsx | 2 +- frontend/package.json | 3 +- frontend/src/HomePageActivities/activity.tsx | 18 +++++ frontend/src/HomePageActivities/constants.ts | 1 + .../ghActivities/starRepo.tsx | 34 +++++++++ .../src/HomePageActivities/ghActivity.tsx | 32 +++++++++ frontend/src/constants.ts | 4 ++ 44 files changed, 501 insertions(+), 253 deletions(-) rename frontend/app/{ => (pages)}/about/layout.tsx (57%) rename frontend/app/{ => (pages)}/about/me/page.tsx (83%) rename frontend/app/{ => (pages)}/about/page.tsx (100%) rename frontend/app/{ => (pages)}/about/website/page.tsx (84%) rename frontend/app/{ => (pages)}/blog/BlogPostCard.tsx (100%) rename frontend/app/{ => (pages)}/blog/Error.tsx (100%) rename frontend/app/{ => (pages)}/blog/Pagination.tsx (92%) rename frontend/app/{ => (pages)}/blog/PostList.tsx (83%) rename frontend/app/{ => (pages)}/blog/api.ts (100%) rename frontend/app/{ => (pages)}/blog/layout.tsx (67%) rename frontend/app/{ => (pages)}/blog/page.tsx (77%) rename frontend/app/{ => (pages)}/blog/post/[id]/ButtonGoBack.tsx (100%) rename frontend/app/{ => (pages)}/blog/post/[id]/Footer.tsx (80%) rename frontend/app/{ => (pages)}/blog/post/[id]/Headers.tsx (64%) rename frontend/app/{ => (pages)}/blog/post/[id]/MinifyPostCard.tsx (84%) create mode 100644 frontend/app/(pages)/blog/post/[id]/PostCard.tsx create mode 100644 frontend/app/(pages)/blog/post/[id]/page.tsx rename frontend/app/{ => (pages)}/friends/layout.tsx (57%) rename frontend/app/{ => (pages)}/friends/page.tsx (100%) create mode 100644 frontend/app/(pages)/layout.tsx rename frontend/app/{ => (pages)}/public/MarkdownRenderer.tsx (100%) rename frontend/app/{ => (pages)}/public/css/article.css (100%) rename frontend/app/{ => (pages)}/public/screenSizeTrigger.ts (100%) rename frontend/app/{ => (pages)}/public/type/Result.ts (100%) rename frontend/app/{ => (pages)}/search/page.tsx (100%) create mode 100644 frontend/app/Activities.tsx delete mode 100644 frontend/app/blog/post/[id]/PostCard.tsx delete mode 100644 frontend/app/blog/post/[id]/page.tsx create mode 100644 frontend/app/entries.tsx delete mode 100644 frontend/app/public/MotionDiv.tsx create mode 100644 frontend/src/HomePageActivities/activity.tsx create mode 100644 frontend/src/HomePageActivities/constants.ts create mode 100644 frontend/src/HomePageActivities/ghActivities/starRepo.tsx create mode 100644 frontend/src/HomePageActivities/ghActivity.tsx create mode 100644 frontend/src/constants.ts diff --git a/README.md b/README.md index c8ea3a4..36ab327 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,16 @@ -# `Nocturne: personal website & CMS` +# `Nocturne: Personal Website` -This is the code repo of my personal website, and [here](https://github.com/Lumither/blog-posts/) is the link of the -content repo. +This is the code repo of my [personal website](https://lumither.com), +and [here](https://github.com/Lumither/blog-posts/) is the link to the +content (blog posts etc.) repo. Powered by Next.js and Rust, built with passion and love -

- GitHub Avatar -

+> [!NOTE] +> This system is not production ready, and contains bugs, (security risk potentially). +> But have fun on it =) -*will add more here in the future - -## Nav: - - - -## Todos: - -### Backend - -- [ ] Add: log system -- [ ] Add: search api -- [ ] Add: tag filter (api) -- [ ] Abandon: UUID mechanism -- [ ] Fix: db connection issue - -### Frontend - -- [ ] Page: Home -- [ ] Page: About -- [ ] Page: Tags -- [ ] Page: Archive -- [ ] Page: Search -- [ ] Add: Right sidebar +# Todos +- [ ] the todo list lol +- [ ] and much more... diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs index 9568ae2..2d531e3 100644 --- a/backend/api_server/src/blog/cron/check_update/mod.rs +++ b/backend/api_server/src/blog/cron/check_update/mod.rs @@ -1,4 +1,5 @@ use std::env; +use std::error::Error; use std::fs::remove_dir_all; use std::path::{Path, PathBuf}; @@ -89,24 +90,22 @@ pub fn task(db_connection: &Pool) { for update in updates { if [Delta::Deleted, Delta::Untracked].contains(&update.status) { if let Some(path) = &update.old_path { - if drop_index( + match drop_index( &get_md_file_basename( path.file_name().unwrap().to_str().unwrap(), ), db_connection, - ) - .is_ok() - { - info!( + ) { + Ok(_) => info!( "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", &git_url, &path.display() - ); - } else { - error!( - "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", + ), + Err(e) => error!( + "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index: {}", &git_url, - &path.display() + &path.display(), + e ) } } @@ -114,7 +113,8 @@ pub fn task(db_connection: &Pool) { if path.components().any(|comp| comp.as_os_str() == "posts") && path.extension().is_some_and(|ext| ext == "md") { - update_md.push(path.to_path_buf()); + // update_md.push(path.to_path_buf()); + update_md.push(dbg!(path.to_path_buf())); } } } diff --git a/backend/api_server/src/blog/cron/check_update/utils.rs b/backend/api_server/src/blog/cron/check_update/utils.rs index d0da864..c20d98a 100644 --- a/backend/api_server/src/blog/cron/check_update/utils.rs +++ b/backend/api_server/src/blog/cron/check_update/utils.rs @@ -38,7 +38,7 @@ pub fn search_md(entry: PathBuf) -> Vec { pub fn index_files(files: &[PathBuf], db_conn: &Pool) { let rt = Runtime::new().unwrap(); rt.block_on(async { - let files_clone = files.to_owned(); + let files_clone = dbg!(files).to_owned(); let handlers = files_clone .into_iter() .map(|file: PathBuf| { diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs index 686fb26..1cb8398 100644 --- a/backend/api_server/src/blog/cron/mod.rs +++ b/backend/api_server/src/blog/cron/mod.rs @@ -7,7 +7,8 @@ pub mod check_update; pub fn get_tasks(db_connection: Pool) -> Vec> { let db_conn_clone = db_connection.clone(); vec![ - BasicTask::new(move || check_update::task(&db_conn_clone), "0/15 * * * * *") + // BasicTask::new(move || check_update::task(&db_conn_clone), "0/15 * * * * *") + BasicTask::new(move || check_update::task(&db_conn_clone), "* * * * * *") .unwrap() .to_task(), ] diff --git a/frontend/app/about/layout.tsx b/frontend/app/(pages)/about/layout.tsx similarity index 57% rename from frontend/app/about/layout.tsx rename to frontend/app/(pages)/about/layout.tsx index ccc425b..e84ea45 100644 --- a/frontend/app/about/layout.tsx +++ b/frontend/app/(pages)/about/layout.tsx @@ -1,4 +1,10 @@ import React from 'react'; +import { Metadata } from 'next'; +import { SITE_CONFIG } from '@/src/constants'; + +export const metadata: Metadata = { + title: `About - ${ SITE_CONFIG.name }` +}; function Layout({ children diff --git a/frontend/app/about/me/page.tsx b/frontend/app/(pages)/about/me/page.tsx similarity index 83% rename from frontend/app/about/me/page.tsx rename to frontend/app/(pages)/about/me/page.tsx index 86cccf5..03821e9 100644 --- a/frontend/app/about/me/page.tsx +++ b/frontend/app/(pages)/about/me/page.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import MarkdownRenderer from '@/app/public/MarkdownRenderer'; +import MarkdownRenderer from '@/app/(pages)/public/MarkdownRenderer'; import { fetchText } from '@/app/fetch'; const Me = async () => { diff --git a/frontend/app/about/page.tsx b/frontend/app/(pages)/about/page.tsx similarity index 100% rename from frontend/app/about/page.tsx rename to frontend/app/(pages)/about/page.tsx diff --git a/frontend/app/about/website/page.tsx b/frontend/app/(pages)/about/website/page.tsx similarity index 84% rename from frontend/app/about/website/page.tsx rename to frontend/app/(pages)/about/website/page.tsx index 0fc5d5a..0d112cb 100644 --- a/frontend/app/about/website/page.tsx +++ b/frontend/app/(pages)/about/website/page.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import MarkdownRenderer from '@/app/public/MarkdownRenderer'; +import MarkdownRenderer from '@/app/(pages)/public/MarkdownRenderer'; import { fetchText } from '@/app/fetch'; const Website = async () => { diff --git a/frontend/app/blog/BlogPostCard.tsx b/frontend/app/(pages)/blog/BlogPostCard.tsx similarity index 100% rename from frontend/app/blog/BlogPostCard.tsx rename to frontend/app/(pages)/blog/BlogPostCard.tsx diff --git a/frontend/app/blog/Error.tsx b/frontend/app/(pages)/blog/Error.tsx similarity index 100% rename from frontend/app/blog/Error.tsx rename to frontend/app/(pages)/blog/Error.tsx diff --git a/frontend/app/blog/Pagination.tsx b/frontend/app/(pages)/blog/Pagination.tsx similarity index 92% rename from frontend/app/blog/Pagination.tsx rename to frontend/app/(pages)/blog/Pagination.tsx index c3b8033..3d381ab 100644 --- a/frontend/app/blog/Pagination.tsx +++ b/frontend/app/(pages)/blog/Pagination.tsx @@ -3,7 +3,7 @@ import React from 'react'; import { Pagination as NextUIPagination } from '@nextui-org/react'; import { useRouter, useSearchParams } from 'next/navigation'; -import { useScreenSizeTrigger } from '@/app/public/screenSizeTrigger'; +import { useScreenSizeTrigger } from '@/app/(pages)/public/screenSizeTrigger'; const Pagination = ({ total }: { total: number, diff --git a/frontend/app/blog/PostList.tsx b/frontend/app/(pages)/blog/PostList.tsx similarity index 83% rename from frontend/app/blog/PostList.tsx rename to frontend/app/(pages)/blog/PostList.tsx index 591c6ea..c815d5c 100644 --- a/frontend/app/blog/PostList.tsx +++ b/frontend/app/(pages)/blog/PostList.tsx @@ -1,11 +1,12 @@ 'use server'; import React from 'react'; -import { fetchNocturneApi } from '@/app/blog/api'; -import { MotionDiv } from '@/app/public/MotionDiv'; -import BlogPostCard from '@/app/blog/BlogPostCard'; +import { fetchNocturneApi } from '@/app/(pages)/blog/api'; +import BlogPostCard from '@/app/(pages)/blog/BlogPostCard'; import { Card, CardBody } from '@nextui-org/card'; +import * as motion from 'motion/react-client'; + const PostList = async ({ page }: { page: number }) => { try { const res = await fetchNocturneApi(`/blog/get_post_list?page=${ page }`); @@ -16,13 +17,13 @@ const PostList = async ({ page }: { page: number }) => {
    { posts.map((post: JSON, key: number) => (
  • - - +
    • ) ) }
diff --git a/frontend/app/blog/api.ts b/frontend/app/(pages)/blog/api.ts similarity index 100% rename from frontend/app/blog/api.ts rename to frontend/app/(pages)/blog/api.ts diff --git a/frontend/app/blog/layout.tsx b/frontend/app/(pages)/blog/layout.tsx similarity index 67% rename from frontend/app/blog/layout.tsx rename to frontend/app/(pages)/blog/layout.tsx index 2c3ea28..7a16d98 100644 --- a/frontend/app/blog/layout.tsx +++ b/frontend/app/(pages)/blog/layout.tsx @@ -1,4 +1,10 @@ import React from 'react'; +import { Metadata } from 'next'; +import { SITE_CONFIG } from '@/src/constants'; + +export const metadata: Metadata = { + title: `Blog - ${ SITE_CONFIG.name }` +}; function Layout({ children diff --git a/frontend/app/blog/page.tsx b/frontend/app/(pages)/blog/page.tsx similarity index 77% rename from frontend/app/blog/page.tsx rename to frontend/app/(pages)/blog/page.tsx index 6690937..f6b4939 100644 --- a/frontend/app/blog/page.tsx +++ b/frontend/app/(pages)/blog/page.tsx @@ -1,6 +1,6 @@ -import PostList from '@/app/blog/PostList'; -import Pagination from '@/app/blog/Pagination'; -import { fetchNocturneApi } from '@/app/blog/api'; +import PostList from '@/app/(pages)/blog/PostList'; +import Pagination from '@/app/(pages)/blog/Pagination'; +import { fetchNocturneApi } from '@/app/(pages)/blog/api'; async function Blog({ searchParams diff --git a/frontend/app/blog/post/[id]/ButtonGoBack.tsx b/frontend/app/(pages)/blog/post/[id]/ButtonGoBack.tsx similarity index 100% rename from frontend/app/blog/post/[id]/ButtonGoBack.tsx rename to frontend/app/(pages)/blog/post/[id]/ButtonGoBack.tsx diff --git a/frontend/app/blog/post/[id]/Footer.tsx b/frontend/app/(pages)/blog/post/[id]/Footer.tsx similarity index 80% rename from frontend/app/blog/post/[id]/Footer.tsx rename to frontend/app/(pages)/blog/post/[id]/Footer.tsx index 6802851..e0ac2b5 100644 --- a/frontend/app/blog/post/[id]/Footer.tsx +++ b/frontend/app/(pages)/blog/post/[id]/Footer.tsx @@ -1,24 +1,32 @@ import React from 'react'; -import MinifyPostCard from '@/app/blog/post/[id]/MinifyPostCard'; +import MinifyPostCard from '@/app/(pages)/blog/post/[id]/MinifyPostCard'; import Link from 'next/link'; import { FaCreativeCommons } from 'react-icons/fa6'; type Props = { - post: any + // post: any + post_id: string; + adjacent_posts: any + perm_link: string | undefined; + author: string | undefined; + author_link: string | undefined; + first_update: string; + last_update: string | undefined; + cc: string | undefined; + title: string; } const Footer = (props: Props) => { - const { post } = props; + const { adjacent_posts, perm_link, post_id, first_update, last_update, title } = props; - const adjacent_posts = post['adj']; - const prev_post = adjacent_posts['prev']; - const next_post = adjacent_posts['next']; - const link = post['meta']['link'] ?? `https://lumither.com/blog/post/${ post['meta']['id'] }`; - const author = post['meta']['author'] ?? 'Tao'; - const author_link = post['meta']['author_link'] ?? '/about'; - const post_date = new Date(post['first_update']); - const update_date: Date | undefined = post['last_update'] ? new Date(post['last_update']) : undefined; - const cc = post['meta']['cc'] ?? 'by-nc-sa'; + const prev_post = adjacent_posts.prev; + const next_post = adjacent_posts.next; + const link = perm_link ?? `https://lumither.com/blog/post/${ post_id }`; + const author = props.author ?? 'Tao'; + const author_link = props.author_link ?? '/about/me'; + const post_date = new Date(first_update); + const update_date: Date | undefined = last_update ? new Date(last_update) : undefined; + const cc = props.cc ?? 'by-nc-sa'; return ( @@ -35,7 +43,7 @@ const Footer = (props: Props) => {
-

{ post['meta']['title'] }

+

{ title }

{ link }

diff --git a/frontend/app/blog/post/[id]/Headers.tsx b/frontend/app/(pages)/blog/post/[id]/Headers.tsx similarity index 64% rename from frontend/app/blog/post/[id]/Headers.tsx rename to frontend/app/(pages)/blog/post/[id]/Headers.tsx index 62e9090..7c83e51 100644 --- a/frontend/app/blog/post/[id]/Headers.tsx +++ b/frontend/app/(pages)/blog/post/[id]/Headers.tsx @@ -3,57 +3,57 @@ import Image from 'next/image'; import { Chip } from '@nextui-org/react'; import { MdCalendarMonth } from 'react-icons/md'; +// type Props = { +// post: any +// }; + type Props = { - post: any -}; + title: string; + subtitle: string; + summary: string | undefined; + category: string; + tags: string[]; + first_update: string; + header_img: string | undefined; +} -const Headers = (props: Props) => { - let post = props.post; - let title = post['title']; - let sub_title = post['sub_title']; - // let summary = (post as any)['summary']; - let category = post['category']; - let tags = post['tags']; - // let last_update = new Date((post as any)['last_update']); - let first_update = new Date(post['first_update']); - // let post_id = (post as any)['post_id']; - let header_img = post['meta']['header_img'] as string | null; +const Headers = (post: Props) => { return ( <> - { header_img &&
+ { post.header_img &&
{ {
}
-
+
{ category } -

{ title }

-

{ sub_title }

+ variant={ `solid` }>{ post.category } +

{ post.title }

+

{ post.subtitle }

    { - tags.map((tag: string, key: any) => ( + post.tags.map((tag: string, key: any) => (
  • { tag }
    • @@ -64,7 +64,7 @@ const Headers = (props: Props) => {

    - { first_update.toLocaleString('default', { + { (new Date(post.first_update)).toLocaleString('default', { month: `long`, day: `numeric`, year: `numeric` diff --git a/frontend/app/blog/post/[id]/MinifyPostCard.tsx b/frontend/app/(pages)/blog/post/[id]/MinifyPostCard.tsx similarity index 84% rename from frontend/app/blog/post/[id]/MinifyPostCard.tsx rename to frontend/app/(pages)/blog/post/[id]/MinifyPostCard.tsx index 20f6925..ac9212f 100644 --- a/frontend/app/blog/post/[id]/MinifyPostCard.tsx +++ b/frontend/app/(pages)/blog/post/[id]/MinifyPostCard.tsx @@ -43,10 +43,10 @@ const MinifyPostCard = (props: Props) => { } { !img_url &&

    -
    -

    { post['title'] }

    -
    -

    { post['sub_title'] }

    +
    +

    { post['title'] }

    +
    +

    { post['sub_title'] }

    } diff --git a/frontend/app/(pages)/blog/post/[id]/PostCard.tsx b/frontend/app/(pages)/blog/post/[id]/PostCard.tsx new file mode 100644 index 0000000..27b0253 --- /dev/null +++ b/frontend/app/(pages)/blog/post/[id]/PostCard.tsx @@ -0,0 +1,55 @@ +import React from 'react'; +import { Card, CardBody } from '@nextui-org/card'; +import Headers from '@/app/(pages)/blog/post/[id]/Headers'; +import Footer from '@/app/(pages)/blog/post/[id]/Footer'; +import MarkdownRenderer from '@/app/(pages)/public/MarkdownRenderer'; + +type Props = { + post: any +} + +async function PostCard(props: Props) { + let post = props.post; + return ( +
    + + + {/* card headers */ } + + + {/* card body */ } + +
    + + { post.content } + +
    +     + + + {/* card footer */ } +
    + + +
    + ); +} + +export default PostCard; diff --git a/frontend/app/(pages)/blog/post/[id]/page.tsx b/frontend/app/(pages)/blog/post/[id]/page.tsx new file mode 100644 index 0000000..6c2e938 --- /dev/null +++ b/frontend/app/(pages)/blog/post/[id]/page.tsx @@ -0,0 +1,70 @@ +import React, { cache } from 'react'; +import PostCard from '@/app/(pages)/blog/post/[id]/PostCard'; +import ButtonGoBack from '@/app/(pages)/blog/post/[id]/ButtonGoBack'; +import * as motion from 'motion/react-client'; +import { fetchNocturneApi } from '@/app/(pages)/blog/api'; +import { Metadata } from 'next'; +import { SITE_CONFIG } from '@/src/constants'; + +const fetchPost = cache(async (id: string) => { + const res = await fetchNocturneApi(`/blog/get_post/${ id }`); + // throw Error('Not Found'); + return await res.json(); +}); + +export async function generateMetadata( + { params }: { params: Promise<{ id: string }> }) + : Promise { + try { + const title = (await fetchPost((await params).id)).title as string; + return { + title: `${ title } - ${ SITE_CONFIG.name }` + }; + } catch (e) { + return { + title: 'Unexpected Error' + }; + } +} + +async function BlogReader({ + params +}: { + params: Promise<{ id: string }> +}) { + try { + let post = await fetchPost((await params).id); + + return ( + +
    + +
    +
    + +
    +     + ); + } catch (e: any) { + return ( +
    +
    +

    + { 'Unexpected Error:' } +

    +

    + { e.message } +

    +
    +
    + ); + } + +} + +export default BlogReader; diff --git a/frontend/app/friends/layout.tsx b/frontend/app/(pages)/friends/layout.tsx similarity index 57% rename from frontend/app/friends/layout.tsx rename to frontend/app/(pages)/friends/layout.tsx index 37dad5b..c6e9396 100644 --- a/frontend/app/friends/layout.tsx +++ b/frontend/app/(pages)/friends/layout.tsx @@ -1,4 +1,10 @@ import React from 'react'; +import { Metadata } from 'next'; +import { SITE_CONFIG } from '@/src/constants'; + +export const metadata: Metadata = { + title: `Friends - ${ SITE_CONFIG.name }` +}; function Layout({ children diff --git a/frontend/app/friends/page.tsx b/frontend/app/(pages)/friends/page.tsx similarity index 100% rename from frontend/app/friends/page.tsx rename to frontend/app/(pages)/friends/page.tsx diff --git a/frontend/app/(pages)/layout.tsx b/frontend/app/(pages)/layout.tsx new file mode 100644 index 0000000..61f980a --- /dev/null +++ b/frontend/app/(pages)/layout.tsx @@ -0,0 +1,30 @@ +import React from 'react'; +import Navbar from '@/components/Navbar'; +import Footer from '@/components/Footer'; + +import * as motion from 'motion/react-client'; + +function Layout({ + children +}: { + children: React.ReactNode; +}) { + return ( +
    + + +
    + { children } +
    +
    +     +
    + ) + ; +} + +export default Layout; diff --git a/frontend/app/public/MarkdownRenderer.tsx b/frontend/app/(pages)/public/MarkdownRenderer.tsx similarity index 100% rename from frontend/app/public/MarkdownRenderer.tsx rename to frontend/app/(pages)/public/MarkdownRenderer.tsx diff --git a/frontend/app/public/css/article.css b/frontend/app/(pages)/public/css/article.css similarity index 100% rename from frontend/app/public/css/article.css rename to frontend/app/(pages)/public/css/article.css diff --git a/frontend/app/public/screenSizeTrigger.ts b/frontend/app/(pages)/public/screenSizeTrigger.ts similarity index 100% rename from frontend/app/public/screenSizeTrigger.ts rename to frontend/app/(pages)/public/screenSizeTrigger.ts diff --git a/frontend/app/public/type/Result.ts b/frontend/app/(pages)/public/type/Result.ts similarity index 100% rename from frontend/app/public/type/Result.ts rename to frontend/app/(pages)/public/type/Result.ts diff --git a/frontend/app/search/page.tsx b/frontend/app/(pages)/search/page.tsx similarity index 100% rename from frontend/app/search/page.tsx rename to frontend/app/(pages)/search/page.tsx diff --git a/frontend/app/Activities.tsx b/frontend/app/Activities.tsx new file mode 100644 index 0000000..2d5294d --- /dev/null +++ b/frontend/app/Activities.tsx @@ -0,0 +1,43 @@ +import React, { useEffect, useState } from 'react'; +import { Activity } from '@/src/HomePageActivities/activity'; +import { getGhActivities } from '@/src/HomePageActivities/ghActivity'; +import * as motion from 'motion/react-client'; + +const Activities = () => { + + const [ activities, setActivities ] = useState(null); + + useEffect(() => { + getGhActivities().then(data => setActivities(data)); + }, []); + + if (!activities) { + return ( +
    +

    loading

    +
    + ); + } else { + return ( +
    +
      + { activities.map((activity, key) => ( + +
    • + { activity.toElement() } +
      • +       + )) } +
    +
    + ); + } + +}; + +export default Activities; \ No newline at end of file diff --git a/frontend/app/blog/post/[id]/PostCard.tsx b/frontend/app/blog/post/[id]/PostCard.tsx deleted file mode 100644 index 958979f..0000000 --- a/frontend/app/blog/post/[id]/PostCard.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import React from 'react'; -import { Card, CardBody } from '@nextui-org/card'; -import Headers from '@/app/blog/post/[id]/Headers'; -import Footer from '@/app/blog/post/[id]/Footer'; -import { fetchNocturneApi } from '@/app/blog/api'; -import MarkdownRenderer from '@/app/public/MarkdownRenderer'; - -type Props = { - id: string -} - -async function PostCard(props: Props) { - - - try { - const res = await fetchNocturneApi(`/blog/get_post/${ props.id }`); - const ret: any = await res.json(); - return ( -
    - - - {/* card headers */ } - - - {/* card body */ } - -
    - - { ret['content'] } - -
    -     - - - {/* card footer */ } -
    - - -
    - ); - } catch (e: any) { - return
    - - - Fatal: { e.toString() } - - -
    ; - } - -} - -export default PostCard; diff --git a/frontend/app/blog/post/[id]/page.tsx b/frontend/app/blog/post/[id]/page.tsx deleted file mode 100644 index c29364d..0000000 --- a/frontend/app/blog/post/[id]/page.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import React from 'react'; -import PostCard from '@/app/blog/post/[id]/PostCard'; -import { MotionDiv } from '@/app/public/MotionDiv'; -import ButtonGoBack from '@/app/blog/post/[id]/ButtonGoBack'; - -async function BlogReader({ - params -}: { - params: Promise<{ id: string }> -}) { - - return ( - -
    - -
    -
    - -
    -     - ); -} - -export default BlogReader; diff --git a/frontend/app/entries.tsx b/frontend/app/entries.tsx new file mode 100644 index 0000000..adcf6a3 --- /dev/null +++ b/frontend/app/entries.tsx @@ -0,0 +1,56 @@ +import React from 'react'; +import { + IoFlask, + IoHome, + IoInformationCircle, + IoLink, + IoLogoGithub, + IoMail, + IoPencil, + IoSearch +} from 'react-icons/io5'; + +export const entries: { display_name: string; href: string; icon: React.ReactNode }[] = [ + { + icon: , + display_name: 'Home', + href: '/' + }, + { + icon: , + display_name: 'About', + href: '/about' + }, + { + icon: , + display_name: 'Blog', + href: '/blog' + }, + { + icon: , + display_name: 'Friends', + href: '/friends' + }, + { + icon: , + display_name: 'Search', + href: '/search' + }, { + icon: , + display_name: 'Lab', + href: 'https://lab.lmt.moe' + } +]; + +export const connections: { label: string, href: string, icon: React.ReactNode }[] = [ + { + label: 'Github', + href: 'https://github.com/Lumither', + icon: + }, + { + label: 'email', + href: 'mailto:lumither@outlook.com', + icon: + } +]; diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index 1c4b3c9..04bb4b9 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -2,13 +2,14 @@ import type { Metadata } from 'next'; import './globals.css'; import React from 'react'; import { ThemeProvider as NextThemesProvider } from 'next-themes'; -import Navbar from '@/components/Navbar'; -import Footer from '@/components/Footer'; - +import { SITE_CONFIG } from '@/src/constants'; export const metadata: Metadata = { - title: 'Lumitherの酒馆', - description: '一间位于时间夹缝中的酒馆，静候下一位旅者的到来...' + openGraph: { + siteName: SITE_CONFIG.name + }, + title: SITE_CONFIG.name, + description: SITE_CONFIG.desc }; export default function RootLayout({ @@ -20,13 +21,7 @@ export default function RootLayout({ -
    - -
    - { children } -
    -
    -
    + { children }     diff --git a/frontend/app/page.tsx b/frontend/app/page.tsx index e9f70cd..bcd8567 100644 --- a/frontend/app/page.tsx +++ b/frontend/app/page.tsx @@ -1,10 +1,51 @@ -import ConstructionPlaceHolder from '@/components/ConstructionPlaceHolder'; +'use client'; + import React from 'react'; +import { useScreenSizeTrigger } from '@/app/(pages)/public/screenSizeTrigger'; +import { Button } from '@nextui-org/react'; +import { entries } from '@/app/entries'; +import Link from 'next/link'; +import { useLocalStorage } from 'react-use'; export default function Home() { + + const isMobile = useScreenSizeTrigger('lg'); + + const [ test, setTest, removeTest ] = useLocalStorage('test', 'test_value'); + console.log(test); + // setTest('test value 2'); + // console.log(test); + // removeTest(); + // console.log(test); + return ( -
    - +
    +
      + { + entries.map((meta, key) => ( +
    • + +
      • + )) + } +
    +
    +
    +
    ); } diff --git a/frontend/app/public/MotionDiv.tsx b/frontend/app/public/MotionDiv.tsx deleted file mode 100644 index 224a656..0000000 --- a/frontend/app/public/MotionDiv.tsx +++ /dev/null @@ -1,5 +0,0 @@ -'use client'; - -import { motion } from 'framer-motion'; - -export const MotionDiv = motion.div; diff --git a/frontend/components/Navbar.tsx b/frontend/components/Navbar.tsx index ea0238c..ab48da5 100644 --- a/frontend/components/Navbar.tsx +++ b/frontend/components/Navbar.tsx @@ -3,65 +3,12 @@ import React, { useEffect, useState } from 'react'; import { Button } from '@nextui-org/react'; import Image from 'next/image'; -import { - IoFlask, - IoHome, - IoInformationCircle, - IoLink, - IoLogoGithub, - IoMail, - IoPencil, - IoSearch -} from 'react-icons/io5'; import Link from 'next/link'; import ThemeSwitcher from '@/components/ThemeSwitcher'; import { usePathname } from 'next/navigation'; -import { useScreenSizeTrigger } from '@/app/public/screenSizeTrigger'; +import { useScreenSizeTrigger } from '@/app/(pages)/public/screenSizeTrigger'; +import { connections, entries } from '@/app/entries'; -const entries: { display_name: string; href: string; icon: React.ReactNode }[] = [ - { - icon: , - display_name: 'Home', - href: '/' - }, - { - icon: , - display_name: 'About', - href: '/about' - }, - { - icon: , - display_name: 'Blog', - href: '/blog' - }, - { - icon: , - display_name: 'Friends', - href: '/friends' - }, - { - icon: , - display_name: 'Search', - href: '/search' - }, { - icon: , - display_name: 'Lab', - href: 'https://lab.lmt.moe' - } -]; - -const connections: { label: string, href: string, icon: React.ReactNode }[] = [ - { - label: 'Github', - href: 'https://github.com/Lumither', - icon: - }, - { - label: 'email', - href: 'mailto:lumither@outlook.com', - icon: - } -]; function Navbar() { diff --git a/frontend/components/ThemeSwitcher.tsx b/frontend/components/ThemeSwitcher.tsx index 0dbc95f..891867a 100644 --- a/frontend/components/ThemeSwitcher.tsx +++ b/frontend/components/ThemeSwitcher.tsx @@ -4,7 +4,7 @@ import React, { useEffect, useState } from 'react'; import { useTheme } from 'next-themes'; import { Button, Dropdown, DropdownItem, DropdownMenu, DropdownTrigger } from '@nextui-org/react'; import { MdOutlineDarkMode, MdOutlineDesktopWindows, MdOutlineLightMode } from 'react-icons/md'; -import { useScreenSizeTrigger } from '@/app/public/screenSizeTrigger'; +import { useScreenSizeTrigger } from '@/app/(pages)/public/screenSizeTrigger'; const switcherElements = new Map( [ diff --git a/frontend/package.json b/frontend/package.json index 92be224..6497820 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,8 +11,8 @@ "dependencies": { "@nextui-org/react": "^2.4.6", "@types/react-syntax-highlighter": "^15.5.13", - "framer-motion": "^11.3.24", "lxgw-wenkai-webfont": "^1.7.0", + "motion": "^12.3.1", "next": "^15.0.3", "next-themes": "^0.3.0", "react": "^18.3.1", @@ -20,6 +20,7 @@ "react-icons": "^5.2.1", "react-markdown": "^9.0.1", "react-syntax-highlighter": "^15.5.0", + "react-use": "^17.6.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "remark-frontmatter": "^5.0.0", diff --git a/frontend/src/HomePageActivities/activity.tsx b/frontend/src/HomePageActivities/activity.tsx new file mode 100644 index 0000000..d7f5ea4 --- /dev/null +++ b/frontend/src/HomePageActivities/activity.tsx @@ -0,0 +1,18 @@ +import React from 'react'; + + +export abstract class Activity { + protected constructor( + readonly time: Date + ) { + } + + abstract toElement(): React.ReactElement; + +} + +export abstract class GhActivity extends Activity { + protected constructor(time: Date) { + super(time); + } +} diff --git a/frontend/src/HomePageActivities/constants.ts b/frontend/src/HomePageActivities/constants.ts new file mode 100644 index 0000000..b75cc9c --- /dev/null +++ b/frontend/src/HomePageActivities/constants.ts @@ -0,0 +1 @@ +export const GH_USER_NAME = 'Lumither'; diff --git a/frontend/src/HomePageActivities/ghActivities/starRepo.tsx b/frontend/src/HomePageActivities/ghActivities/starRepo.tsx new file mode 100644 index 0000000..8cacb89 --- /dev/null +++ b/frontend/src/HomePageActivities/ghActivities/starRepo.tsx @@ -0,0 +1,34 @@ +import { GhActivity } from '@/src/HomePageActivities/activity'; +import { Card, CardBody } from '@nextui-org/card'; +import Link from 'next/link'; + +export class StarRepo extends GhActivity { + constructor( + readonly repoName: string, + readonly repoUrl: string, + time: Date + ) { + super(time); + } + + toElement() { + return ( + //
    + // { `${ GH_USER_NAME } started ${ this.repoName } at ${ this.time }` } + //
    + + +

    { `Stared GitHub repository '${ this.repoName }'` }

    +     +     + + ); + } +} diff --git a/frontend/src/HomePageActivities/ghActivity.tsx b/frontend/src/HomePageActivities/ghActivity.tsx new file mode 100644 index 0000000..689a6ad --- /dev/null +++ b/frontend/src/HomePageActivities/ghActivity.tsx @@ -0,0 +1,32 @@ +import { GhActivity } from '@/src/HomePageActivities/activity'; +import { match } from 'ts-pattern'; +import { StarRepo } from '@/src/HomePageActivities/ghActivities/starRepo'; +import { GH_USER_NAME } from '@/src/HomePageActivities/constants'; + + +export const getGhActivities: () => Promise = async () => { + const data = await fetch(`https://api.github.com/users/${ GH_USER_NAME }/events`, { + headers: { + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28' + } + }); + return data + .json() + .then((ret: any): GhActivity[] => ( + ret + .map((res: any) => ( + match(res) + .returnType() + // star repo + .with( + { type: 'WatchEvent', payload: { action: 'started' } } + , () => ( + new StarRepo(res.repo.name, res.repo.url, res.created_at) + )) + // to be continued + .otherwise(() => undefined)) + ) + .filter((res: GhActivity | undefined) => res !== undefined) + )); +}; \ No newline at end of file diff --git a/frontend/src/constants.ts b/frontend/src/constants.ts new file mode 100644 index 0000000..37ff01d --- /dev/null +++ b/frontend/src/constants.ts @@ -0,0 +1,4 @@ +export const SITE_CONFIG = { + name: 'Atelier Lumiaethor', + desc: '' +}; \ No newline at end of file From acd05255139d852f57e6aa6a4ac7c75dc110ecc4 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 17 Feb 2025 15:27:41 -0800 Subject: [PATCH 02/32] feat(backend) rebuild start --- backend/Cargo.toml | 3 +- backend/api_server/Cargo.toml | 25 -- .../api_server/src/blog/api/get_page_count.rs | 26 -- backend/api_server/src/blog/api/get_post.rs | 153 -------- .../api_server/src/blog/api/get_post_list.rs | 130 ------- backend/api_server/src/blog/api/mod.rs | 3 - .../src/blog/cron/check_update/error.rs | 49 --- .../src/blog/cron/check_update/index.rs | 351 ------------------ .../src/blog/cron/check_update/mod.rs | 146 -------- .../src/blog/cron/check_update/utils.rs | 86 ----- backend/api_server/src/blog/cron/mod.rs | 15 - backend/api_server/src/blog/mod.rs | 17 - backend/api_server/src/init/cron.rs | 12 - backend/api_server/src/init/database.rs | 41 -- backend/api_server/src/init/logger.rs | 106 ------ backend/api_server/src/init/misc.rs | 14 - backend/api_server/src/init/mod.rs | 58 --- backend/api_server/src/init/schema.rs | 49 --- backend/api_server/src/init/server.rs | 52 --- backend/api_server/src/main.rs | 13 - backend/api_server/src/scheduler/mod.rs | 194 ---------- .../src/scheduler/scheduler_error.rs | 44 --- backend/api_server/src/scheduler/sleeper.rs | 77 ---- backend/api_server/src/scheduler/task_func.rs | 3 - .../api_server/src/scheduler/tasks/basic.rs | 37 -- backend/api_server/src/scheduler/tasks/mod.rs | 8 - backend/api_server/src/utils/git.rs | 195 ---------- backend/api_server/src/utils/mod.rs | 1 - backend/constants/Cargo.toml | 6 - backend/macros/src/log.rs | 21 +- backend/server/schema.sql | 32 ++ .../src => server/src/constants}/blog.rs | 0 .../src => server/src/constants}/config.rs | 0 .../lib.rs => server/src/constants/mod.rs} | 0 .../src => server/src/constants}/time.rs | 0 backend/server/src/main.rs | 7 + loadbalancer/.gitignore | 3 + openresty/conf/nocturne.conf => nocturne.conf | 19 +- openresty/deploy.sh | 4 - openresty/lua/hello_world.lua | 1 - scripts/dev_server.sh | 14 - scripts/flush_env.sh | 6 - scripts/refresh.sh | 42 --- scripts/setup.sh | 15 - scripts/start_server.sh | 20 - 45 files changed, 69 insertions(+), 2029 deletions(-) delete mode 100644 backend/api_server/Cargo.toml delete mode 100644 backend/api_server/src/blog/api/get_page_count.rs delete mode 100644 backend/api_server/src/blog/api/get_post.rs delete mode 100644 backend/api_server/src/blog/api/get_post_list.rs delete mode 100644 backend/api_server/src/blog/api/mod.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/error.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/index.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/mod.rs delete mode 100644 backend/api_server/src/blog/cron/check_update/utils.rs delete mode 100644 backend/api_server/src/blog/cron/mod.rs delete mode 100644 backend/api_server/src/blog/mod.rs delete mode 100644 backend/api_server/src/init/cron.rs delete mode 100644 backend/api_server/src/init/database.rs delete mode 100644 backend/api_server/src/init/logger.rs delete mode 100644 backend/api_server/src/init/misc.rs delete mode 100644 backend/api_server/src/init/mod.rs delete mode 100644 backend/api_server/src/init/schema.rs delete mode 100644 backend/api_server/src/init/server.rs delete mode 100644 backend/api_server/src/main.rs delete mode 100644 backend/api_server/src/scheduler/mod.rs delete mode 100644 backend/api_server/src/scheduler/scheduler_error.rs delete mode 100644 backend/api_server/src/scheduler/sleeper.rs delete mode 100644 backend/api_server/src/scheduler/task_func.rs delete mode 100644 backend/api_server/src/scheduler/tasks/basic.rs delete mode 100644 backend/api_server/src/scheduler/tasks/mod.rs delete mode 100644 backend/api_server/src/utils/git.rs delete mode 100644 backend/api_server/src/utils/mod.rs delete mode 100644 backend/constants/Cargo.toml create mode 100644 backend/server/schema.sql rename backend/{constants/src => server/src/constants}/blog.rs (100%) rename backend/{constants/src => server/src/constants}/config.rs (100%) rename backend/{constants/src/lib.rs => server/src/constants/mod.rs} (100%) rename backend/{constants/src => server/src/constants}/time.rs (100%) create mode 100644 backend/server/src/main.rs create mode 100644 loadbalancer/.gitignore rename openresty/conf/nocturne.conf => nocturne.conf (65%) delete mode 100755 openresty/deploy.sh delete mode 100644 openresty/lua/hello_world.lua delete mode 100755 scripts/dev_server.sh delete mode 100755 scripts/flush_env.sh delete mode 100755 scripts/refresh.sh delete mode 100755 scripts/setup.sh delete mode 100755 scripts/start_server.sh diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6c75b0a..154acf9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,8 +1,7 @@ [workspace] resolver = "2" members = [ - "api_server", - "constants", + "server", "macros", "markdown", ] diff --git a/backend/api_server/Cargo.toml b/backend/api_server/Cargo.toml deleted file mode 100644 index af17326..0000000 --- a/backend/api_server/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "api_server" -version = "0.1.0" -edition = "2021" - -[dependencies] -macros = { path = "../macros" } -constants = { path = "../constants" } -markdown = { path = "../markdown" } -chrono = { version = "0.4.38", features = ["serde", "alloc"] } -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } -tracing-appender = "0.2.3" -dotenv = "0.15.0" -dirs = "5.0.1" -tokio = { version = "1.40.0", features = ["full"] } -sqlx = { version = "0.8.2", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono", "uuid"] } -axum = { version = "0.7.5", features = ["macros"] } -serde = { version = "1.0.199", features = ["derive"] } -serde_json = "1.0.128" -uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } -futures = { version = "0.3.30", features = ["executor"] } -cron = "0.13.0" -git2 = "0.19.0" -thiserror = "2.0.3" diff --git a/backend/api_server/src/blog/api/get_page_count.rs b/backend/api_server/src/blog/api/get_page_count.rs deleted file mode 100644 index db540af..0000000 --- a/backend/api_server/src/blog/api/get_page_count.rs +++ /dev/null @@ -1,26 +0,0 @@ -use constants::blog::PAGE_SIZE; - -use axum::extract::State; -use axum::Json; -use serde_json::{json, Value}; -use sqlx::{query, PgPool, Row}; -use tracing::error; - -pub async fn get_page_count( - State(db_connection): State, -) -> Result, Json> { - match query("select cast(ceil(count(*) / $1) as integer) as res from post") - .bind(PAGE_SIZE as f32) - .fetch_one(&db_connection) - .await - { - Ok(res) => { - let res: i32 = res.get("res"); - Ok(Json::from(json!({"res": res}))) - } - Err(e) => { - error!("Failed to get blog page count: {}", e.to_string()); - Err(Json::from(json!({"error": e.to_string()}))) - } - } -} diff --git a/backend/api_server/src/blog/api/get_post.rs b/backend/api_server/src/blog/api/get_post.rs deleted file mode 100644 index 2374dec..0000000 --- a/backend/api_server/src/blog/api/get_post.rs +++ /dev/null @@ -1,153 +0,0 @@ -use axum::{ - extract::{Path, State}, - http::StatusCode, - Json, -}; -use serde_json::{json, Value}; -use sqlx::{query, Error, PgPool, Row}; -use std::str::FromStr; -use tracing::error; -use uuid::Uuid; - -pub async fn get_post( - State(db_connection): State, - Path(post_id): Path, -) -> Result, (StatusCode, Json)> { - let post_id = match Uuid::from_str(post_id.as_str()) { - Ok(id) => id, - Err(e) => { - error!( - "Failed to parse UUID <{}>: {}", - post_id.as_str(), - e.to_string() - ); - return Err(( - StatusCode::BAD_REQUEST, - Json::from(json!(("error", e.to_string()))), - )); - } - }; - - let post = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post -FROM post t -WHERE post_id = $1; -"#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("post"), - Err(e) => { - error!( - "Database error on Reading `base post information` from table `post`: <{}>: {}", - post_id.to_string(), - e.to_string() - ); - return match e { - Error::RowNotFound => Err(( - StatusCode::BAD_REQUEST, - Json::from(json!({"error": e.to_string()})), - )), - _ => Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )), - }; - } - }; - - let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;") - .bind(post_id) - .fetch_all(&db_connection) - .await - { - Ok(tags) => tags.iter().map(|tag| tag.get::("tag")).collect(), - Err(e) => { - error!("Database query error :{}", e.to_string()); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let meta = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta -FROM meta t -WHERE post_id = $1; - "#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("meta"), - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "post meta data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let adjacent = match query(r" -WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1), - prev AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date < (SELECT date FROM target_post) - ORDER BY t.date DESC - LIMIT 1), - next AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date > (SELECT date FROM target_post) - ORDER BY t.date - LIMIT 1) -SELECT json_strip_nulls(json_build_object( - 'prev', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM prev), - 'next', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM next) - )) AS result; -" - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(res) => {res.get::("result")} - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "adjoint posts data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - - }; - - let mut res = post.as_object().unwrap().to_owned(); - res.insert("tags".to_string(), tags.into()); - res.insert("meta".to_string(), meta); - res.insert("adj".to_string(), adjacent); - - Ok(Json::from(json!(res))) -} diff --git a/backend/api_server/src/blog/api/get_post_list.rs b/backend/api_server/src/blog/api/get_post_list.rs deleted file mode 100644 index 9db7c61..0000000 --- a/backend/api_server/src/blog/api/get_post_list.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::error::Error; - -use constants::blog::PAGE_SIZE; - -use axum::{extract::Query, extract::State, http::StatusCode, Json}; -use chrono::{DateTime, Utc}; -use futures::future::join_all; -use serde::Deserialize; -use serde_json::{json, Map, Value}; -use sqlx::{postgres::PgRow, query, PgPool, Row}; -use tracing::error; -use uuid::Uuid; - -#[derive(Deserialize)] -pub struct Params { - page: Option, -} - -#[axum::debug_handler] -pub async fn get_post_list( - State(db_connection): State, - Query(params): Query, -) -> Result, (StatusCode, Json)> { - let search_param_page = params.page.unwrap_or(1); - if search_param_page < 1 { - return Err(( - StatusCode::BAD_REQUEST, - Json::from(json!({"error": "`page` should be greater than 0"})), - )); - } - - let offset = PAGE_SIZE * (search_param_page - 1); - - let post_list = sqlx::query("SELECT * FROM post ORDER BY first_update DESC LIMIT $1 OFFSET $2") - .bind(PAGE_SIZE as i32) - .bind(offset as i32) - .fetch_all(&db_connection) - .await; - let post_list = match post_list { - Ok(post_list) => post_list, - Err(e) => { - error!( - "Database query error on fetching post list: {}", - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let mut res = Vec::new(); - let posts: Vec<_> = post_list - .into_iter() - .map(|post| async { analyze_post(&db_connection, post).await }) - .collect(); - let posts = join_all(posts).await; - for post in posts { - match post { - Ok(p) => res.push(p), - Err(e) => { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!(("error", e.to_string()))), - )) - } - } - } - - // res.insert("posts".to_string(), Value::from(posts)); - Ok(Json::from(Value::from(res))) -} - -async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result> { - let post_id: Uuid = post.get("post_id"); - let tags = fetch_post_tags(db_connection, post_id).await; - let meta = fetch_post_meta(db_connection, post_id).await.unwrap(); - let header_img = meta.get("header_img"); - let last_update = meta.get("last_update"); - - Ok(json!({ - "post_id": post_id.to_string(), - "title": post.get::("title"), - "category": post.get::("category"), - "header_img": header_img, - "sub_title": post.get::("sub_title"), - "summary": post.get::("summary"), - "last_update": last_update, - "first_update": post.get::, _>("first_update"), - "tags": tags, - })) -} - -async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option { - match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta -FROM meta t -WHERE post_id = $1; - "#, - ) - .bind(post_id) - .fetch_one(db_connection) - .await - { - Ok(value) => Some(value.get::("meta")), - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "post meta data", - "meta", - post_id.to_string(), - e.to_string() - ); - None - } - } -} - -async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec { - match query("SELECT tag FROM Tag WHERE post_id = $1") - .bind(post_id) - .fetch_all(db_connection) - .await - { - Ok(res) => res.iter().map(|tag| tag.get::("tag")).collect(), - Err(_) => vec![], - } -} diff --git a/backend/api_server/src/blog/api/mod.rs b/backend/api_server/src/blog/api/mod.rs deleted file mode 100644 index 5e23c3d..0000000 --- a/backend/api_server/src/blog/api/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod get_page_count; -pub mod get_post; -pub mod get_post_list; diff --git a/backend/api_server/src/blog/cron/check_update/error.rs b/backend/api_server/src/blog/cron/check_update/error.rs deleted file mode 100644 index 0d27c29..0000000 --- a/backend/api_server/src/blog/cron/check_update/error.rs +++ /dev/null @@ -1,49 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum PostIdxError { - #[error("missing essential field `{}` for file `{}`", field, filename)] - MissingField { field: String, filename: String }, - - #[error( - "invalid field `{}` for file `{}`, <{}>: {}", - field_name, - filename, - err_field, - msg - )] - InvalidField { - field_name: String, - err_field: String, - filename: String, - msg: String, - }, - - #[error( - "database error on writing `{}` to table `{}`: <{}>: {}", - data_desc, - db_table, - id, - msg - )] - DBWriteFailure { - data_desc: String, - db_table: String, - id: String, - msg: String, - }, - - #[error( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - data_desc, - db_table, - id, - msg - )] - DBReadFailure { - data_desc: String, - db_table: String, - id: String, - msg: String, - }, -} diff --git a/backend/api_server/src/blog/cron/check_update/index.rs b/backend/api_server/src/blog/cron/check_update/index.rs deleted file mode 100644 index f391639..0000000 --- a/backend/api_server/src/blog/cron/check_update/index.rs +++ /dev/null @@ -1,351 +0,0 @@ -use std::error::Error; -use std::path::Path; - -use crate::blog::cron::check_update::utils::get_md_file_basename; -use crate::blog::cron::check_update::{error::PostIdxError, utils::parse_date}; -use markdown::MdFile; - -use sqlx::{query, PgPool, Pool, Postgres, Row}; -use tokio::runtime::Runtime; -use tracing::{info, warn}; - -async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - let post_id = post.file_id; - let title = match post.meta.get("title") { - None => { - return Err(PostIdxError::MissingField { - field: "title".to_string(), - filename: post.filename.clone(), - }) - } - Some(title) => title.as_str().unwrap_or_else(|| { - warn!("invalid title for `{}`", &post.filename); - "invalid title" - }), - }; - let sub_title = post - .meta - .get("sub_title") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let summary = post - .meta - .get("summary") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let content = post.content.as_str(); - let category = match post.meta.get("category") { - None => { - return Err(PostIdxError::MissingField { - field: "category".to_string(), - filename: post.filename.clone(), - }) - } - Some(title) => title.as_str().unwrap_or_else(|| { - warn!("invalid category for `{}`", &post.filename); - "invalid category" - }), - }; - let last_update = { - if let Some(last_update) = post.meta.get("last_update") { - match parse_date(last_update.as_str().unwrap()) { - Ok(date) => Some(date), - Err(e) => { - return Err(PostIdxError::InvalidField { - field_name: "last_update".to_string(), - err_field: last_update.to_string(), - filename: post.filename.clone(), - msg: e.to_string(), - }) - } - } - } else { - None - } - }; - let first_update = match post.meta.get("date") { - None => { - return Err(PostIdxError::MissingField { - field: "date".to_string(), - filename: post.filename.clone(), - }) - } - Some(first_update) => match parse_date(first_update.as_str().unwrap()) { - Ok(date) => date, - Err(e) => { - return Err(PostIdxError::InvalidField { - field_name: "date".to_string(), - err_field: first_update.to_string(), - filename: post.filename.clone(), - msg: e.to_string(), - }) - } - }, - }; - let filename = get_md_file_basename(&post.filename); - - if let Err(e) = query( - r##" -INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) -ON CONFLICT (post_id) DO -UPDATE SET - title = EXCLUDED.title, - summary = EXCLUDED.summary, - content = EXCLUDED.content, - last_update = EXCLUDED.last_update, - first_update = EXCLUDED.first_update, - sub_title = EXCLUDED.sub_title, - category = EXCLUDED.category, - filename = EXCLUDED.filename; - "##, - ) - .bind(post_id) - .bind(title) - .bind(summary) - .bind(content) - .bind(last_update) - .bind(first_update) - .bind(sub_title) - .bind(category) - .bind(filename) - .execute(db_connection) - .await - { - return Err( - PostIdxError::DBWriteFailure { - data_desc: "post base information".to_string(), - db_table: "Post".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - } - ); - } - Ok(()) -} - -async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - if let Some(tags_string) = post.meta.get("tags").to_owned() { - let tags = match tags_string.as_array().map(|vec| { - vec.iter() - .map(|val| val.as_str().unwrap()) - .collect::>() - }) { - None => { - return Err(PostIdxError::MissingField { - field: "tags".to_string(), - filename: post.filename.clone(), - }) - } - Some(val) => val, - }; - - let post_id = post.file_id; - - // remove old record - if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") - .bind(post_id) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - }); - } - - // add new record - for tag in tags { - match query("INSERT INTO Tag VALUES ($1, $2);") - .bind(post_id) - .bind(tag) - .execute(db_connection) - .await - { - Ok(_) => {} - Err(e) => { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post tag information".to_string(), - db_table: "Tag".to_string(), - id: post_id.to_string(), - msg: e.to_string(), - }) - } - } - } - } - - Ok(()) -} - -async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { - let base64_form = post.hash.clone(); - match query("SELECT hash FROM Hash WHERE post_id = $1") - .bind(post.file_id) - .fetch_optional(db_connection) - .await - { - Ok(row) => { - let db_record: String = match row { - None => "".to_string(), - Some(row) => row.try_get("hash").unwrap_or("".to_string()), - }; - if base64_form == db_record { - Ok(false) - } else { - if let Err(e) = query( - r#" -INSERT INTO Hash (post_id, hash) VALUES ($1, $2) - ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, - ) - .bind(post.file_id) - .bind(&base64_form) - .execute(db_connection) - .await - { - return Err(PostIdxError::DBWriteFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }); - } - Ok(true) - } - } - Err(e) => Err(PostIdxError::DBReadFailure { - data_desc: "post hash".to_string(), - db_table: "Hash".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }), - } -} - -async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { - if let Err(e) = query( - // todo: sanitization - format!( - r#" -DO -$$ - DECLARE - _post_id UUID := '{}'; - _meta_cols JSONB := '{}'; - _exists BOOLEAN; - _col_name TEXT; - _col_value TEXT; - _sql TEXT; - _insert_columns TEXT := 'post_id'; - _insert_values TEXT := quote_literal(_post_id); - _update_set TEXT := ''; - _blacklist TEXT[] := ARRAY['tags']; - BEGIN - SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; - - FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) - LOOP - IF _col_name = ANY(_blacklist) THEN - CONTINUE; - END IF; - - IF NOT EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'meta' - AND column_name = _col_name - -- may change in the future - AND table_schema = 'public') THEN - _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; - EXECUTE _sql; - END IF; - _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); - _insert_values := _insert_values || ', ' || quote_literal(_col_value); - _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); - END LOOP; - - _update_set := substring(_update_set FROM 3); - - IF _exists THEN - _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); - EXECUTE _sql; - ELSE - _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; - EXECUTE _sql; - END IF; - END; -$$ LANGUAGE plpgsql; - "#, post.file_id.to_string().as_str(), post.meta - ).as_str() - ) - .execute(db_connection) - .await { - Err(PostIdxError::DBWriteFailure { - data_desc: "post meta data".to_string(), - db_table: "Meta".to_string(), - id: post.file_id.to_string(), - msg: e.to_string(), - }) - } else { - Ok(()) - } -} - -pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { - let md_file = MdFile::from_file(path)?; - - async fn trail(db_conn: &Pool, md_file: &MdFile) -> Result<(), PostIdxError> { - base(db_conn, md_file).await?; - tag(db_conn, md_file).await?; - meta(db_conn, md_file).await?; - hash(db_conn, md_file).await.map(|_| ()) - } - let mut cnt = 5; - - while let Err(e) = trail(db_conn, &md_file).await { - match e { - PostIdxError::DBWriteFailure { .. } => { - cnt -= 1; - } - PostIdxError::DBReadFailure { .. } => { - cnt -= 1; - } - _ => return Err(e.into()), - } - if cnt <= 0 { - return Err(e.into()); - } - } - - info!("indexed markdown file: {}", path.display()); - Ok(()) -} - -pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - if let Err(e) = query( - r#" -WITH deleted_post AS ( - DELETE FROM post - WHERE id = (SELECT id FROM post WHERE filename = $1) - RETURNING id -) -DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); -DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); -DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); - "#, - ) - .bind(file_basename) - .execute(db_conn) - .await - { - Err(Box::from( - format!("failed to drop index: {}", e).to_string(), - )) - } else { - Ok(()) - } - }) -} diff --git a/backend/api_server/src/blog/cron/check_update/mod.rs b/backend/api_server/src/blog/cron/check_update/mod.rs deleted file mode 100644 index 2d531e3..0000000 --- a/backend/api_server/src/blog/cron/check_update/mod.rs +++ /dev/null @@ -1,146 +0,0 @@ -use std::env; -use std::error::Error; -use std::fs::remove_dir_all; -use std::path::{Path, PathBuf}; - -use crate::{ - blog::cron::check_update::{ - index::drop_index, - utils::{expand_path, get_md_file_basename, index_files, search_md}, - }, - utils::git, -}; -use constants::config::general::{default_value, var_name}; -use macros::panic_with_log; - -use git2::{Delta, Repository}; -use sqlx::{Pool, Postgres}; -use tracing::{error, info, warn, Level}; - -mod error; -mod index; -mod utils; - -pub fn task(db_connection: &Pool) { - // init - let git_work_dir = { - let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { - expand_path(default_log_path) - } else { - let fallback_path = expand_path(default_value::WORK_DIR.to_string()); - warn!( - "unset env var `{}`, using default path: {}/blog_git", - var_name::WORK_DIR, - fallback_path - ); - fallback_path - }; - Path::new(&path).join("blog_git") - }; - - // clone & check update - - let git_url = match env::var(var_name::BLOG_GIT_URL) { - Ok(url) => url, - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to read `{}`: {}", - var_name::BLOG_GIT_URL, - e - ) - } - }; - let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) - .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); - let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) - .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); - - match Repository::open(&git_work_dir) { - Ok(repo) => { - // prevent git remote url mismatch (clean cache) - match repo.find_remote(&git_remote_name) { - Ok(remote) => { - if remote.url().is_some_and(|url| url != git_url) { - remove_dir_all(&git_work_dir).unwrap(); - warn!("blog content git({}) url mismatch, reinitialized", &git_url); - return; - } - } - Err(e) => { - remove_dir_all(&git_work_dir).unwrap(); - error!( - "unexpected error on indexing blog content git({}), reinitialized: {}", - &git_url, e - ); - } - } - - match git::sync(&git_remote_name, &git_remote_branch, &repo) { - Ok(updates) => { - if !updates.is_empty() { - // incremental update - info!( - "blog content git({}) update detected: {:?}", - &git_url, updates - ); - - let mut update_md: Vec = vec![]; - - for update in updates { - if [Delta::Deleted, Delta::Untracked].contains(&update.status) { - if let Some(path) = &update.old_path { - match drop_index( - &get_md_file_basename( - path.file_name().unwrap().to_str().unwrap(), - ), - db_connection, - ) { - Ok(_) => info!( - "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", - &git_url, - &path.display() - ), - Err(e) => error!( - "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index: {}", - &git_url, - &path.display(), - e - ) - } - } - } else if let Some(path) = &update.new_path { - if path.components().any(|comp| comp.as_os_str() == "posts") - && path.extension().is_some_and(|ext| ext == "md") - { - // update_md.push(path.to_path_buf()); - update_md.push(dbg!(path.to_path_buf())); - } - } - } - - index_files(&update_md, db_connection); - } - } - Err(e) => { - error!( - "failed to sync blog content git({}) updates: {}", - &git_url, e - ); - } - } - } - Err(_) => { - match Repository::clone(&git_url, &git_work_dir) { - Ok(_repo) => { - // full repo index - let files = search_md(git_work_dir.join("posts")); - index_files(&files, db_connection); - } - Err(e) => { - error!("failed to clone blog content git({}): {}", &git_url, e); - } - } - } - }; -} diff --git a/backend/api_server/src/blog/cron/check_update/utils.rs b/backend/api_server/src/blog/cron/check_update/utils.rs deleted file mode 100644 index c20d98a..0000000 --- a/backend/api_server/src/blog/cron/check_update/utils.rs +++ /dev/null @@ -1,86 +0,0 @@ -use dirs::home_dir; -use std::path::PathBuf; - -use crate::blog::cron::check_update::index::index_file; -use chrono::{DateTime, FixedOffset, ParseResult}; -use constants::config::general::{default_value, var_name}; -use constants::time::DEFAULT_DATE_FORMAT; - -use futures::executor::block_on; -use futures::future::join_all; -use sqlx::{Pool, Postgres}; -use tokio::runtime::Runtime; -use tokio::spawn; -use tracing::{error, warn}; - -pub fn search_md(entry: PathBuf) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry] - } else { - vec![] - } -} - -pub fn index_files(files: &[PathBuf], db_conn: &Pool) { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let files_clone = dbg!(files).to_owned(); - let handlers = files_clone - .into_iter() - .map(|file: PathBuf| { - let conn_clone = db_conn.clone(); - spawn(async move { - if let Err(e) = index_file(&file, &conn_clone).await { - error!("failed to index markdown file `{}`: {}", file.display(), e); - } - }) - }) - .collect::>(); - - block_on(join_all(handlers)); - }); -} - -pub fn expand_path(path: String) -> String { - if path.starts_with('~') { - home_dir() - .map(|dir| { - dir.join(path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .unwrap() - }) - .unwrap_or_else(|| { - warn!( - "failed to parse env var `{}`, using default value `{}`", - var_name::WORK_DIR, - default_value::WORK_DIR - ); - default_value::WORK_DIR.to_string() - }) - } else { - path - } -} - -pub fn parse_date(date_str: &str) -> ParseResult> { - DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) -} - -pub fn get_md_file_basename(f_name: &str) -> String { - f_name.replace(".md", "") -} diff --git a/backend/api_server/src/blog/cron/mod.rs b/backend/api_server/src/blog/cron/mod.rs deleted file mode 100644 index 1cb8398..0000000 --- a/backend/api_server/src/blog/cron/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -use crate::scheduler::tasks::basic::BasicTask; -use crate::scheduler::tasks::Task; -use sqlx::{Pool, Postgres}; - -pub mod check_update; - -pub fn get_tasks(db_connection: Pool) -> Vec> { - let db_conn_clone = db_connection.clone(); - vec![ - // BasicTask::new(move || check_update::task(&db_conn_clone), "0/15 * * * * *") - BasicTask::new(move || check_update::task(&db_conn_clone), "* * * * * *") - .unwrap() - .to_task(), - ] -} diff --git a/backend/api_server/src/blog/mod.rs b/backend/api_server/src/blog/mod.rs deleted file mode 100644 index 6d14b15..0000000 --- a/backend/api_server/src/blog/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod api; -pub mod cron; - -use crate::blog::api::{ - get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, -}; - -use axum::routing::get; -use axum::Router; -use sqlx::{Pool, Postgres}; - -pub fn get_router() -> Router> { - Router::new() - .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count)) - .route("/get_post/:post_id", get(get_post)) -} diff --git a/backend/api_server/src/init/cron.rs b/backend/api_server/src/init/cron.rs deleted file mode 100644 index 429537a..0000000 --- a/backend/api_server/src/init/cron.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::blog; -use crate::scheduler::Scheduler; -use sqlx::{Pool, Postgres}; - -pub fn start(db_pool: Pool) { - let scheduler = Scheduler::new(); - let _ = vec![blog::cron::get_tasks(db_pool.clone())] - .into_iter() - .map(|list| scheduler.insert_list(list)) - .collect::>(); - scheduler.start().unwrap() -} diff --git a/backend/api_server/src/init/database.rs b/backend/api_server/src/init/database.rs deleted file mode 100644 index c2a8aad..0000000 --- a/backend/api_server/src/init/database.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::env; -use std::error::Error; -use std::str::FromStr; - -use constants::config::database::{default_value, var_name}; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; -use sqlx::{ConnectOptions, Pool, Postgres}; -use tracing::info; - -pub async fn init() -> Result, Box> { - let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) - .unwrap() - .disable_statement_logging(); - Ok(PgPoolOptions::new() - .max_connections(1) // todo: temporal solution, to be fixed - .connect_with(db_connect_option) - .await?) -} - -fn parse_db_uri() -> String { - env::var(var_name::URI).unwrap_or_else(|_| { - info!( - "`{}` env var is not set, \ - using `{}`, `{}`, `{}`, `{}` and `{}`", - var_name::URI, - var_name::USERNAME, - var_name::PASSWORD, - var_name::HOST, - var_name::PORT, - var_name::DBNAME - ); - let postgres_user = env::var(var_name::USERNAME).unwrap_or(default_value::USERNAME.into()); - let postgres_password = - env::var(var_name::PASSWORD).unwrap_or(default_value::PASSWORD.into()); - let db_host = env::var(var_name::HOST).unwrap_or(default_value::HOST.into()); - let db_port = env::var(var_name::PORT).unwrap_or(default_value::PORT.into()); - let postgres_db = env::var(var_name::DBNAME).unwrap_or(default_value::DBNAME.into()); - let db_type = default_value::DATABASE_TYPE; - format!("{db_type}://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") - }) -} diff --git a/backend/api_server/src/init/logger.rs b/backend/api_server/src/init/logger.rs deleted file mode 100644 index 5fd56b4..0000000 --- a/backend/api_server/src/init/logger.rs +++ /dev/null @@ -1,106 +0,0 @@ -use std::path::Path; -use std::{env, io}; - -use chrono::Local; -use constants::config::general::{default_value, var_name}; -use dirs::home_dir; -use tracing::info; -use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; -use tracing_subscriber::{ - filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, - layer::SubscriberExt, Layer, Registry, -}; - -struct LocalTimer; - -impl FormatTime for LocalTimer { - fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { - write!( - w, - "{}", - Local::now().format(constants::time::DEFAULT_TIME_FORMAT) - ) - } -} - -type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; - -pub fn init() -> (WorkerGuard, WorkerGuard) { - let log_path = match env::var(var_name::LOG_DIR) { - Ok(path) => Path::new(&path).to_path_buf(), - Err(_) => { - let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { - expand_home_path(default_log_path) - } else { - let fallback_path = expand_home_path(default_value::WORK_DIR.to_string()); - println!( - "[warn] unset env var `{}` and `{}`, trying to log on fallback path: {}/log", - var_name::LOG_DIR, - var_name::WORK_DIR, - fallback_path - ); - fallback_path - }; - Path::new(&path).join("log") - } - }; - - let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); - let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); - - let info_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_ansi(false); - let err_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_ansi(false); - let stdout_format = tracing_subscriber::fmt::format() - .with_level(true) - .with_target(true) - .with_source_location(true) - .with_timer(LocalTimer); - - let subscribers = Registry::default() - .with( - fmt::Layer::default() - .with_writer(info_log_writer) - .event_format(info_format), - ) - .with( - fmt::Layer::default() - .with_writer(err_log_writer) - .event_format(err_format) - .with_filter(LevelFilter::ERROR), - ) - .with( - fmt::Layer::default() - .with_writer(io::stdout) - .with_ansi(true) - .event_format(stdout_format), - ); - - tracing::subscriber::set_global_default(subscribers) - .expect("[fatal] failed to setup logging system"); - - info!("logging to dir: {}", log_path.to_str().unwrap()); - - (info_guard, error_guard) -} - -fn expand_home_path(path: String) -> String { - if path.starts_with('~') { - let home_path = home_dir() - .unwrap_or_else(|| panic!("[fatal] unable to locate home dir when parsing env `{}`, try to use absolute path instead", var_name::WORK_DIR)); - home_path - .join(path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .unwrap() - } else { - path - } -} diff --git a/backend/api_server/src/init/misc.rs b/backend/api_server/src/init/misc.rs deleted file mode 100644 index 6a08bc2..0000000 --- a/backend/api_server/src/init/misc.rs +++ /dev/null @@ -1,14 +0,0 @@ -use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds}; -use tracing::warn; - -pub fn init() { - // git2 option - unsafe { - if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) { - warn!("failed to set git2 server connect timeout: {}", e); - } - if let Err(e) = set_server_timeout_in_milliseconds(60_000) { - warn!("failed to set git2 server timeout: {}", e); - } - } -} diff --git a/backend/api_server/src/init/mod.rs b/backend/api_server/src/init/mod.rs deleted file mode 100644 index 507d52a..0000000 --- a/backend/api_server/src/init/mod.rs +++ /dev/null @@ -1,58 +0,0 @@ -mod cron; -mod database; -pub mod logger; -mod misc; -mod schema; -pub mod server; - -use macros::panic_with_log; -use std::{env, panic}; -use tracing::{info, Level}; - -pub fn load_env() { - if let Some(env_file) = env::args().nth(1) { - match dotenv::from_filename(&env_file) { - Ok(_) => { - println!("[info] env loaded from {}, starting up", env_file); - } - Err(e) => { - panic!("[fatal] failed to read {}: {}", env_file, e); - } - } - } else { - println!("[info] no .env file referred, starting up") - } -} - -pub async fn start() { - misc::init(); - - // connect database - let db_pool = match database::init().await { - Ok(pool) => { - info!("database connection pool initialized"); - pool - } - Err(e) => { - panic_with_log!( - Level::ERROR, - "database connection pool failed to initialize: {}", - e - ); - } - }; - - // schema init - schema::init(&db_pool).await; - - // load cron task - cron::start(db_pool.clone()); - - // api serve - match server::start(db_pool).await { - Ok(_) => {} - Err(e) => { - panic_with_log!(Level::ERROR, "server panicked: {}", e) - } - } -} diff --git a/backend/api_server/src/init/schema.rs b/backend/api_server/src/init/schema.rs deleted file mode 100644 index ef9def0..0000000 --- a/backend/api_server/src/init/schema.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sqlx::{query, Pool, Postgres}; -use tracing::info; - -const POST_BASE_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Post -( - post_id UUID PRIMARY KEY, - title VARCHAR(255) NOT NULL, - category VARCHAR(255) NOT NULL, - filename TEXT UNIQUE NOT NULL, - summary TEXT, - sub_title TEXT, - content TEXT NOT NULL, - last_update TIMESTAMPTZ, - first_update TIMESTAMPTZ NOT NULL -); -"##; - -// base model, will add more cols at runtime -const POST_META_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Meta -( - post_id UUID PRIMARY KEY -); -"##; - -const POST_HASH_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Hash -( - post_id UUID PRIMARY KEY, - hash VARCHAR(44) -); -"##; - -const TAG_MODEL: &str = r##" -CREATE TABLE IF NOT EXISTS Tag -( - post_id UUID NOT NULL, - tag VARCHAR(255) NOT NULL -); -"##; - -pub async fn init(db_connection: &Pool) { - query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); - query(POST_META_MODEL).execute(db_connection).await.unwrap(); - query(TAG_MODEL).execute(db_connection).await.unwrap(); - query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); - info!("database schema initialized"); -} diff --git a/backend/api_server/src/init/server.rs b/backend/api_server/src/init/server.rs deleted file mode 100644 index 703a043..0000000 --- a/backend/api_server/src/init/server.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::env; -use std::error::Error; -use std::num::ParseIntError; -use std::process::exit; - -use crate::blog; -use axum::Router; -use constants::config::server::{default_value, var_name}; -use macros::panic_with_log; -use sqlx::{Pool, Postgres}; -use tracing::{error, info, warn, Level}; - -pub async fn start(db_pool: Pool) -> Result<(), Box> { - let app = Router::new() - .nest("/blog", blog::get_router()) - .with_state(db_pool); - - let port: u32 = match env::var(var_name::PORT) { - Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { - warn!( - "failed to parse `{}`, using default port: {}", - var_name::PORT, - e.to_string() - ); - default_value::PORT - }), - Err(_) => default_value::PORT, - }; - - let listener = - match tokio::net::TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await { - Ok(listener) => { - info!("server started on {}:{}", default_value::HOST, port); - listener - } - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to start server on {}:{}: {}", - default_value::HOST, - port, - e.to_string() - ); - } - }; - - axum::serve(listener, app).await.unwrap_or_else(|e| { - error!("failed to start axum: {}", e.to_string()); - exit(1); - }); - Ok(()) -} diff --git a/backend/api_server/src/main.rs b/backend/api_server/src/main.rs deleted file mode 100644 index 8aab2c7..0000000 --- a/backend/api_server/src/main.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod blog; -mod init; -mod scheduler; -mod utils; - -#[tokio::main] -async fn main() { - init::load_env(); - - let _guards = init::logger::init(); - - init::start().await; -} diff --git a/backend/api_server/src/scheduler/mod.rs b/backend/api_server/src/scheduler/mod.rs deleted file mode 100644 index ce86797..0000000 --- a/backend/api_server/src/scheduler/mod.rs +++ /dev/null @@ -1,194 +0,0 @@ -use crate::{ - scheduler::scheduler_error::SchedulerError, - scheduler::scheduler_error::SchedulerError::IdAlreadyRunning, - scheduler::scheduler_error::SchedulerError::IdNotFound, scheduler::sleeper::SkippableSleeper, - scheduler::tasks::Task, -}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::RwLock; -use std::{collections::HashMap, sync::Arc, thread}; -use tracing::log::error; -use uuid::Uuid; - -pub mod scheduler_error; -mod sleeper; -mod task_func; -pub mod tasks; - -struct TaskStatus { - sleeper: Option>, - is_alive: AtomicBool, -} - -#[derive(Default)] -pub struct Scheduler { - tasks: Arc>>>>, - running: Arc>>>, -} - -impl Scheduler { - /// Start all tasks in the scheduler - pub fn start(&self) -> Result<(), SchedulerError> { - for (k, _v) in self.tasks.read()?.iter() { - self.run_non_blocking(k)? - } - Ok(()) - } - - /// Terminate all tasks in the scheduler - pub fn halt() {} - - fn run_task( - running: Arc>>>, - id: Uuid, - task: Arc>, - ) -> Result<(), SchedulerError> { - if running.read()?.get(&id).is_some() { - return Err(IdAlreadyRunning); - } else { - running.write()?.insert( - id, - TaskStatus { - sleeper: None, - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - loop { - if let Some(next_execution) = task.get_next_execution() { - // guarantee is alive - let task_status = { - let running_reading_guard = running.read()?; - running_reading_guard.get(&id).cloned().clone() - }; - if let Some(task_status) = task_status { - if !&task_status.is_alive.load(Ordering::SeqCst) { - { - let mut running_lock_guard = running.write()?; - running_lock_guard.remove(&id); - } - return Ok(()); - } else { - let sleeper = Arc::from(SkippableSleeper::from(next_execution)); - { - let mut running_lock_guard = running.write()?; - running_lock_guard.insert( - id, - TaskStatus { - sleeper: Some(sleeper.clone()), - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - sleeper.start(); - task.call(); - } - } - } else { - error!("unexpected error "); - break; - } - } - Ok(()) - } - - /// Run a task with `id` - pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - Self::run_task(self.running.clone(), id, task.clone()) - } else { - Err(IdNotFound) - } - } - - pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - let running_clone = self.running.clone(); - let task_clone = task.clone(); - thread::spawn(move || Self::run_task(running_clone, id, task_clone)); - Ok(()) - } else { - Err(IdNotFound) - } - } - - fn get_running(&self, id: &Uuid) -> Option> { - let ret = self.running.read().unwrap().get(id).cloned(); - ret - } - - /// Stop a task with `id` - pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - status.is_alive.store(false, Ordering::SeqCst); - if let Some(s) = &status.sleeper { - s.skip(); - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } - } - } - - pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - if let Some(s) = &status.sleeper { - s.skip() - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } - } - } - - pub fn insert(&self, task: Box) -> Result { - let id = Uuid::new_v4(); - self.tasks.write()?.insert(id, Arc::new(task)); - Ok(id) - } - - pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { - tasks.into_iter().map(|t| self.insert(t)).collect() - } - - pub fn new() -> Self { - Self::default() - } -} - -#[cfg(test)] -mod tests { - use crate::scheduler::tasks::basic::BasicTask; - use crate::scheduler::Scheduler; - use std::error::Error; - use std::thread; - use std::time::Duration; - - #[test] - fn test_run() -> Result<(), Box> { - let scheduler = Scheduler::default(); - let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; - let id = scheduler.insert(Box::new(task))?; - for i in 1..=2 { - println!("Loop {} start", i); - println!("Run {}", i); - scheduler.run_non_blocking(&id)?; - thread::sleep(Duration::from_secs(4)); - scheduler.stop(&id)?; - println!("Stop {}", i); - thread::sleep(Duration::from_secs(2)); - println!("Loop {} finished", i); - } - Ok(()) - } -} diff --git a/backend/api_server/src/scheduler/scheduler_error.rs b/backend/api_server/src/scheduler/scheduler_error.rs deleted file mode 100644 index 43f30dd..0000000 --- a/backend/api_server/src/scheduler/scheduler_error.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; -use std::error::Error; -use std::fmt::{Debug, Display, Formatter}; -use std::sync::PoisonError; - -pub enum SchedulerError { - IdNotFound, - IdAlreadyRunning, - MutexPoison(String), -} - -impl From> for SchedulerError { - fn from(value: PoisonError) -> Self { - MutexPoison(value.to_string()) - } -} - -impl Debug for SchedulerError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self) - } -} - -impl Display for SchedulerError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - SchedulerError::IdNotFound => { - "SchedulerError::IdNotFound".to_string() - } - SchedulerError::IdAlreadyRunning => { - "SchedulerError::IdAlreadyRunning".to_string() - } - MutexPoison(str) => { - format!("SchedulerError::MutexPoison: {}", str) - } - } - ) - } -} - -impl Error for SchedulerError {} diff --git a/backend/api_server/src/scheduler/sleeper.rs b/backend/api_server/src/scheduler/sleeper.rs deleted file mode 100644 index 4155e57..0000000 --- a/backend/api_server/src/scheduler/sleeper.rs +++ /dev/null @@ -1,77 +0,0 @@ -use chrono::{DateTime, Utc}; -use std::sync::{Arc, Condvar, Mutex}; -use std::thread; -use std::time::Duration; - -#[derive(Clone, Debug, Default)] -pub struct SkippableSleeper { - pub duration: Duration, - state: Arc<(Mutex, Condvar)>, -} - -impl SkippableSleeper { - pub fn new(duration: Duration) -> Self { - SkippableSleeper { - duration, - state: Arc::new((Mutex::new(false), Condvar::new())), - } - } - - // Blocks until the duration has passed or skip() is called - pub fn start(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - let state_clone = Arc::clone(&self.state); - let duration_clone = self.duration; - thread::spawn(move || { - thread::sleep(duration_clone); - let (lock, cvar) = &*state_clone; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); - }); - while !*received { - received = cvar.wait(received).unwrap(); - } - } - - // Skip the blocking delay - pub fn skip(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); - } -} - -impl From> for SkippableSleeper { - fn from(value: DateTime) -> Self { - let from_utc_timestamp = value.timestamp() as u64; - let curr_utc_timestamp = Utc::now().timestamp() as u64; - SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) - } -} - -#[cfg(test)] -mod test { - use crate::scheduler::sleeper::SkippableSleeper; - use std::thread; - use std::time::Duration; - - #[test] - fn usage() { - let sleeper = SkippableSleeper::new(Duration::new(10, 0)); - thread::spawn({ - let sleeper = sleeper.clone(); - move || { - println!("Sleeping..."); - sleeper.start(); - println!("Woke up!"); - } - }); - println!("Start Sleeping..."); - thread::sleep(Duration::from_secs(4)); - println!("Skipping sleep..."); - sleeper.skip(); - } -} diff --git a/backend/api_server/src/scheduler/task_func.rs b/backend/api_server/src/scheduler/task_func.rs deleted file mode 100644 index af27ded..0000000 --- a/backend/api_server/src/scheduler/task_func.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub trait TaskFunc: Fn() + Send + Sync + 'static {} - -impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} diff --git a/backend/api_server/src/scheduler/tasks/basic.rs b/backend/api_server/src/scheduler/tasks/basic.rs deleted file mode 100644 index 71fc3a6..0000000 --- a/backend/api_server/src/scheduler/tasks/basic.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::scheduler::task_func::TaskFunc; -use crate::scheduler::tasks::Task; -use chrono::{DateTime, Utc}; -use cron::Schedule; -use std::error::Error; -use std::str::FromStr; - -pub struct BasicTask { - schedule: Schedule, - func: Box, -} - -impl BasicTask { - pub fn new(func: F, cron_expr: &str) -> Result> - where - F: TaskFunc, - { - Ok(BasicTask { - schedule: Schedule::from_str(cron_expr)?, - func: Box::new(func), - }) - } - - pub fn to_task(self) -> Box { - Box::new(self) - } -} - -impl Task for BasicTask { - fn get_next_execution(&self) -> Option> { - self.schedule.upcoming(Utc).next() - } - - fn call(&self) { - (self.func)() - } -} diff --git a/backend/api_server/src/scheduler/tasks/mod.rs b/backend/api_server/src/scheduler/tasks/mod.rs deleted file mode 100644 index 7e3b87a..0000000 --- a/backend/api_server/src/scheduler/tasks/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -use chrono::{DateTime, Utc}; - -pub mod basic; - -pub trait Task: Send + Sync { - fn get_next_execution(&self) -> Option>; - fn call(&self); -} diff --git a/backend/api_server/src/utils/git.rs b/backend/api_server/src/utils/git.rs deleted file mode 100644 index 6743058..0000000 --- a/backend/api_server/src/utils/git.rs +++ /dev/null @@ -1,195 +0,0 @@ -/// git merge -/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs -/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 -/// -use git2::{ - build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, -}; -use std::path::PathBuf; -use tracing::{error, warn}; - - -#[derive(Debug, Eq, PartialEq)] -pub struct FileDelta { - pub old_path: Option, - pub new_path: Option, - pub status: Delta, -} - -fn do_fetch<'a>( - repo: &'a Repository, - refs: &[&str], - remote: &'a mut Remote, -) -> Result, git2::Error> { - let mut fetch_option = FetchOptions::new(); - - // Always fetch all tags. - // Perform a download and also update tips - fetch_option.download_tags(AutotagOption::All); - remote.fetch(refs, Some(&mut fetch_option), None)?; - - let fetch_head = repo.find_reference("FETCH_HEAD")?; - repo.reference_to_annotated_commit(&fetch_head) -} - -fn fast_forward( - repo: &Repository, - lb: &mut Reference, - rc: &AnnotatedCommit, -) -> Result<(), git2::Error> { - let name = match lb.name() { - Some(s) => s.to_string(), - None => String::from_utf8_lossy(lb.name_bytes()).to_string(), - }; - let msg = format!("fast-forward({}) -> {}", name, rc.id()); - // println!("{}", msg); - lb.set_target(rc.id(), &msg)?; - repo.set_head(&name)?; - repo.checkout_head(Some( - build::CheckoutBuilder::default() - // For some reason, the force is required to make the working directory actually get updated - // I suspect we should be adding some logic to handle dirty working directory states, - // but this is just an example so maybe not. - .force(), - ))?; - Ok(()) -} - -fn normal_merge( - repo: &Repository, - local: &AnnotatedCommit, - remote: &AnnotatedCommit, -) -> Result<(), git2::Error> { - let local_tree = repo.find_commit(local.id())?.tree()?; - let remote_tree = repo.find_commit(remote.id())?.tree()?; - let ancestor = repo - .find_commit(repo.merge_base(local.id(), remote.id())?)? - .tree()?; - let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?; - - if idx.has_conflicts() { - warn!("merge conflicts detected"); - repo.checkout_index(Some(&mut idx), None)?; - return Ok(()); - } - let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; - // now create the merge commit - let msg = format!("merge({}) -> {}", remote.id(), local.id()); - let sig = repo.signature()?; - let local_commit = repo.find_commit(local.id())?; - let remote_commit = repo.find_commit(remote.id())?; - - // Do our merge commit and set the current branch head to that commit. - let _merge_commit = repo.commit( - Some("HEAD"), - &sig, - &sig, - &msg, - &result_tree, - &[&local_commit, &remote_commit], - )?; - // Set working tree to match head. - repo.checkout_head(None)?; - Ok(()) -} - -fn do_merge<'a>( - repo: &'a Repository, - remote_branch: &str, - fetch_commit: AnnotatedCommit<'a>, -) -> Result, git2::Error> { - // 1. do a merge analysis - let analysis = repo.merge_analysis(&[&fetch_commit])?; - - let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; - - // 2. Do the appropriate merge - if analysis.0.is_fast_forward() { - // println!("Doing a fast-forward"); - // do a fast-forward - let ref_name = format!("refs/heads/{}", remote_branch); - match repo.find_reference(&ref_name) { - Ok(mut r) => { - fast_forward(repo, &mut r, &fetch_commit)?; - } - Err(_) => { - // The branch doesn't exist, so set the reference to the - // commit directly. Usually this is because you are pulling - // into an empty repository. - repo.reference( - &ref_name, - fetch_commit.id(), - true, - &format!("({}) -> {}", remote_branch, fetch_commit.id()), - )?; - repo.set_head(&ref_name)?; - repo.checkout_head(Some( - build::CheckoutBuilder::default() - .allow_conflicts(true) - .conflict_style_merge(true) - .force(), - ))?; - } - }; - } else if analysis.0.is_normal() { - // do a normal merge - normal_merge(repo, &head_commit, &fetch_commit)?; - } - // else { - // // already up-to-date - // println!("Nothing to do..."); - // } - - calculate_diff(repo, &head_commit, &fetch_commit) -} - -fn calculate_diff( - repo: &Repository, - local: &AnnotatedCommit, - remote: &AnnotatedCommit, -) -> Result, git2::Error> { - let local_tree = repo.find_commit(local.id())?.tree()?; - let remote_tree = repo.find_commit(remote.id())?.tree()?; - let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?; - - let mut updated_files: Vec = Vec::new(); - - if let Err(e) = diff.foreach( - &mut |d, _| { - // let delta = d.status(); - // - // let n_path = d.new_file().path(); - // let o_path = d.old_file().path(); - // - // if let Some(path) = n_path { - // updated_files.insert(path.to_owned()); - // } - // if let Some(path) = o_path { - // updated_files.insert(path.to_owned()); - // } - updated_files.push(FileDelta { - old_path: d.old_file().path().map(PathBuf::from), - new_path: d.new_file().path().map(PathBuf::from), - status: d.status(), - }); - true // Continue - }, - None, - None, - None, - ) { - error!("error at calculating diff: {:?}", e); - } - - Ok(updated_files.into_iter().collect()) -} - -pub fn sync( - remote_name: &str, - remote_branch: &str, - repo: &Repository, -) -> Result, git2::Error> { - let mut remote = repo.find_remote(remote_name)?; - let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; - do_merge(repo, remote_branch, fetch_commit) -} diff --git a/backend/api_server/src/utils/mod.rs b/backend/api_server/src/utils/mod.rs deleted file mode 100644 index c2bf1c3..0000000 --- a/backend/api_server/src/utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod git; diff --git a/backend/constants/Cargo.toml b/backend/constants/Cargo.toml deleted file mode 100644 index db345c7..0000000 --- a/backend/constants/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "constants" -version = "0.1.0" -edition = "2021" - -[dependencies] diff --git a/backend/macros/src/log.rs b/backend/macros/src/log.rs index f7dd357..b3aed4e 100644 --- a/backend/macros/src/log.rs +++ b/backend/macros/src/log.rs @@ -6,7 +6,7 @@ /// use macros::panic_with_log; /// use tracing::Level; /// -/// panic_with_log!(Level::ERROR, "Test panicked: {}", 42); +/// panic_with_log!(Level::ERROR, "test panicked: {}", 42); /// ``` #[macro_export] macro_rules! panic_with_log { @@ -18,6 +18,25 @@ macro_rules! panic_with_log { }}; } +/// log the message with `Level::ERROR` and exit(1) +/// +/// # Example +/// ```rust +/// +/// use macros::error_panic; +/// +/// error_panic!("test panicked: {}", 42); +/// ``` +#[macro_export] +macro_rules! error_panic { + ($msg:literal $(, $($arg:tt)*)?) => {{ + use std::process; + use tracing::{event, Level}; + event!(Level::ERROR, $msg $(, $($arg)*)?); + process::exit(1); + }}; +} + #[cfg(test)] mod tests { #[test] diff --git a/backend/server/schema.sql b/backend/server/schema.sql new file mode 100644 index 0000000..675175e --- /dev/null +++ b/backend/server/schema.sql @@ -0,0 +1,32 @@ +CREATE TABLE tag +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT +); + +CREATE TABLE category +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT +); + +CREATE TABLE post +( + id SERIAL PRIMARY KEY, + identifier VARCHAR(50) UNIQUE NOT NULL, + title VARCHAR(255) NOT NULL, + subtitle VARCHAR(255) NOT NULL, + published_date DATE NOT NULL, + last_update DATE, + content TEXT NOT NULL, + + category SERIAL REFERENCES category (id) NOT NULL +); + +CREATE TABLE post_tag +( + post SERIAL REFERENCES post (id), + tag SERIAL REFERENCES tag (id) +) \ No newline at end of file diff --git a/backend/constants/src/blog.rs b/backend/server/src/constants/blog.rs similarity index 100% rename from backend/constants/src/blog.rs rename to backend/server/src/constants/blog.rs diff --git a/backend/constants/src/config.rs b/backend/server/src/constants/config.rs similarity index 100% rename from backend/constants/src/config.rs rename to backend/server/src/constants/config.rs diff --git a/backend/constants/src/lib.rs b/backend/server/src/constants/mod.rs similarity index 100% rename from backend/constants/src/lib.rs rename to backend/server/src/constants/mod.rs diff --git a/backend/constants/src/time.rs b/backend/server/src/constants/time.rs similarity index 100% rename from backend/constants/src/time.rs rename to backend/server/src/constants/time.rs diff --git a/backend/server/src/main.rs b/backend/server/src/main.rs new file mode 100644 index 0000000..5e6abc2 --- /dev/null +++ b/backend/server/src/main.rs @@ -0,0 +1,7 @@ +mod constants; + +const BUILD_ID: &str = env!("BUILD_ID"); + +fn main() { + println!("{}", BUILD_ID); +} diff --git a/loadbalancer/.gitignore b/loadbalancer/.gitignore new file mode 100644 index 0000000..596b9e4 --- /dev/null +++ b/loadbalancer/.gitignore @@ -0,0 +1,3 @@ +target/ + +Cargo.lock \ No newline at end of file diff --git a/openresty/conf/nocturne.conf b/nocturne.conf similarity index 65% rename from openresty/conf/nocturne.conf rename to nocturne.conf index 787c98f..00df227 100644 --- a/openresty/conf/nocturne.conf +++ b/nocturne.conf @@ -1,24 +1,17 @@ -# server { -# listen 80; -# server_name lumither.com; - -# return 301 https://$host$request_uri; -# } - server { listen 80; - # listen 443 ssl; - # server_name lumither.com; + listen 443 ssl; + server_name lumither.com; # ssl_certificate certs/cert.pem; # ssl_certificate_key certs/key.pem; - # ssl_session_cache shared:SSL:1m; - # ssl_session_timeout 5m; + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 5m; - # ssl_ciphers HIGH:!aNULL:!MD5; - # ssl_prefer_server_ciphers on; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; location / { default_type text/html; diff --git a/openresty/deploy.sh b/openresty/deploy.sh deleted file mode 100755 index dc0a79c..0000000 --- a/openresty/deploy.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -rm -rf /opt/openresty/nginx/lua -cp -r ./lua /opt/openresty/nginx diff --git a/openresty/lua/hello_world.lua b/openresty/lua/hello_world.lua deleted file mode 100644 index 7d90d01..0000000 --- a/openresty/lua/hello_world.lua +++ /dev/null @@ -1 +0,0 @@ -ngx.say("hello world") diff --git a/scripts/dev_server.sh b/scripts/dev_server.sh deleted file mode 100755 index 2082756..0000000 --- a/scripts/dev_server.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# refresh env -/bin/bash ./flush_env.sh - -# start frontend -cd frontend || exit -screen -S nocturne_frontend -d -m bun dev -cd .. - -# start backend -cd backend || exit -screen -S nocturne_backend -d -m cargo r -cd .. diff --git a/scripts/flush_env.sh b/scripts/flush_env.sh deleted file mode 100755 index f746f85..0000000 --- a/scripts/flush_env.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/env bash - -cp .env ./backend -cp .env ./frontend -cp .env.local ./backend -cp .env.local ./frontend diff --git a/scripts/refresh.sh b/scripts/refresh.sh deleted file mode 100755 index f2fa921..0000000 --- a/scripts/refresh.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -source .env - -if [ -z "$BLOG_ENDPOINT" ]; then - echo "BLOG_ENDPOINT is not set in the .env file" - exit 1 -fi - -if [ -z "$GIT_WORK_DIR" ]; then - echo "GIT_WORK_DIR is not set in the .env file" - exit 1 -fi - -if [ -z "$BACKEND_PORT" ]; then - echo "BACKEND_PORT is not set in the .env file" - exit 1 -fi - -check_for_updates() { - cd "$GIT_WORK_DIR" || exit - res=$(git pull) - if echo "$res" | grep -q "Already up to date"; then - echo "Repo is up to date." - else - echo "Repo has updated; calling db update" - curl --request POST -sL \ - --url "localhost:$BACKEND_PORT" - #todo: to be continued - fi -} - -if [ -d "$GIT_WORK_DIR" ]; then - check_for_updates -else - git clone "$BLOG_ENDPOINT" "$GIT_WORK_DIR" - if [ $? -eq 0 ]; then - check_for_updates - else - echo "Failed to clone the repo" - fi -fi diff --git a/scripts/setup.sh b/scripts/setup.sh deleted file mode 100755 index 2efab7c..0000000 --- a/scripts/setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. "$DIR/.env" - -sudo groupadd nocturne -USERNAME="$(whoami)" -echo "$USERNAME" -sudo usermod -aG nocturne "$USERNAME" - -sudo mkdir -p "$LOG_ROOT_DIR/backend" "$LOG_ROOT_DIR/frontend" -sudo chown -R :nocturne "$LOG_ROOT_DIR" -sudo chmod -R 775 "$LOG_ROOT_DIR" - diff --git a/scripts/start_server.sh b/scripts/start_server.sh deleted file mode 100755 index 50c2199..0000000 --- a/scripts/start_server.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# refresh env -/bin/bash ./flush_env.sh - -# start frontend -( -cd frontend || exit -bun i -bun next build -screen -S nocturne_frontend -d -m bun start -cd .. -) & - -# start backend -( -cd backend || exit -screen -S nocturne_backend -d -m cargo r -r -cd .. -) & wait From eef32ad6951e4f27362c3914795f42eb5c2aa6d0 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 17 Feb 2025 15:28:33 -0800 Subject: [PATCH 03/32] feat(backend, lb) add missing components --- backend/server/Cargo.toml | 15 +++++++++++++++ backend/server/build.rs | 15 +++++++++++++++ loadbalancer/Cargo.toml | 6 ++++++ loadbalancer/src/main.rs | 3 +++ 4 files changed, 39 insertions(+) create mode 100644 backend/server/Cargo.toml create mode 100644 backend/server/build.rs create mode 100644 loadbalancer/Cargo.toml create mode 100644 loadbalancer/src/main.rs diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml new file mode 100644 index 0000000..481e539 --- /dev/null +++ b/backend/server/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "server" +version = "0.1.0" +edition = "2021" + +[dependencies] +macros = { path = "../macros" } + + +tracing = "0.1.41" +git2 = "0.20.0" + +[build-dependencies] +git2 = "0.20.0" +chrono = "0.4.39" \ No newline at end of file diff --git a/backend/server/build.rs b/backend/server/build.rs new file mode 100644 index 0000000..b7d91af --- /dev/null +++ b/backend/server/build.rs @@ -0,0 +1,15 @@ +use chrono::Utc; +use git2::Repository; + +fn main() { + let repo = Repository::open("../..").expect("failed to open repository"); + let commit = repo + .head() + .expect("failed to get HEAD") + .peel_to_commit() + .expect("failed to get HEAD commit"); + let hash = commit.id().to_string(); + let utc_date = Utc::now().date_naive().format("%Y%m%d").to_string(); + let build_id = format!("{}@{}", utc_date, &hash[..8]); + println!("cargo:rustc-env=BUILD_ID={}", build_id); +} diff --git a/loadbalancer/Cargo.toml b/loadbalancer/Cargo.toml new file mode 100644 index 0000000..bf98b23 --- /dev/null +++ b/loadbalancer/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "loadbalancer" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/loadbalancer/src/main.rs b/loadbalancer/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/loadbalancer/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From 5da9ca3ae1461b51879a97a459a7830beb4ef435 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 17 Feb 2025 15:58:11 -0800 Subject: [PATCH 04/32] feat(backend) move module `scheduler` --- backend/server/Cargo.toml | 4 +- backend/server/src/main.rs | 1 + backend/server/src/scheduler/mod.rs | 202 ++++++++++++++++++ .../server/src/scheduler/scheduler_error.rs | 47 ++++ backend/server/src/scheduler/sleeper.rs | 80 +++++++ backend/server/src/scheduler/task_func.rs | 3 + backend/server/src/scheduler/tasks/basic.rs | 37 ++++ backend/server/src/scheduler/tasks/mod.rs | 8 + 8 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 backend/server/src/scheduler/mod.rs create mode 100644 backend/server/src/scheduler/scheduler_error.rs create mode 100644 backend/server/src/scheduler/sleeper.rs create mode 100644 backend/server/src/scheduler/task_func.rs create mode 100644 backend/server/src/scheduler/tasks/basic.rs create mode 100644 backend/server/src/scheduler/tasks/mod.rs diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml index 481e539..1070414 100644 --- a/backend/server/Cargo.toml +++ b/backend/server/Cargo.toml @@ -7,8 +7,10 @@ edition = "2021" macros = { path = "../macros" } -tracing = "0.1.41" +tracing = { version = "0.1.41", features = ["log"] } git2 = "0.20.0" +chrono = "0.4.39" +uuid = { version = "1.13.2", features = ["v4"] } [build-dependencies] git2 = "0.20.0" diff --git a/backend/server/src/main.rs b/backend/server/src/main.rs index 5e6abc2..655a616 100644 --- a/backend/server/src/main.rs +++ b/backend/server/src/main.rs @@ -1,4 +1,5 @@ mod constants; +mod scheduler; const BUILD_ID: &str = env!("BUILD_ID"); diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs new file mode 100644 index 0000000..71cb62e --- /dev/null +++ b/backend/server/src/scheduler/mod.rs @@ -0,0 +1,202 @@ +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, RwLock, + }, + thread, +}; + +use crate::scheduler::{ + scheduler_error::{ + SchedulerError, SchedulerError::IdAlreadyRunning, SchedulerError::IdNotFound, + }, + sleeper::SkippableSleeper, + tasks::Task, +}; + +use tracing::log::error; +use uuid::Uuid; + +pub mod scheduler_error; +mod sleeper; +mod task_func; +pub mod tasks; + +struct TaskStatus { + sleeper: Option>, + is_alive: AtomicBool, +} + +#[derive(Default)] +pub struct Scheduler { + tasks: Arc>>>>, + running: Arc>>>, +} + +impl Scheduler { + /// Start all tasks in the scheduler + pub fn start(&self) -> Result<(), SchedulerError> { + for (k, _v) in self.tasks.read()?.iter() { + self.run_non_blocking(k)? + } + Ok(()) + } + + /// Terminate all tasks in the scheduler + pub fn halt() {} + + fn run_task( + running: Arc>>>, + id: Uuid, + task: Arc>, + ) -> Result<(), SchedulerError> { + if running.read()?.get(&id).is_some() { + return Err(IdAlreadyRunning); + } else { + running.write()?.insert( + id, + TaskStatus { + sleeper: None, + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + loop { + if let Some(next_execution) = task.get_next_execution() { + // guarantee is alive + let task_status = { + let running_reading_guard = running.read()?; + running_reading_guard.get(&id).cloned().clone() + }; + if let Some(task_status) = task_status { + if !&task_status.is_alive.load(Ordering::SeqCst) { + { + let mut running_lock_guard = running.write()?; + running_lock_guard.remove(&id); + } + return Ok(()); + } else { + let sleeper = Arc::from(SkippableSleeper::from(next_execution)); + { + let mut running_lock_guard = running.write()?; + running_lock_guard.insert( + id, + TaskStatus { + sleeper: Some(sleeper.clone()), + is_alive: AtomicBool::new(true), + } + .into(), + ); + } + sleeper.start(); + task.call(); + } + } + } else { + error!("unexpected error "); + break; + } + } + Ok(()) + } + + /// Run a task with `id` + pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + Self::run_task(self.running.clone(), id, task.clone()) + } else { + Err(IdNotFound) + } + } + + pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { + let id = id.to_owned(); + if let Some(task) = self.tasks.read()?.get(&id) { + let running_clone = self.running.clone(); + let task_clone = task.clone(); + thread::spawn(move || Self::run_task(running_clone, id, task_clone)); + Ok(()) + } else { + Err(IdNotFound) + } + } + + fn get_running(&self, id: &Uuid) -> Option> { + let ret = self.running.read().unwrap().get(id).cloned(); + ret + } + + /// Stop a task with `id` + pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + status.is_alive.store(false, Ordering::SeqCst); + if let Some(s) = &status.sleeper { + s.skip(); + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { + match self.get_running(id) { + None => Err(IdNotFound), + Some(status) => { + if let Some(s) = &status.sleeper { + s.skip() + } else { + return Err(IdAlreadyRunning); + } + Ok(()) + } + } + } + + pub fn insert(&self, task: Box) -> Result { + let id = Uuid::new_v4(); + self.tasks.write()?.insert(id, Arc::new(task)); + Ok(id) + } + + pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { + tasks.into_iter().map(|t| self.insert(t)).collect() + } + + pub fn new() -> Self { + Self::default() + } +} + +#[cfg(test)] +mod tests { + use crate::scheduler::tasks::basic::BasicTask; + use crate::scheduler::Scheduler; + use std::error::Error; + use std::thread; + use std::time::Duration; + + #[test] + fn test_run() -> Result<(), Box> { + let scheduler = Scheduler::default(); + let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; + let id = scheduler.insert(Box::new(task))?; + for i in 1..=2 { + println!("Loop {} start", i); + println!("Run {}", i); + scheduler.run_non_blocking(&id)?; + thread::sleep(Duration::from_secs(4)); + scheduler.stop(&id)?; + println!("Stop {}", i); + thread::sleep(Duration::from_secs(2)); + println!("Loop {} finished", i); + } + Ok(()) + } +} diff --git a/backend/server/src/scheduler/scheduler_error.rs b/backend/server/src/scheduler/scheduler_error.rs new file mode 100644 index 0000000..d9783f7 --- /dev/null +++ b/backend/server/src/scheduler/scheduler_error.rs @@ -0,0 +1,47 @@ +use std::{ + error::Error, + fmt::{Debug, Display, Formatter}, + sync::PoisonError, +}; + +use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; + +pub enum SchedulerError { + IdNotFound, + IdAlreadyRunning, + MutexPoison(String), +} + +impl From> for SchedulerError { + fn from(value: PoisonError) -> Self { + MutexPoison(value.to_string()) + } +} + +impl Debug for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) + } +} + +impl Display for SchedulerError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + SchedulerError::IdNotFound => { + "SchedulerError::IdNotFound".to_string() + } + SchedulerError::IdAlreadyRunning => { + "SchedulerError::IdAlreadyRunning".to_string() + } + MutexPoison(str) => { + format!("SchedulerError::MutexPoison: {}", str) + } + } + ) + } +} + +impl Error for SchedulerError {} diff --git a/backend/server/src/scheduler/sleeper.rs b/backend/server/src/scheduler/sleeper.rs new file mode 100644 index 0000000..abc085a --- /dev/null +++ b/backend/server/src/scheduler/sleeper.rs @@ -0,0 +1,80 @@ +use std::{ + sync::{Arc, Condvar, Mutex}, + thread, + time::Duration, +}; + +use chrono::{DateTime, Utc}; + +#[derive(Clone, Debug, Default)] +pub struct SkippableSleeper { + pub duration: Duration, + state: Arc<(Mutex, Condvar)>, +} + +impl SkippableSleeper { + pub fn new(duration: Duration) -> Self { + SkippableSleeper { + duration, + state: Arc::new((Mutex::new(false), Condvar::new())), + } + } + + // Blocks until the duration has passed or skip() is called + pub fn start(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + let state_clone = Arc::clone(&self.state); + let duration_clone = self.duration; + thread::spawn(move || { + thread::sleep(duration_clone); + let (lock, cvar) = &*state_clone; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + }); + while !*received { + received = cvar.wait(received).unwrap(); + } + } + + // Skip the blocking delay + pub fn skip(&self) { + let (lock, cvar) = &*self.state; + let mut received = lock.lock().unwrap(); + *received = true; + cvar.notify_one(); + } +} + +impl From> for SkippableSleeper { + fn from(value: DateTime) -> Self { + let from_utc_timestamp = value.timestamp() as u64; + let curr_utc_timestamp = Utc::now().timestamp() as u64; + SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) + } +} + +#[cfg(test)] +mod test { + use crate::scheduler::sleeper::SkippableSleeper; + use std::thread; + use std::time::Duration; + + #[test] + fn usage() { + let sleeper = SkippableSleeper::new(Duration::new(10, 0)); + thread::spawn({ + let sleeper = sleeper.clone(); + move || { + println!("Sleeping..."); + sleeper.start(); + println!("Woke up!"); + } + }); + println!("Start Sleeping..."); + thread::sleep(Duration::from_secs(4)); + println!("Skipping sleep..."); + sleeper.skip(); + } +} diff --git a/backend/server/src/scheduler/task_func.rs b/backend/server/src/scheduler/task_func.rs new file mode 100644 index 0000000..af27ded --- /dev/null +++ b/backend/server/src/scheduler/task_func.rs @@ -0,0 +1,3 @@ +pub trait TaskFunc: Fn() + Send + Sync + 'static {} + +impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} diff --git a/backend/server/src/scheduler/tasks/basic.rs b/backend/server/src/scheduler/tasks/basic.rs new file mode 100644 index 0000000..71fc3a6 --- /dev/null +++ b/backend/server/src/scheduler/tasks/basic.rs @@ -0,0 +1,37 @@ +use crate::scheduler::task_func::TaskFunc; +use crate::scheduler::tasks::Task; +use chrono::{DateTime, Utc}; +use cron::Schedule; +use std::error::Error; +use std::str::FromStr; + +pub struct BasicTask { + schedule: Schedule, + func: Box, +} + +impl BasicTask { + pub fn new(func: F, cron_expr: &str) -> Result> + where + F: TaskFunc, + { + Ok(BasicTask { + schedule: Schedule::from_str(cron_expr)?, + func: Box::new(func), + }) + } + + pub fn to_task(self) -> Box { + Box::new(self) + } +} + +impl Task for BasicTask { + fn get_next_execution(&self) -> Option> { + self.schedule.upcoming(Utc).next() + } + + fn call(&self) { + (self.func)() + } +} diff --git a/backend/server/src/scheduler/tasks/mod.rs b/backend/server/src/scheduler/tasks/mod.rs new file mode 100644 index 0000000..7e3b87a --- /dev/null +++ b/backend/server/src/scheduler/tasks/mod.rs @@ -0,0 +1,8 @@ +use chrono::{DateTime, Utc}; + +pub mod basic; + +pub trait Task: Send + Sync { + fn get_next_execution(&self) -> Option>; + fn call(&self); +} From bd90cd94d1979c7c43935eb7f08b29be7b6f0560 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Mon, 17 Feb 2025 16:02:16 -0800 Subject: [PATCH 05/32] fix(backend) fix import error --- backend/server/Cargo.toml | 1 + backend/server/src/scheduler/tasks/basic.rs | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml index 1070414..d02915f 100644 --- a/backend/server/Cargo.toml +++ b/backend/server/Cargo.toml @@ -11,6 +11,7 @@ tracing = { version = "0.1.41", features = ["log"] } git2 = "0.20.0" chrono = "0.4.39" uuid = { version = "1.13.2", features = ["v4"] } +cron = "0.15.0" [build-dependencies] git2 = "0.20.0" diff --git a/backend/server/src/scheduler/tasks/basic.rs b/backend/server/src/scheduler/tasks/basic.rs index 71fc3a6..24124e3 100644 --- a/backend/server/src/scheduler/tasks/basic.rs +++ b/backend/server/src/scheduler/tasks/basic.rs @@ -1,10 +1,11 @@ -use crate::scheduler::task_func::TaskFunc; -use crate::scheduler::tasks::Task; -use chrono::{DateTime, Utc}; -use cron::Schedule; use std::error::Error; use std::str::FromStr; +use crate::scheduler::{task_func::TaskFunc, tasks::Task}; + +use chrono::{DateTime, Utc}; +use cron::Schedule; + pub struct BasicTask { schedule: Schedule, func: Box, From 84ec754c65c965f248a2c17505e11c5186509cc3 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 19 Feb 2025 16:24:00 -0800 Subject: [PATCH 06/32] feat(backend) update (unfinished) --- backend/macros/src/include_str.rs | 24 ++ backend/macros/src/lib.rs | 1 + backend/server/Cargo.toml | 19 +- backend/server/src/init/database.rs | 42 +++ backend/server/src/init/logger.rs | 107 ++++++ backend/server/src/init/misc.rs | 14 + backend/server/src/init/mod.rs | 51 +++ backend/server/src/init/schema.rs | 49 +++ backend/server/src/main.rs | 13 +- .../src/modules/blog/api/get_page_count.rs | 27 ++ .../server/src/modules/blog/api/get_post.rs | 153 ++++++++ .../src/modules/blog/api/get_post_list.rs | 130 +++++++ backend/server/src/modules/blog/api/mod.rs | 3 + .../modules/blog/cron/check_update/error.rs | 49 +++ .../modules/blog/cron/check_update/index.rs | 353 ++++++++++++++++++ .../src/modules/blog/cron/check_update/mod.rs | 148 ++++++++ .../modules/blog/cron/check_update/utils.rs | 88 +++++ backend/server/src/modules/blog/cron/mod.rs | 1 + backend/server/src/modules/blog/mod.rs | 50 +++ backend/server/src/modules/mod.rs | 84 +++++ backend/server/src/scheduler/mod.rs | 10 +- .../server/src/scheduler/scheduler_error.rs | 1 + backend/server/src/scheduler/tasks/basic.rs | 6 +- backend/server/src/scheduler/tasks/mod.rs | 2 +- backend/server/src/utils/git.rs | 195 ++++++++++ backend/server/src/utils/mod.rs | 1 + 26 files changed, 1607 insertions(+), 14 deletions(-) create mode 100644 backend/macros/src/include_str.rs create mode 100644 backend/server/src/init/database.rs create mode 100644 backend/server/src/init/logger.rs create mode 100644 backend/server/src/init/misc.rs create mode 100644 backend/server/src/init/mod.rs create mode 100644 backend/server/src/init/schema.rs create mode 100644 backend/server/src/modules/blog/api/get_page_count.rs create mode 100644 backend/server/src/modules/blog/api/get_post.rs create mode 100644 backend/server/src/modules/blog/api/get_post_list.rs create mode 100644 backend/server/src/modules/blog/api/mod.rs create mode 100644 backend/server/src/modules/blog/cron/check_update/error.rs create mode 100644 backend/server/src/modules/blog/cron/check_update/index.rs create mode 100644 backend/server/src/modules/blog/cron/check_update/mod.rs create mode 100644 backend/server/src/modules/blog/cron/check_update/utils.rs create mode 100644 backend/server/src/modules/blog/cron/mod.rs create mode 100644 backend/server/src/modules/blog/mod.rs create mode 100644 backend/server/src/modules/mod.rs create mode 100644 backend/server/src/utils/git.rs create mode 100644 backend/server/src/utils/mod.rs diff --git a/backend/macros/src/include_str.rs b/backend/macros/src/include_str.rs new file mode 100644 index 0000000..4e4a624 --- /dev/null +++ b/backend/macros/src/include_str.rs @@ -0,0 +1,24 @@ +/// Includes a UTF-8 encoded file as a string +/// +/// The file is located relative to `CARGO_MANIFEST_DIR`. +/// +/// # Example +/// ```rust +/// use macros::include_str_from_crate_root; +/// +/// let str: &str = include_str_from_crate_root!("/query/scheme.sql"); +/// +// todo: impl +// const PATH_PREFIX: &str = "/src"; +// let str: &str = include_str_from_crate_root!("{}/query/scheme.sql", PATH_PREFIX); +/// ``` +#[macro_export] +macro_rules! include_str_from_crate_root { + // ($path:literal $(, $($arg:tt)*)?) => { + // ($path:expr, $($args:tt)*) => { + // include_str!(concat!(env!("CARGO_MANIFEST_DIR"), format_args!($path, $($args)*))) + // }; + ($path:expr) => { + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), $path)) + }; +} diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs index aad31d7..6003095 100644 --- a/backend/macros/src/lib.rs +++ b/backend/macros/src/lib.rs @@ -1 +1,2 @@ +mod include_str; mod log; diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml index d02915f..33944ed 100644 --- a/backend/server/Cargo.toml +++ b/backend/server/Cargo.toml @@ -5,14 +5,25 @@ edition = "2021" [dependencies] macros = { path = "../macros" } +markdown = { path = "../markdown" } - +axum = { version = "0.8.1", features = ["macros"] } tracing = { version = "0.1.41", features = ["log"] } git2 = "0.20.0" -chrono = "0.4.39" -uuid = { version = "1.13.2", features = ["v4"] } +chrono = { version = "0.4.39", features = ["serde"] } cron = "0.15.0" +sqlx = { version = "0.8.3", features = ["runtime-tokio", "tls-native-tls", "postgres", "chrono"] } +serde_json = "1.0.133" +serde = { version = "1.0.215", features = ["derive"] } +tracing-subscriber = "0.3.18" +tracing-appender = "0.2.3" +dirs = "6.0.0" +thiserror = "2.0.11" +tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros"] } +dotenv = "0.15.0" +uuid = { version = "1.13.2", features = ["v4"] } +futures = "0.3.31" [build-dependencies] git2 = "0.20.0" -chrono = "0.4.39" \ No newline at end of file +chrono = "0.4.39" diff --git a/backend/server/src/init/database.rs b/backend/server/src/init/database.rs new file mode 100644 index 0000000..f237e1e --- /dev/null +++ b/backend/server/src/init/database.rs @@ -0,0 +1,42 @@ +use std::env; +use std::error::Error; +use std::str::FromStr; + +use crate::constants::config::database::{default_value, var_name}; + +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, Pool, Postgres}; +use tracing::info; + +pub async fn init() -> Result, Box> { + let db_connect_option = PgConnectOptions::from_str(&parse_db_uri()) + .unwrap() + .disable_statement_logging(); + Ok(PgPoolOptions::new() + .max_connections(1) // todo: temporary solution, to be fixed + .connect_with(db_connect_option) + .await?) +} + +fn parse_db_uri() -> String { + env::var(var_name::URI).unwrap_or_else(|_| { + info!( + "`{}` env var is not set, \ + using `{}`, `{}`, `{}`, `{}` and `{}`", + var_name::URI, + var_name::USERNAME, + var_name::PASSWORD, + var_name::HOST, + var_name::PORT, + var_name::DBNAME + ); + let postgres_user = env::var(var_name::USERNAME).unwrap_or(default_value::USERNAME.into()); + let postgres_password = + env::var(var_name::PASSWORD).unwrap_or(default_value::PASSWORD.into()); + let db_host = env::var(var_name::HOST).unwrap_or(default_value::HOST.into()); + let db_port = env::var(var_name::PORT).unwrap_or(default_value::PORT.into()); + let postgres_db = env::var(var_name::DBNAME).unwrap_or(default_value::DBNAME.into()); + let db_type = default_value::DATABASE_TYPE; + format!("{db_type}://{postgres_user}:{postgres_password}@{db_host}:{db_port}/{postgres_db}") + }) +} diff --git a/backend/server/src/init/logger.rs b/backend/server/src/init/logger.rs new file mode 100644 index 0000000..57ee6be --- /dev/null +++ b/backend/server/src/init/logger.rs @@ -0,0 +1,107 @@ +use std::path::Path; +use std::{env, io}; + +use crate::constants::config::general::{default_value, var_name}; + +use chrono::Local; +use dirs::home_dir; +use tracing::info; +use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; +use tracing_subscriber::{ + filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, + layer::SubscriberExt, Layer, Registry, +}; + +struct LocalTimer; + +impl FormatTime for LocalTimer { + fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result { + write!( + w, + "{}", + Local::now().format(crate::constants::time::DEFAULT_TIME_FORMAT) + ) + } +} + +type _LogWriter = dyn for<'writer> MakeWriter<'writer, Writer = dyn io::Write + 'static> + 'static; + +pub fn init() -> (WorkerGuard, WorkerGuard) { + let log_path = match env::var(var_name::LOG_DIR) { + Ok(path) => Path::new(&path).to_path_buf(), + Err(_) => { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_home_path(default_log_path) + } else { + let fallback_path = expand_home_path(default_value::WORK_DIR.to_string()); + println!( + "[warn] unset env var `{}` and `{}`, trying to log on fallback path: {}/log", + var_name::LOG_DIR, + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("log") + } + }; + + let (info_log_writer, info_guard) = non_blocking(rolling::never(&log_path, "info.log")); + let (err_log_writer, error_guard) = non_blocking(rolling::never(&log_path, "error.log")); + + let info_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let err_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_ansi(false); + let stdout_format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_source_location(true) + .with_timer(LocalTimer); + + let subscribers = Registry::default() + .with( + fmt::Layer::default() + .with_writer(info_log_writer) + .event_format(info_format), + ) + .with( + fmt::Layer::default() + .with_writer(err_log_writer) + .event_format(err_format) + .with_filter(LevelFilter::ERROR), + ) + .with( + fmt::Layer::default() + .with_writer(io::stdout) + .with_ansi(true) + .event_format(stdout_format), + ); + + tracing::subscriber::set_global_default(subscribers) + .expect("[fatal] failed to setup logging system"); + + info!("logging to dir: {}", log_path.to_str().unwrap()); + + (info_guard, error_guard) +} + +fn expand_home_path(path: String) -> String { + if path.starts_with('~') { + let home_path = home_dir() + .unwrap_or_else(|| panic!("[fatal] unable to locate home dir when parsing env `{}`, try to use absolute path instead", var_name::WORK_DIR)); + home_path + .join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + } else { + path + } +} diff --git a/backend/server/src/init/misc.rs b/backend/server/src/init/misc.rs new file mode 100644 index 0000000..6a08bc2 --- /dev/null +++ b/backend/server/src/init/misc.rs @@ -0,0 +1,14 @@ +use git2::opts::{set_server_connect_timeout_in_milliseconds, set_server_timeout_in_milliseconds}; +use tracing::warn; + +pub fn init() { + // git2 option + unsafe { + if let Err(e) = set_server_connect_timeout_in_milliseconds(10_000) { + warn!("failed to set git2 server connect timeout: {}", e); + } + if let Err(e) = set_server_timeout_in_milliseconds(60_000) { + warn!("failed to set git2 server timeout: {}", e); + } + } +} diff --git a/backend/server/src/init/mod.rs b/backend/server/src/init/mod.rs new file mode 100644 index 0000000..4da997a --- /dev/null +++ b/backend/server/src/init/mod.rs @@ -0,0 +1,51 @@ +mod database; +pub mod logger; +mod misc; +mod schema; + +use std::{env, panic}; + +use crate::modules::{blog::Blog, ModuleTree}; +use macros::error_panic; + +use tracing::info; + +pub fn load_env() { + if let Some(env_file) = env::args().nth(1) { + match dotenv::from_filename(&env_file) { + Ok(_) => { + println!("[info] env loaded from {}, starting up", env_file); + } + Err(e) => { + panic!("[fatal] failed to read {}: {}", env_file, e); + } + } + } else { + println!("[info] no .env file referred, starting up") + } +} + +pub async fn start() { + misc::init(); + + // connect database + let db_pool = match database::init().await { + Ok(pool) => { + info!("database connection pool initialized"); + pool + } + Err(e) => { + error_panic!("database connection pool failed to initialize: {}", e); + } + }; + + // // schema init + // schema::init(&db_pool).await; + + ModuleTree::new() + .add_module(Blog { + db_handler: db_pool, + }) + .blocking_serve() + .await +} diff --git a/backend/server/src/init/schema.rs b/backend/server/src/init/schema.rs new file mode 100644 index 0000000..6cd1363 --- /dev/null +++ b/backend/server/src/init/schema.rs @@ -0,0 +1,49 @@ +use sqlx::{query, Pool, Postgres}; +use tracing::info; +// +// const POST_BASE_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Post +// ( +// post_id UUID PRIMARY KEY, +// title VARCHAR(255) NOT NULL, +// category VARCHAR(255) NOT NULL, +// filename TEXT UNIQUE NOT NULL, +// summary TEXT, +// sub_title TEXT, +// content TEXT NOT NULL, +// last_update TIMESTAMPTZ, +// first_update TIMESTAMPTZ NOT NULL +// ); +// "##; +// +// // base model, will add more cols at runtime +// const POST_META_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Meta +// ( +// post_id UUID PRIMARY KEY +// ); +// "##; +// +// const POST_HASH_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Hash +// ( +// post_id UUID PRIMARY KEY, +// hash VARCHAR(44) +// ); +// "##; +// +// const TAG_MODEL: &str = r##" +// CREATE TABLE IF NOT EXISTS Tag +// ( +// post_id UUID NOT NULL, +// tag VARCHAR(255) NOT NULL +// ); +// "##; + +pub async fn init(db_connection: &Pool) { + // query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); + // query(POST_META_MODEL).execute(db_connection).await.unwrap(); + // query(TAG_MODEL).execute(db_connection).await.unwrap(); + // query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); + info!("database schema initialized"); +} diff --git a/backend/server/src/main.rs b/backend/server/src/main.rs index 655a616..8971580 100644 --- a/backend/server/src/main.rs +++ b/backend/server/src/main.rs @@ -1,8 +1,19 @@ mod constants; +mod init; +mod modules; mod scheduler; +mod utils; + +use crate::init::{logger, start}; const BUILD_ID: &str = env!("BUILD_ID"); -fn main() { +#[tokio::main] +async fn main() { println!("{}", BUILD_ID); + + init::load_env(); + let _guards = logger::init(); + + start().await; } diff --git a/backend/server/src/modules/blog/api/get_page_count.rs b/backend/server/src/modules/blog/api/get_page_count.rs new file mode 100644 index 0000000..771098f --- /dev/null +++ b/backend/server/src/modules/blog/api/get_page_count.rs @@ -0,0 +1,27 @@ +use crate::constants::blog::PAGE_SIZE; + +use axum::extract::State; +use axum::Json; +use serde_json::{json, Value}; +use sqlx::{query, PgPool, Row}; +use tracing::error; + +pub async fn get_page_count( + State(db_connection): State, +) -> Result, Json> { + match query("select cast(ceil(count(*) / $1) as integer) as res from post") + .bind(PAGE_SIZE as f32) + .fetch_one(&db_connection) + .await + { + Ok(res) => { + let res: i32 = res.get("res"); + Ok(Json::from(json!({"res": res}))) + } + Err(e) => { + // todo: rename + error!("Failed to get blog page count: {}", e.to_string()); + Err(Json::from(json!({"error": e.to_string()}))) + } + } +} diff --git a/backend/server/src/modules/blog/api/get_post.rs b/backend/server/src/modules/blog/api/get_post.rs new file mode 100644 index 0000000..2374dec --- /dev/null +++ b/backend/server/src/modules/blog/api/get_post.rs @@ -0,0 +1,153 @@ +use axum::{ + extract::{Path, State}, + http::StatusCode, + Json, +}; +use serde_json::{json, Value}; +use sqlx::{query, Error, PgPool, Row}; +use std::str::FromStr; +use tracing::error; +use uuid::Uuid; + +pub async fn get_post( + State(db_connection): State, + Path(post_id): Path, +) -> Result, (StatusCode, Json)> { + let post_id = match Uuid::from_str(post_id.as_str()) { + Ok(id) => id, + Err(e) => { + error!( + "Failed to parse UUID <{}>: {}", + post_id.as_str(), + e.to_string() + ); + return Err(( + StatusCode::BAD_REQUEST, + Json::from(json!(("error", e.to_string()))), + )); + } + }; + + let post = match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post +FROM post t +WHERE post_id = $1; +"#, + ) + .bind(post_id) + .fetch_one(&db_connection) + .await + { + Ok(value) => value.get::("post"), + Err(e) => { + error!( + "Database error on Reading `base post information` from table `post`: <{}>: {}", + post_id.to_string(), + e.to_string() + ); + return match e { + Error::RowNotFound => Err(( + StatusCode::BAD_REQUEST, + Json::from(json!({"error": e.to_string()})), + )), + _ => Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )), + }; + } + }; + + let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;") + .bind(post_id) + .fetch_all(&db_connection) + .await + { + Ok(tags) => tags.iter().map(|tag| tag.get::("tag")).collect(), + Err(e) => { + error!("Database query error :{}", e.to_string()); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + }; + + let meta = match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta +FROM meta t +WHERE post_id = $1; + "#, + ) + .bind(post_id) + .fetch_one(&db_connection) + .await + { + Ok(value) => value.get::("meta"), + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "post meta data", + "meta", + post_id.to_string(), + e.to_string() + ); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + }; + + let adjacent = match query(r" +WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1), + prev AS (SELECT post_id, t.title, t.sub_title, t.header_img + FROM meta t + WHERE t.date < (SELECT date FROM target_post) + ORDER BY t.date DESC + LIMIT 1), + next AS (SELECT post_id, t.title, t.sub_title, t.header_img + FROM meta t + WHERE t.date > (SELECT date FROM target_post) + ORDER BY t.date + LIMIT 1) +SELECT json_strip_nulls(json_build_object( + 'prev', + (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) + FROM prev), + 'next', + (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) + FROM next) + )) AS result; +" + ) + .bind(post_id) + .fetch_one(&db_connection) + .await + { + Ok(res) => {res.get::("result")} + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "adjoint posts data", + "meta", + post_id.to_string(), + e.to_string() + ); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + + }; + + let mut res = post.as_object().unwrap().to_owned(); + res.insert("tags".to_string(), tags.into()); + res.insert("meta".to_string(), meta); + res.insert("adj".to_string(), adjacent); + + Ok(Json::from(json!(res))) +} diff --git a/backend/server/src/modules/blog/api/get_post_list.rs b/backend/server/src/modules/blog/api/get_post_list.rs new file mode 100644 index 0000000..81800aa --- /dev/null +++ b/backend/server/src/modules/blog/api/get_post_list.rs @@ -0,0 +1,130 @@ +use std::error::Error; + +use crate::constants::blog::PAGE_SIZE; + +use axum::{extract::Query, extract::State, http::StatusCode, Json}; +use chrono::{DateTime, Utc}; +use futures::future::join_all; +use serde::Deserialize; +use serde_json::{json, Map, Value}; +use sqlx::{postgres::PgRow, query, PgPool, Row}; +use tracing::error; +use uuid::Uuid; + +#[derive(Deserialize)] +pub struct Params { + page: Option, +} + +#[axum::debug_handler] +pub async fn get_post_list( + State(db_connection): State, + Query(params): Query, +) -> Result, (StatusCode, Json)> { + let search_param_page = params.page.unwrap_or(1); + if search_param_page < 1 { + return Err(( + StatusCode::BAD_REQUEST, + Json::from(json!({"error": "`page` should be greater than 0"})), + )); + } + + let offset = PAGE_SIZE * (search_param_page - 1); + + let post_list = sqlx::query("SELECT * FROM post ORDER BY first_update DESC LIMIT $1 OFFSET $2") + .bind(PAGE_SIZE as i32) + .bind(offset as i32) + .fetch_all(&db_connection) + .await; + let post_list = match post_list { + Ok(post_list) => post_list, + Err(e) => { + error!( + "Database query error on fetching post list: {}", + e.to_string() + ); + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!({"error": e.to_string()})), + )); + } + }; + + let mut res = Vec::new(); + let posts: Vec<_> = post_list + .into_iter() + .map(|post| async { analyze_post(&db_connection, post).await }) + .collect(); + let posts = join_all(posts).await; + for post in posts { + match post { + Ok(p) => res.push(p), + Err(e) => { + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json::from(json!(("error", e.to_string()))), + )) + } + } + } + + // res.insert("posts".to_string(), Value::from(posts)); + Ok(Json::from(Value::from(res))) +} + +async fn analyze_post(db_connection: &PgPool, post: PgRow) -> Result> { + let post_id: Uuid = post.get("post_id"); + let tags = fetch_post_tags(db_connection, post_id).await; + let meta = fetch_post_meta(db_connection, post_id).await.unwrap(); + let header_img = meta.get("header_img"); + let last_update = meta.get("last_update"); + + Ok(json!({ + "post_id": post_id.to_string(), + "title": post.get::("title"), + "category": post.get::("category"), + "header_img": header_img, + "sub_title": post.get::("sub_title"), + "summary": post.get::("summary"), + "last_update": last_update, + "first_update": post.get::, _>("first_update"), + "tags": tags, + })) +} + +async fn fetch_post_meta(db_connection: &PgPool, post_id: Uuid) -> Option { + match query( + r#" +SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta +FROM meta t +WHERE post_id = $1; + "#, + ) + .bind(post_id) + .fetch_one(db_connection) + .await + { + Ok(value) => Some(value.get::("meta")), + Err(e) => { + error!( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + "post meta data", + "meta", + post_id.to_string(), + e.to_string() + ); + None + } + } +} + +async fn fetch_post_tags(db_connection: &PgPool, post_id: Uuid) -> Vec { + match query("SELECT tag FROM Tag WHERE post_id = $1") + .bind(post_id) + .fetch_all(db_connection) + .await + { + Ok(res) => res.iter().map(|tag| tag.get::("tag")).collect(), + Err(_) => vec![], + } +} diff --git a/backend/server/src/modules/blog/api/mod.rs b/backend/server/src/modules/blog/api/mod.rs new file mode 100644 index 0000000..5e23c3d --- /dev/null +++ b/backend/server/src/modules/blog/api/mod.rs @@ -0,0 +1,3 @@ +pub mod get_page_count; +pub mod get_post; +pub mod get_post_list; diff --git a/backend/server/src/modules/blog/cron/check_update/error.rs b/backend/server/src/modules/blog/cron/check_update/error.rs new file mode 100644 index 0000000..0d27c29 --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/error.rs @@ -0,0 +1,49 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PostIdxError { + #[error("missing essential field `{}` for file `{}`", field, filename)] + MissingField { field: String, filename: String }, + + #[error( + "invalid field `{}` for file `{}`, <{}>: {}", + field_name, + filename, + err_field, + msg + )] + InvalidField { + field_name: String, + err_field: String, + filename: String, + msg: String, + }, + + #[error( + "database error on writing `{}` to table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBWriteFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, + + #[error( + "Database error on Reading `{}` from table `{}`: <{}>: {}", + data_desc, + db_table, + id, + msg + )] + DBReadFailure { + data_desc: String, + db_table: String, + id: String, + msg: String, + }, +} diff --git a/backend/server/src/modules/blog/cron/check_update/index.rs b/backend/server/src/modules/blog/cron/check_update/index.rs new file mode 100644 index 0000000..fabd478 --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/index.rs @@ -0,0 +1,353 @@ +use std::error::Error; +use std::path::Path; + +use crate::{ + modules::blog::cron::check_update::utils::get_md_file_basename, + modules::blog::cron::check_update::{error::PostIdxError, utils::parse_date}, +}; +use markdown::MdFile; + +use sqlx::{query, PgPool, Pool, Postgres, Row}; +use tokio::runtime::Runtime; +use tracing::{info, warn}; + +async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + let post_id = post.file_id; + let title = match post.meta.get("title") { + None => { + return Err(PostIdxError::MissingField { + field: "title".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid title for `{}`", &post.filename); + "invalid title" + }), + }; + let sub_title = post + .meta + .get("sub_title") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let summary = post + .meta + .get("summary") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let content = post.content.as_str(); + let category = match post.meta.get("category") { + None => { + return Err(PostIdxError::MissingField { + field: "category".to_string(), + filename: post.filename.clone(), + }) + } + Some(title) => title.as_str().unwrap_or_else(|| { + warn!("invalid category for `{}`", &post.filename); + "invalid category" + }), + }; + let last_update = { + if let Some(last_update) = post.meta.get("last_update") { + match parse_date(last_update.as_str().unwrap()) { + Ok(date) => Some(date), + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "last_update".to_string(), + err_field: last_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + } + } else { + None + } + }; + let first_update = match post.meta.get("date") { + None => { + return Err(PostIdxError::MissingField { + field: "date".to_string(), + filename: post.filename.clone(), + }) + } + Some(first_update) => match parse_date(first_update.as_str().unwrap()) { + Ok(date) => date, + Err(e) => { + return Err(PostIdxError::InvalidField { + field_name: "date".to_string(), + err_field: first_update.to_string(), + filename: post.filename.clone(), + msg: e.to_string(), + }) + } + }, + }; + let filename = get_md_file_basename(&post.filename); + + if let Err(e) = query( + r##" +INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, filename) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (post_id) DO +UPDATE SET + title = EXCLUDED.title, + summary = EXCLUDED.summary, + content = EXCLUDED.content, + last_update = EXCLUDED.last_update, + first_update = EXCLUDED.first_update, + sub_title = EXCLUDED.sub_title, + category = EXCLUDED.category, + filename = EXCLUDED.filename; + "##, + ) + .bind(post_id) + .bind(title) + .bind(summary) + .bind(content) + .bind(last_update) + .bind(first_update) + .bind(sub_title) + .bind(category) + .bind(filename) + .execute(db_connection) + .await + { + return Err( + PostIdxError::DBWriteFailure { + data_desc: "post base information".to_string(), + db_table: "Post".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + } + ); + } + Ok(()) +} + +async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Some(tags_string) = post.meta.get("tags").to_owned() { + let tags = match tags_string.as_array().map(|vec| { + vec.iter() + .map(|val| val.as_str().unwrap()) + .collect::>() + }) { + None => { + return Err(PostIdxError::MissingField { + field: "tags".to_string(), + filename: post.filename.clone(), + }) + } + Some(val) => val, + }; + + let post_id = post.file_id; + + // remove old record + if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1") + .bind(post_id) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }); + } + + // add new record + for tag in tags { + match query("INSERT INTO Tag VALUES ($1, $2);") + .bind(post_id) + .bind(tag) + .execute(db_connection) + .await + { + Ok(_) => {} + Err(e) => { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post tag information".to_string(), + db_table: "Tag".to_string(), + id: post_id.to_string(), + msg: e.to_string(), + }) + } + } + } + } + + Ok(()) +} + +async fn hash(db_connection: &PgPool, post: &MdFile) -> Result { + let base64_form = post.hash.clone(); + match query("SELECT hash FROM Hash WHERE post_id = $1") + .bind(post.file_id) + .fetch_optional(db_connection) + .await + { + Ok(row) => { + let db_record: String = match row { + None => "".to_string(), + Some(row) => row.try_get("hash").unwrap_or("".to_string()), + }; + if base64_form == db_record { + Ok(false) + } else { + if let Err(e) = query( + r#" +INSERT INTO Hash (post_id, hash) VALUES ($1, $2) + ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#, + ) + .bind(post.file_id) + .bind(&base64_form) + .execute(db_connection) + .await + { + return Err(PostIdxError::DBWriteFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }); + } + Ok(true) + } + } + Err(e) => Err(PostIdxError::DBReadFailure { + data_desc: "post hash".to_string(), + db_table: "Hash".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }), + } +} + +async fn meta(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> { + if let Err(e) = query( + // todo: sanitization + format!( + r#" +DO +$$ + DECLARE + _post_id UUID := '{}'; + _meta_cols JSONB := '{}'; + _exists BOOLEAN; + _col_name TEXT; + _col_value TEXT; + _sql TEXT; + _insert_columns TEXT := 'post_id'; + _insert_values TEXT := quote_literal(_post_id); + _update_set TEXT := ''; + _blacklist TEXT[] := ARRAY['tags']; + BEGIN + SELECT EXISTS(SELECT 1 FROM meta WHERE post_id = _post_id) INTO _exists; + + FOR _col_name, _col_value IN SELECT key, value FROM jsonb_each_text(_meta_cols) + LOOP + IF _col_name = ANY(_blacklist) THEN + CONTINUE; + END IF; + + IF NOT EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'meta' + AND column_name = _col_name + -- may change in the future + AND table_schema = 'public') THEN + _sql := 'ALTER TABLE meta ADD COLUMN ' || quote_ident(_col_name) || ' TEXT'; + EXECUTE _sql; + END IF; + _insert_columns := _insert_columns || ', ' || quote_ident(_col_name); + _insert_values := _insert_values || ', ' || quote_literal(_col_value); + _update_set := _update_set || ', ' || quote_ident(_col_name) || ' = ' || quote_literal(_col_value); + END LOOP; + + _update_set := substring(_update_set FROM 3); + + IF _exists THEN + _sql := 'UPDATE meta SET ' || _update_set || ' WHERE post_id = ' || quote_literal(_post_id); + EXECUTE _sql; + ELSE + _sql := 'INSERT INTO meta (' || _insert_columns || ') VALUES (' || _insert_values || ')'; + EXECUTE _sql; + END IF; + END; +$$ LANGUAGE plpgsql; + "#, post.file_id.to_string().as_str(), post.meta + ).as_str() + ) + .execute(db_connection) + .await { + Err(PostIdxError::DBWriteFailure { + data_desc: "post meta data".to_string(), + db_table: "Meta".to_string(), + id: post.file_id.to_string(), + msg: e.to_string(), + }) + } else { + Ok(()) + } +} + +pub async fn index_file(path: &Path, db_conn: &Pool) -> Result<(), Box> { + let md_file = MdFile::from_file(path)?; + + async fn trail(db_conn: &Pool, md_file: &MdFile) -> Result<(), PostIdxError> { + base(db_conn, md_file).await?; + tag(db_conn, md_file).await?; + meta(db_conn, md_file).await?; + hash(db_conn, md_file).await.map(|_| ()) + } + let mut cnt = 5; + + while let Err(e) = trail(db_conn, &md_file).await { + match e { + PostIdxError::DBWriteFailure { .. } => { + cnt -= 1; + } + PostIdxError::DBReadFailure { .. } => { + cnt -= 1; + } + _ => return Err(e.into()), + } + if cnt <= 0 { + return Err(e.into()); + } + } + + info!("indexed markdown file: {}", path.display()); + Ok(()) +} + +pub fn drop_index(file_basename: &str, db_conn: &Pool) -> Result<(), Box> { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + if let Err(e) = query( + r#" +WITH deleted_post AS ( + DELETE FROM post + WHERE id = (SELECT id FROM post WHERE filename = $1) + RETURNING id +) +DELETE FROM tag WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM meta WHERE post_id IN (SELECT id FROM deleted_post); +DELETE FROM hash WHERE post_id IN (SELECT id FROM deleted_post); + "#, + ) + .bind(file_basename) + .execute(db_conn) + .await + { + Err(Box::from( + format!("failed to drop index: {}", e).to_string(), + )) + } else { + Ok(()) + } + }) +} diff --git a/backend/server/src/modules/blog/cron/check_update/mod.rs b/backend/server/src/modules/blog/cron/check_update/mod.rs new file mode 100644 index 0000000..4471e04 --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/mod.rs @@ -0,0 +1,148 @@ +use std::{ + env, + fs::remove_dir_all, + path::{Path, PathBuf}, +}; + +use crate::{ + constants::config::general::{default_value, var_name}, + modules::blog::cron::check_update::{ + index::drop_index, + utils::{expand_path, get_md_file_basename, index_files, search_md}, + }, + utils::git, +}; +use macros::panic_with_log; + +use git2::{Delta, Repository}; +use sqlx::{Pool, Postgres}; +use tracing::{error, info, warn, Level}; + +mod error; +mod index; +mod utils; + +pub fn task(db_connection: &Pool) { + // init + let git_work_dir = { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_path(default_log_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") + }; + + // clone & check update + + let git_url = match env::var(var_name::BLOG_GIT_URL) { + Ok(url) => url, + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to read `{}`: {}", + var_name::BLOG_GIT_URL, + e + ) + } + }; + let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) + .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); + let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) + .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); + + match Repository::open(&git_work_dir) { + Ok(repo) => { + // prevent git remote url mismatch (clean cache) + match repo.find_remote(&git_remote_name) { + Ok(remote) => { + if remote.url().is_some_and(|url| url != git_url) { + remove_dir_all(&git_work_dir).unwrap(); + warn!("blog content git({}) url mismatch, reinitialized", &git_url); + return; + } + } + Err(e) => { + remove_dir_all(&git_work_dir).unwrap(); + error!( + "unexpected error on indexing blog content git({}), reinitialized: {}", + &git_url, e + ); + } + } + + match git::sync(&git_remote_name, &git_remote_branch, &repo) { + Ok(updates) => { + if !updates.is_empty() { + // incremental update + info!( + "blog content git({}) update detected: {:?}", + &git_url, updates + ); + + let mut update_md: Vec = vec![]; + + for update in updates { + if [Delta::Deleted, Delta::Untracked].contains(&update.status) { + if let Some(path) = &update.old_path { + if drop_index( + &get_md_file_basename( + path.file_name().unwrap().to_str().unwrap(), + ), + db_connection, + ) + .is_ok() + { + info!( + "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", + &git_url, + &path.display() + ); + } else { + error!( + "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", + &git_url, + &path.display() + ) + } + } + } else if let Some(path) = &update.new_path { + if path.components().any(|comp| comp.as_os_str() == "posts") + && path.extension().is_some_and(|ext| ext == "md") + { + update_md.push(path.to_path_buf()); + } + } + } + + index_files(&update_md, db_connection); + } + } + Err(e) => { + error!( + "failed to sync blog content git({}) updates: {}", + &git_url, e + ); + } + } + } + Err(_) => { + match Repository::clone(&git_url, &git_work_dir) { + Ok(_repo) => { + // full repo index + let files = search_md(git_work_dir.join("posts")); + index_files(&files, db_connection); + } + Err(e) => { + error!("failed to clone blog content git({}): {}", &git_url, e); + } + } + } + }; +} diff --git a/backend/server/src/modules/blog/cron/check_update/utils.rs b/backend/server/src/modules/blog/cron/check_update/utils.rs new file mode 100644 index 0000000..85086ca --- /dev/null +++ b/backend/server/src/modules/blog/cron/check_update/utils.rs @@ -0,0 +1,88 @@ +use std::path::PathBuf; + +use crate::{ + constants::{ + config::general::{default_value, var_name}, + time::DEFAULT_DATE_FORMAT, + }, + modules::blog::cron::check_update::index::index_file, +}; + +use chrono::{DateTime, FixedOffset, ParseResult}; +use dirs::home_dir; +use futures::{executor::block_on, future::join_all}; +use sqlx::{Pool, Postgres}; +use tokio::{runtime::Runtime, spawn}; +use tracing::{error, warn}; + +pub fn search_md(entry: PathBuf) -> Vec { + if entry.ends_with(".git") { + return vec![]; + } + if entry.is_dir() { + if let Ok(files) = entry.read_dir() { + files + .flat_map(|file| { + let file_path = file.unwrap().path(); + search_md(file_path) + }) + .collect() + } else { + vec![] + } + } else if entry.extension().is_some_and(|ext| ext == "md") { + vec![entry] + } else { + vec![] + } +} + +pub fn index_files(files: &[PathBuf], db_conn: &Pool) { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let files_clone = files.to_owned(); + let handlers = files_clone + .into_iter() + .map(|file: PathBuf| { + let conn_clone = db_conn.clone(); + spawn(async move { + if let Err(e) = index_file(&file, &conn_clone).await { + error!("failed to index markdown file `{}`: {}", file.display(), e); + } + }) + }) + .collect::>(); + + block_on(join_all(handlers)); + }); +} + +pub fn expand_path(path: String) -> String { + if path.starts_with('~') { + home_dir() + .map(|dir| { + dir.join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + }) + .unwrap_or_else(|| { + warn!( + "failed to parse env var `{}`, using default value `{}`", + var_name::WORK_DIR, + default_value::WORK_DIR + ); + default_value::WORK_DIR.to_string() + }) + } else { + path + } +} + +pub fn parse_date(date_str: &str) -> ParseResult> { + DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) +} + +pub fn get_md_file_basename(f_name: &str) -> String { + f_name.replace(".md", "") +} diff --git a/backend/server/src/modules/blog/cron/mod.rs b/backend/server/src/modules/blog/cron/mod.rs new file mode 100644 index 0000000..be1305e --- /dev/null +++ b/backend/server/src/modules/blog/cron/mod.rs @@ -0,0 +1 @@ +pub mod check_update; diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs new file mode 100644 index 0000000..7832bf6 --- /dev/null +++ b/backend/server/src/modules/blog/mod.rs @@ -0,0 +1,50 @@ +mod api; +mod cron; + +use crate::{ + modules::{ + blog::{ + api::{ + get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, + }, + cron::check_update, + }, + Module, + }, + scheduler::tasks::basic::BasicTask, + scheduler::tasks::CronTask, +}; + +use axum::{routing::get, Router}; +use sqlx::PgPool; + +const MOUNT_POINT: &str = "/blog"; + +pub struct Blog { + pub db_handler: PgPool, +} + +impl Module for Blog { + fn get_server_router(&self) -> Router { + let db_handler = self.db_handler.clone(); + // todo: remove new in get func? + Router::new() + .route("/get_post_list", get(get_post_list)) + .route("/get_page_count", get(get_page_count)) + .route("/get_post/:post_id", get(get_post)) + .with_state(db_handler) + } + + fn get_mount_point(&self) -> &'static str { + MOUNT_POINT + } + + fn get_cron_tasks(&self) -> Vec> { + let db_handler = self.db_handler.clone(); + vec![ + BasicTask::new(move || check_update::task(&db_handler), "0/15 * * * * *") + .unwrap() + .to_task(), + ] + } +} diff --git a/backend/server/src/modules/mod.rs b/backend/server/src/modules/mod.rs new file mode 100644 index 0000000..dd7e83e --- /dev/null +++ b/backend/server/src/modules/mod.rs @@ -0,0 +1,84 @@ +pub mod blog; + +use std::env; +use std::num::ParseIntError; + +use crate::{ + constants::config::server::{default_value, var_name}, + scheduler::{tasks::CronTask, Scheduler}, +}; +use macros::error_panic; + +use axum::Router; +use tracing::{info, warn}; + +// todo: merge `get_server_router` & `get_mount_point`, make Option +pub trait Module { + fn get_server_router(&self) -> Router; + + fn get_mount_point(&self) -> &'static str; + + fn get_cron_tasks(&self) -> Vec>; +} + +pub struct ModuleTree { + pub modules: Vec>, +} + +impl ModuleTree { + pub fn new() -> Self { + ModuleTree { modules: vec![] } + } + + pub fn add_module(&mut self, module: impl Module + 'static) -> &mut Self { + self.modules.push(Box::new(module)); + self + } + + pub async fn blocking_serve(&self) { + let scheduler = Scheduler::new(); + let mut app = Router::new(); + + let port: u32 = match env::var(var_name::PORT) { + Ok(value) => value.parse().unwrap_or_else(|e: ParseIntError| { + warn!( + "failed to parse `{}`, using default port: {}", + var_name::PORT, + e.to_string() + ); + default_value::PORT + }), + Err(_) => default_value::PORT, + }; + + let listener = match tokio::net::TcpListener::bind(format!( + "{}:{}", + default_value::HOST, + port + )) + .await + { + Ok(listener) => { + info!("server started on {}:{}", default_value::HOST, port); + listener + } + Err(e) => { + error_panic!( + "failed to start server on {}:{}: {}", + default_value::HOST, + port, + e.to_string() + ); + } + }; + + for module in &self.modules { + let _ = scheduler.insert_list(module.get_cron_tasks()); + app = app.nest(module.get_mount_point(), module.get_server_router()); + } + + axum::serve(listener, app).await.unwrap_or_else(|e| { + error_panic!("failed to start axum: {}", e.to_string()); + }); + } +} diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index 71cb62e..edf5b1a 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -12,7 +12,7 @@ use crate::scheduler::{ SchedulerError, SchedulerError::IdAlreadyRunning, SchedulerError::IdNotFound, }, sleeper::SkippableSleeper, - tasks::Task, + tasks::CronTask, }; use tracing::log::error; @@ -30,7 +30,7 @@ struct TaskStatus { #[derive(Default)] pub struct Scheduler { - tasks: Arc>>>>, + tasks: Arc>>>>, running: Arc>>>, } @@ -49,7 +49,7 @@ impl Scheduler { fn run_task( running: Arc>>>, id: Uuid, - task: Arc>, + task: Arc>, ) -> Result<(), SchedulerError> { if running.read()?.get(&id).is_some() { return Err(IdAlreadyRunning); @@ -159,13 +159,13 @@ impl Scheduler { } } - pub fn insert(&self, task: Box) -> Result { + pub fn insert(&self, task: Box) -> Result { let id = Uuid::new_v4(); self.tasks.write()?.insert(id, Arc::new(task)); Ok(id) } - pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { + pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { tasks.into_iter().map(|t| self.insert(t)).collect() } diff --git a/backend/server/src/scheduler/scheduler_error.rs b/backend/server/src/scheduler/scheduler_error.rs index d9783f7..8961bf6 100644 --- a/backend/server/src/scheduler/scheduler_error.rs +++ b/backend/server/src/scheduler/scheduler_error.rs @@ -6,6 +6,7 @@ use std::{ use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; +// todo: refactor with thiserror pub enum SchedulerError { IdNotFound, IdAlreadyRunning, diff --git a/backend/server/src/scheduler/tasks/basic.rs b/backend/server/src/scheduler/tasks/basic.rs index 24124e3..4c69db7 100644 --- a/backend/server/src/scheduler/tasks/basic.rs +++ b/backend/server/src/scheduler/tasks/basic.rs @@ -1,7 +1,7 @@ use std::error::Error; use std::str::FromStr; -use crate::scheduler::{task_func::TaskFunc, tasks::Task}; +use crate::scheduler::{task_func::TaskFunc, tasks::CronTask}; use chrono::{DateTime, Utc}; use cron::Schedule; @@ -22,12 +22,12 @@ impl BasicTask { }) } - pub fn to_task(self) -> Box { + pub fn to_task(self) -> Box { Box::new(self) } } -impl Task for BasicTask { +impl CronTask for BasicTask { fn get_next_execution(&self) -> Option> { self.schedule.upcoming(Utc).next() } diff --git a/backend/server/src/scheduler/tasks/mod.rs b/backend/server/src/scheduler/tasks/mod.rs index 7e3b87a..8058bf6 100644 --- a/backend/server/src/scheduler/tasks/mod.rs +++ b/backend/server/src/scheduler/tasks/mod.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; pub mod basic; -pub trait Task: Send + Sync { +pub trait CronTask: Send + Sync { fn get_next_execution(&self) -> Option>; fn call(&self); } diff --git a/backend/server/src/utils/git.rs b/backend/server/src/utils/git.rs new file mode 100644 index 0000000..6743058 --- /dev/null +++ b/backend/server/src/utils/git.rs @@ -0,0 +1,195 @@ +/// git merge +/// based on https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs +/// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 +/// +use git2::{ + build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, +}; +use std::path::PathBuf; +use tracing::{error, warn}; + + +#[derive(Debug, Eq, PartialEq)] +pub struct FileDelta { + pub old_path: Option, + pub new_path: Option, + pub status: Delta, +} + +fn do_fetch<'a>( + repo: &'a Repository, + refs: &[&str], + remote: &'a mut Remote, +) -> Result, git2::Error> { + let mut fetch_option = FetchOptions::new(); + + // Always fetch all tags. + // Perform a download and also update tips + fetch_option.download_tags(AutotagOption::All); + remote.fetch(refs, Some(&mut fetch_option), None)?; + + let fetch_head = repo.find_reference("FETCH_HEAD")?; + repo.reference_to_annotated_commit(&fetch_head) +} + +fn fast_forward( + repo: &Repository, + lb: &mut Reference, + rc: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let name = match lb.name() { + Some(s) => s.to_string(), + None => String::from_utf8_lossy(lb.name_bytes()).to_string(), + }; + let msg = format!("fast-forward({}) -> {}", name, rc.id()); + // println!("{}", msg); + lb.set_target(rc.id(), &msg)?; + repo.set_head(&name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + // For some reason, the force is required to make the working directory actually get updated + // I suspect we should be adding some logic to handle dirty working directory states, + // but this is just an example so maybe not. + .force(), + ))?; + Ok(()) +} + +fn normal_merge( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result<(), git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let ancestor = repo + .find_commit(repo.merge_base(local.id(), remote.id())?)? + .tree()?; + let mut idx = repo.merge_trees(&ancestor, &local_tree, &remote_tree, None)?; + + if idx.has_conflicts() { + warn!("merge conflicts detected"); + repo.checkout_index(Some(&mut idx), None)?; + return Ok(()); + } + let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; + // now create the merge commit + let msg = format!("merge({}) -> {}", remote.id(), local.id()); + let sig = repo.signature()?; + let local_commit = repo.find_commit(local.id())?; + let remote_commit = repo.find_commit(remote.id())?; + + // Do our merge commit and set the current branch head to that commit. + let _merge_commit = repo.commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &result_tree, + &[&local_commit, &remote_commit], + )?; + // Set working tree to match head. + repo.checkout_head(None)?; + Ok(()) +} + +fn do_merge<'a>( + repo: &'a Repository, + remote_branch: &str, + fetch_commit: AnnotatedCommit<'a>, +) -> Result, git2::Error> { + // 1. do a merge analysis + let analysis = repo.merge_analysis(&[&fetch_commit])?; + + let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; + + // 2. Do the appropriate merge + if analysis.0.is_fast_forward() { + // println!("Doing a fast-forward"); + // do a fast-forward + let ref_name = format!("refs/heads/{}", remote_branch); + match repo.find_reference(&ref_name) { + Ok(mut r) => { + fast_forward(repo, &mut r, &fetch_commit)?; + } + Err(_) => { + // The branch doesn't exist, so set the reference to the + // commit directly. Usually this is because you are pulling + // into an empty repository. + repo.reference( + &ref_name, + fetch_commit.id(), + true, + &format!("({}) -> {}", remote_branch, fetch_commit.id()), + )?; + repo.set_head(&ref_name)?; + repo.checkout_head(Some( + build::CheckoutBuilder::default() + .allow_conflicts(true) + .conflict_style_merge(true) + .force(), + ))?; + } + }; + } else if analysis.0.is_normal() { + // do a normal merge + normal_merge(repo, &head_commit, &fetch_commit)?; + } + // else { + // // already up-to-date + // println!("Nothing to do..."); + // } + + calculate_diff(repo, &head_commit, &fetch_commit) +} + +fn calculate_diff( + repo: &Repository, + local: &AnnotatedCommit, + remote: &AnnotatedCommit, +) -> Result, git2::Error> { + let local_tree = repo.find_commit(local.id())?.tree()?; + let remote_tree = repo.find_commit(remote.id())?.tree()?; + let diff = repo.diff_tree_to_tree(Some(&local_tree), Some(&remote_tree), None)?; + + let mut updated_files: Vec = Vec::new(); + + if let Err(e) = diff.foreach( + &mut |d, _| { + // let delta = d.status(); + // + // let n_path = d.new_file().path(); + // let o_path = d.old_file().path(); + // + // if let Some(path) = n_path { + // updated_files.insert(path.to_owned()); + // } + // if let Some(path) = o_path { + // updated_files.insert(path.to_owned()); + // } + updated_files.push(FileDelta { + old_path: d.old_file().path().map(PathBuf::from), + new_path: d.new_file().path().map(PathBuf::from), + status: d.status(), + }); + true // Continue + }, + None, + None, + None, + ) { + error!("error at calculating diff: {:?}", e); + } + + Ok(updated_files.into_iter().collect()) +} + +pub fn sync( + remote_name: &str, + remote_branch: &str, + repo: &Repository, +) -> Result, git2::Error> { + let mut remote = repo.find_remote(remote_name)?; + let fetch_commit = do_fetch(repo, &[remote_branch], &mut remote)?; + do_merge(repo, remote_branch, fetch_commit) +} diff --git a/backend/server/src/utils/mod.rs b/backend/server/src/utils/mod.rs new file mode 100644 index 0000000..c2bf1c3 --- /dev/null +++ b/backend/server/src/utils/mod.rs @@ -0,0 +1 @@ +pub mod git; From 8358a8ae3318bdd2a3e17c8c315aabdb271116e9 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Sat, 22 Feb 2025 00:00:13 -0800 Subject: [PATCH 07/32] feat(backend) update (unfinished) --- backend/server/build.rs | 5 +- backend/server/schema.sql | 32 ---- backend/server/src/init/mod.rs | 1 - backend/server/src/init/schema.rs | 49 ------ backend/server/src/main.rs | 6 +- .../src/modules/blog/api/get_page_count.rs | 27 ---- .../modules/blog/api/get_page_count/mod.rs | 25 +++ .../api/get_page_count/post_page_count.sql | 2 + .../server/src/modules/blog/api/get_post.rs | 153 ------------------ .../blog/api/get_post/get_post_by_i64_id.sql | 0 .../blog/api/get_post/get_post_by_str_id.sql | 0 .../src/modules/blog/api/get_post/mod.rs | 30 ++++ backend/server/src/modules/blog/api/mod.rs | 2 +- backend/server/src/modules/blog/cron/mod.rs | 2 +- backend/server/src/modules/blog/mod.rs | 24 +-- backend/server/src/modules/blog/schema.sql | 41 +++++ backend/server/src/utils/git.rs | 47 ++---- {loadbalancer => gateway}/.gitignore | 0 {loadbalancer => gateway}/Cargo.toml | 2 +- {loadbalancer => gateway}/src/main.rs | 0 20 files changed, 131 insertions(+), 317 deletions(-) delete mode 100644 backend/server/schema.sql delete mode 100644 backend/server/src/init/schema.rs delete mode 100644 backend/server/src/modules/blog/api/get_page_count.rs create mode 100644 backend/server/src/modules/blog/api/get_page_count/mod.rs create mode 100644 backend/server/src/modules/blog/api/get_page_count/post_page_count.sql delete mode 100644 backend/server/src/modules/blog/api/get_post.rs create mode 100644 backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql create mode 100644 backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql create mode 100644 backend/server/src/modules/blog/api/get_post/mod.rs create mode 100644 backend/server/src/modules/blog/schema.sql rename {loadbalancer => gateway}/.gitignore (100%) rename {loadbalancer => gateway}/Cargo.toml (73%) rename {loadbalancer => gateway}/src/main.rs (100%) diff --git a/backend/server/build.rs b/backend/server/build.rs index b7d91af..aae7d8b 100644 --- a/backend/server/build.rs +++ b/backend/server/build.rs @@ -9,7 +9,8 @@ fn main() { .peel_to_commit() .expect("failed to get HEAD commit"); let hash = commit.id().to_string(); - let utc_date = Utc::now().date_naive().format("%Y%m%d").to_string(); - let build_id = format!("{}@{}", utc_date, &hash[..8]); + let utc_timestamp = Utc::now().date_naive().format("%Y%m%d").to_string(); + let pkg_version = env!("CARGO_PKG_VERSION"); + let build_id = format!("{}:{}@{}", pkg_version, &hash[..8], utc_timestamp); println!("cargo:rustc-env=BUILD_ID={}", build_id); } diff --git a/backend/server/schema.sql b/backend/server/schema.sql deleted file mode 100644 index 675175e..0000000 --- a/backend/server/schema.sql +++ /dev/null @@ -1,32 +0,0 @@ -CREATE TABLE tag -( - id SERIAL PRIMARY KEY, - name VARCHAR(20) UNIQUE NOT NULL, - description TEXT -); - -CREATE TABLE category -( - id SERIAL PRIMARY KEY, - name VARCHAR(20) UNIQUE NOT NULL, - description TEXT -); - -CREATE TABLE post -( - id SERIAL PRIMARY KEY, - identifier VARCHAR(50) UNIQUE NOT NULL, - title VARCHAR(255) NOT NULL, - subtitle VARCHAR(255) NOT NULL, - published_date DATE NOT NULL, - last_update DATE, - content TEXT NOT NULL, - - category SERIAL REFERENCES category (id) NOT NULL -); - -CREATE TABLE post_tag -( - post SERIAL REFERENCES post (id), - tag SERIAL REFERENCES tag (id) -) \ No newline at end of file diff --git a/backend/server/src/init/mod.rs b/backend/server/src/init/mod.rs index 4da997a..465d530 100644 --- a/backend/server/src/init/mod.rs +++ b/backend/server/src/init/mod.rs @@ -1,7 +1,6 @@ mod database; pub mod logger; mod misc; -mod schema; use std::{env, panic}; diff --git a/backend/server/src/init/schema.rs b/backend/server/src/init/schema.rs deleted file mode 100644 index 6cd1363..0000000 --- a/backend/server/src/init/schema.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sqlx::{query, Pool, Postgres}; -use tracing::info; -// -// const POST_BASE_MODEL: &str = r##" -// CREATE TABLE IF NOT EXISTS Post -// ( -// post_id UUID PRIMARY KEY, -// title VARCHAR(255) NOT NULL, -// category VARCHAR(255) NOT NULL, -// filename TEXT UNIQUE NOT NULL, -// summary TEXT, -// sub_title TEXT, -// content TEXT NOT NULL, -// last_update TIMESTAMPTZ, -// first_update TIMESTAMPTZ NOT NULL -// ); -// "##; -// -// // base model, will add more cols at runtime -// const POST_META_MODEL: &str = r##" -// CREATE TABLE IF NOT EXISTS Meta -// ( -// post_id UUID PRIMARY KEY -// ); -// "##; -// -// const POST_HASH_MODEL: &str = r##" -// CREATE TABLE IF NOT EXISTS Hash -// ( -// post_id UUID PRIMARY KEY, -// hash VARCHAR(44) -// ); -// "##; -// -// const TAG_MODEL: &str = r##" -// CREATE TABLE IF NOT EXISTS Tag -// ( -// post_id UUID NOT NULL, -// tag VARCHAR(255) NOT NULL -// ); -// "##; - -pub async fn init(db_connection: &Pool) { - // query(POST_BASE_MODEL).execute(db_connection).await.unwrap(); - // query(POST_META_MODEL).execute(db_connection).await.unwrap(); - // query(TAG_MODEL).execute(db_connection).await.unwrap(); - // query(POST_HASH_MODEL).execute(db_connection).await.unwrap(); - info!("database schema initialized"); -} diff --git a/backend/server/src/main.rs b/backend/server/src/main.rs index 8971580..04e2712 100644 --- a/backend/server/src/main.rs +++ b/backend/server/src/main.rs @@ -1,7 +1,11 @@ +#[allow(dead_code)] mod constants; + +#[allow(dead_code)] +mod scheduler; + mod init; mod modules; -mod scheduler; mod utils; use crate::init::{logger, start}; diff --git a/backend/server/src/modules/blog/api/get_page_count.rs b/backend/server/src/modules/blog/api/get_page_count.rs deleted file mode 100644 index 771098f..0000000 --- a/backend/server/src/modules/blog/api/get_page_count.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::constants::blog::PAGE_SIZE; - -use axum::extract::State; -use axum::Json; -use serde_json::{json, Value}; -use sqlx::{query, PgPool, Row}; -use tracing::error; - -pub async fn get_page_count( - State(db_connection): State, -) -> Result, Json> { - match query("select cast(ceil(count(*) / $1) as integer) as res from post") - .bind(PAGE_SIZE as f32) - .fetch_one(&db_connection) - .await - { - Ok(res) => { - let res: i32 = res.get("res"); - Ok(Json::from(json!({"res": res}))) - } - Err(e) => { - // todo: rename - error!("Failed to get blog page count: {}", e.to_string()); - Err(Json::from(json!({"error": e.to_string()}))) - } - } -} diff --git a/backend/server/src/modules/blog/api/get_page_count/mod.rs b/backend/server/src/modules/blog/api/get_page_count/mod.rs new file mode 100644 index 0000000..c27a427 --- /dev/null +++ b/backend/server/src/modules/blog/api/get_page_count/mod.rs @@ -0,0 +1,25 @@ +use crate::constants::blog::PAGE_SIZE; + +use axum::{extract::State, Json}; +use serde_json::{json, Value}; +use sqlx::{query, PgPool, Row}; +use tracing::error; + +const GET_PAGE_COUNT_QUERY: &str = include_str!("post_page_count.sql"); + +pub async fn handler(State(db_connection): State) -> Result, Json> { + match query(GET_PAGE_COUNT_QUERY) + .bind(PAGE_SIZE as i64) + .fetch_one(&db_connection) + .await + { + Ok(res) => { + let res: i32 = res.get("count"); + Ok(Json::from(json!({"res": res}))) + } + Err(e) => { + error!("failed to get blog page count: {}", e.to_string()); + Err(Json::from(json!({"error": e.to_string()}))) + } + } +} diff --git a/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql b/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql new file mode 100644 index 0000000..5ccce6b --- /dev/null +++ b/backend/server/src/modules/blog/api/get_page_count/post_page_count.sql @@ -0,0 +1,2 @@ +SELECT cast(ceil(count(*) / cast($1 AS FLOAT)) AS INTEGER) AS count +FROM posts; diff --git a/backend/server/src/modules/blog/api/get_post.rs b/backend/server/src/modules/blog/api/get_post.rs deleted file mode 100644 index 2374dec..0000000 --- a/backend/server/src/modules/blog/api/get_post.rs +++ /dev/null @@ -1,153 +0,0 @@ -use axum::{ - extract::{Path, State}, - http::StatusCode, - Json, -}; -use serde_json::{json, Value}; -use sqlx::{query, Error, PgPool, Row}; -use std::str::FromStr; -use tracing::error; -use uuid::Uuid; - -pub async fn get_post( - State(db_connection): State, - Path(post_id): Path, -) -> Result, (StatusCode, Json)> { - let post_id = match Uuid::from_str(post_id.as_str()) { - Ok(id) => id, - Err(e) => { - error!( - "Failed to parse UUID <{}>: {}", - post_id.as_str(), - e.to_string() - ); - return Err(( - StatusCode::BAD_REQUEST, - Json::from(json!(("error", e.to_string()))), - )); - } - }; - - let post = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS post -FROM post t -WHERE post_id = $1; -"#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("post"), - Err(e) => { - error!( - "Database error on Reading `base post information` from table `post`: <{}>: {}", - post_id.to_string(), - e.to_string() - ); - return match e { - Error::RowNotFound => Err(( - StatusCode::BAD_REQUEST, - Json::from(json!({"error": e.to_string()})), - )), - _ => Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )), - }; - } - }; - - let tags: Vec<_> = match query("SELECT tag FROM Tag WHERE post_id = $1;") - .bind(post_id) - .fetch_all(&db_connection) - .await - { - Ok(tags) => tags.iter().map(|tag| tag.get::("tag")).collect(), - Err(e) => { - error!("Database query error :{}", e.to_string()); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let meta = match query( - r#" -SELECT jsonb_strip_nulls(to_jsonb(t.*)) - 'post_id' AS meta -FROM meta t -WHERE post_id = $1; - "#, - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(value) => value.get::("meta"), - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "post meta data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - }; - - let adjacent = match query(r" -WITH target_post AS (SELECT t.date FROM meta t WHERE post_id = $1), - prev AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date < (SELECT date FROM target_post) - ORDER BY t.date DESC - LIMIT 1), - next AS (SELECT post_id, t.title, t.sub_title, t.header_img - FROM meta t - WHERE t.date > (SELECT date FROM target_post) - ORDER BY t.date - LIMIT 1) -SELECT json_strip_nulls(json_build_object( - 'prev', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM prev), - 'next', - (SELECT json_build_object('id', post_id, 'title', title, 'sub_title', sub_title, 'header_img', header_img) - FROM next) - )) AS result; -" - ) - .bind(post_id) - .fetch_one(&db_connection) - .await - { - Ok(res) => {res.get::("result")} - Err(e) => { - error!( - "Database error on Reading `{}` from table `{}`: <{}>: {}", - "adjoint posts data", - "meta", - post_id.to_string(), - e.to_string() - ); - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - Json::from(json!({"error": e.to_string()})), - )); - } - - }; - - let mut res = post.as_object().unwrap().to_owned(); - res.insert("tags".to_string(), tags.into()); - res.insert("meta".to_string(), meta); - res.insert("adj".to_string(), adjacent); - - Ok(Json::from(json!(res))) -} diff --git a/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql b/backend/server/src/modules/blog/api/get_post/get_post_by_i64_id.sql new file mode 100644 index 0000000..e69de29 diff --git a/backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql b/backend/server/src/modules/blog/api/get_post/get_post_by_str_id.sql new file mode 100644 index 0000000..e69de29 diff --git a/backend/server/src/modules/blog/api/get_post/mod.rs b/backend/server/src/modules/blog/api/get_post/mod.rs new file mode 100644 index 0000000..a9dd943 --- /dev/null +++ b/backend/server/src/modules/blog/api/get_post/mod.rs @@ -0,0 +1,30 @@ +use axum::response::IntoResponse; +use axum::{ + extract::{Path, State}, + Json, +}; +use serde_json::json; +use sqlx::PgPool; + +const GET_POST_BY_I64_ID: &str = include_str!("get_post_by_i64_id.sql"); +const GET_POST_BY_STR_ID: &str = include_str!("get_post_by_str_id.sql"); + +pub async fn handler( + State(db_connection): State, + Path(identifier): Path, +) -> impl IntoResponse { + let post = if let Ok(id) = identifier.parse::() { + select_post_with_i64_id(&db_connection, id).await + } else { + select_post_with_str_id(&db_connection, &identifier).await + }; + Json::from(json!({"res": "ok"})).into_response() +} + +async fn select_post_with_i64_id(db: &PgPool, id: i64) { + todo!() +} + +async fn select_post_with_str_id(db: &PgPool, id: &str) { + todo!() +} diff --git a/backend/server/src/modules/blog/api/mod.rs b/backend/server/src/modules/blog/api/mod.rs index 5e23c3d..3092117 100644 --- a/backend/server/src/modules/blog/api/mod.rs +++ b/backend/server/src/modules/blog/api/mod.rs @@ -1,3 +1,3 @@ pub mod get_page_count; pub mod get_post; -pub mod get_post_list; +// pub mod get_post_list; diff --git a/backend/server/src/modules/blog/cron/mod.rs b/backend/server/src/modules/blog/cron/mod.rs index be1305e..f5d8ad7 100644 --- a/backend/server/src/modules/blog/cron/mod.rs +++ b/backend/server/src/modules/blog/cron/mod.rs @@ -1 +1 @@ -pub mod check_update; +// pub mod check_update; diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index 7832bf6..9976b25 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -5,13 +5,14 @@ use crate::{ modules::{ blog::{ api::{ - get_page_count::get_page_count, get_post::get_post, get_post_list::get_post_list, + get_page_count, + get_post, + // get_post_list::get_post_list, }, - cron::check_update, + // cron::check_update, }, Module, }, - scheduler::tasks::basic::BasicTask, scheduler::tasks::CronTask, }; @@ -27,11 +28,12 @@ pub struct Blog { impl Module for Blog { fn get_server_router(&self) -> Router { let db_handler = self.db_handler.clone(); - // todo: remove new in get func? + // todo: move resource init to new func + // todo: remove debug comment Router::new() - .route("/get_post_list", get(get_post_list)) - .route("/get_page_count", get(get_page_count)) - .route("/get_post/:post_id", get(get_post)) + // .route("/get_post_list", get(get_post_list)) + .route("/get_page_count", get(get_page_count::handler)) + .route("/get_post/{identifier}", get(get_post::handler)) .with_state(db_handler) } @@ -40,11 +42,11 @@ impl Module for Blog { } fn get_cron_tasks(&self) -> Vec> { - let db_handler = self.db_handler.clone(); + // let db_handler = self.db_handler.clone(); vec![ - BasicTask::new(move || check_update::task(&db_handler), "0/15 * * * * *") - .unwrap() - .to_task(), + // BasicTask::new(move || check_update::task(&db_handler), "0/15 * * * * *") + // .unwrap() + // .to_task(), ] } } diff --git a/backend/server/src/modules/blog/schema.sql b/backend/server/src/modules/blog/schema.sql new file mode 100644 index 0000000..d6cd75d --- /dev/null +++ b/backend/server/src/modules/blog/schema.sql @@ -0,0 +1,41 @@ +CREATE TABLE tags +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT +); + +CREATE TABLE categories +( + id SERIAL PRIMARY KEY, + name VARCHAR(20) UNIQUE NOT NULL, + description TEXT +); + +CREATE TABLE posts +( + id SERIAL PRIMARY KEY, + identifier VARCHAR(50) UNIQUE NOT NULL, + title VARCHAR(255) NOT NULL, + subtitle VARCHAR(255) NOT NULL, + published_date DATE NOT NULL, + last_update DATE, + content TEXT NOT NULL, + + category SERIAL REFERENCES categories (id) NOT NULL +); + +CREATE TABLE post_tag +( + post SERIAL REFERENCES posts (id), + tag SERIAL REFERENCES tags (id) +); + +CREATE TABLE metadata +( + pid SERIAL REFERENCES posts (id) NOT NULL, + meta_key VARCHAR(50) NOT NULL, + meta_value TEXT NOT NULL, + + CONSTRAINT meta_pk PRIMARY KEY (pid, meta_key) +); diff --git a/backend/server/src/utils/git.rs b/backend/server/src/utils/git.rs index 6743058..cf906e6 100644 --- a/backend/server/src/utils/git.rs +++ b/backend/server/src/utils/git.rs @@ -16,6 +16,7 @@ pub struct FileDelta { pub status: Delta, } +#[allow(dead_code)] fn do_fetch<'a>( repo: &'a Repository, refs: &[&str], @@ -23,8 +24,6 @@ fn do_fetch<'a>( ) -> Result, git2::Error> { let mut fetch_option = FetchOptions::new(); - // Always fetch all tags. - // Perform a download and also update tips fetch_option.download_tags(AutotagOption::All); remote.fetch(refs, Some(&mut fetch_option), None)?; @@ -32,6 +31,7 @@ fn do_fetch<'a>( repo.reference_to_annotated_commit(&fetch_head) } +#[allow(dead_code)] fn fast_forward( repo: &Repository, lb: &mut Reference, @@ -45,16 +45,11 @@ fn fast_forward( // println!("{}", msg); lb.set_target(rc.id(), &msg)?; repo.set_head(&name)?; - repo.checkout_head(Some( - build::CheckoutBuilder::default() - // For some reason, the force is required to make the working directory actually get updated - // I suspect we should be adding some logic to handle dirty working directory states, - // but this is just an example so maybe not. - .force(), - ))?; + repo.checkout_head(Some(build::CheckoutBuilder::default().force()))?; Ok(()) } +#[allow(dead_code)] fn normal_merge( repo: &Repository, local: &AnnotatedCommit, @@ -73,13 +68,12 @@ fn normal_merge( return Ok(()); } let result_tree = repo.find_tree(idx.write_tree_to(repo)?)?; - // now create the merge commit + let msg = format!("merge({}) -> {}", remote.id(), local.id()); let sig = repo.signature()?; let local_commit = repo.find_commit(local.id())?; let remote_commit = repo.find_commit(remote.id())?; - // Do our merge commit and set the current branch head to that commit. let _merge_commit = repo.commit( Some("HEAD"), &sig, @@ -88,34 +82,26 @@ fn normal_merge( &result_tree, &[&local_commit, &remote_commit], )?; - // Set working tree to match head. repo.checkout_head(None)?; Ok(()) } +#[allow(dead_code)] fn do_merge<'a>( repo: &'a Repository, remote_branch: &str, fetch_commit: AnnotatedCommit<'a>, ) -> Result, git2::Error> { - // 1. do a merge analysis let analysis = repo.merge_analysis(&[&fetch_commit])?; - let head_commit = repo.reference_to_annotated_commit(&repo.head()?)?; - // 2. Do the appropriate merge if analysis.0.is_fast_forward() { - // println!("Doing a fast-forward"); - // do a fast-forward let ref_name = format!("refs/heads/{}", remote_branch); match repo.find_reference(&ref_name) { Ok(mut r) => { fast_forward(repo, &mut r, &fetch_commit)?; } Err(_) => { - // The branch doesn't exist, so set the reference to the - // commit directly. Usually this is because you are pulling - // into an empty repository. repo.reference( &ref_name, fetch_commit.id(), @@ -132,17 +118,12 @@ fn do_merge<'a>( } }; } else if analysis.0.is_normal() { - // do a normal merge normal_merge(repo, &head_commit, &fetch_commit)?; } - // else { - // // already up-to-date - // println!("Nothing to do..."); - // } - calculate_diff(repo, &head_commit, &fetch_commit) } +#[allow(dead_code)] fn calculate_diff( repo: &Repository, local: &AnnotatedCommit, @@ -156,23 +137,12 @@ fn calculate_diff( if let Err(e) = diff.foreach( &mut |d, _| { - // let delta = d.status(); - // - // let n_path = d.new_file().path(); - // let o_path = d.old_file().path(); - // - // if let Some(path) = n_path { - // updated_files.insert(path.to_owned()); - // } - // if let Some(path) = o_path { - // updated_files.insert(path.to_owned()); - // } updated_files.push(FileDelta { old_path: d.old_file().path().map(PathBuf::from), new_path: d.new_file().path().map(PathBuf::from), status: d.status(), }); - true // Continue + true }, None, None, @@ -184,6 +154,7 @@ fn calculate_diff( Ok(updated_files.into_iter().collect()) } +#[allow(dead_code)] pub fn sync( remote_name: &str, remote_branch: &str, diff --git a/loadbalancer/.gitignore b/gateway/.gitignore similarity index 100% rename from loadbalancer/.gitignore rename to gateway/.gitignore diff --git a/loadbalancer/Cargo.toml b/gateway/Cargo.toml similarity index 73% rename from loadbalancer/Cargo.toml rename to gateway/Cargo.toml index bf98b23..3417b86 100644 --- a/loadbalancer/Cargo.toml +++ b/gateway/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "loadbalancer" +name = "gateway" version = "0.1.0" edition = "2021" diff --git a/loadbalancer/src/main.rs b/gateway/src/main.rs similarity index 100% rename from loadbalancer/src/main.rs rename to gateway/src/main.rs From ca86dd236e84a4e32ba34d8d714d5682ce4f1c5d Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Sat, 22 Feb 2025 01:15:46 -0800 Subject: [PATCH 08/32] feat(backend) update (unfinished) --- backend/macros/src/dev.rs | 6 ++++++ backend/macros/src/lib.rs | 1 + backend/server/src/modules/blog/api/get_post/mod.rs | 6 +++++- backend/server/src/modules/blog/api/mod.rs | 3 +++ 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 backend/macros/src/dev.rs diff --git a/backend/macros/src/dev.rs b/backend/macros/src/dev.rs new file mode 100644 index 0000000..1173d67 --- /dev/null +++ b/backend/macros/src/dev.rs @@ -0,0 +1,6 @@ +#[macro_export] +macro_rules! dev_consume { + ($($var:ident),+) => { + $(let _ = $var;)* + }; +} diff --git a/backend/macros/src/lib.rs b/backend/macros/src/lib.rs index 6003095..e1ac3e9 100644 --- a/backend/macros/src/lib.rs +++ b/backend/macros/src/lib.rs @@ -1,2 +1,3 @@ +mod dev; mod include_str; mod log; diff --git a/backend/server/src/modules/blog/api/get_post/mod.rs b/backend/server/src/modules/blog/api/get_post/mod.rs index a9dd943..5e55731 100644 --- a/backend/server/src/modules/blog/api/get_post/mod.rs +++ b/backend/server/src/modules/blog/api/get_post/mod.rs @@ -1,3 +1,5 @@ +use macros::dev_consume; + use axum::response::IntoResponse; use axum::{ extract::{Path, State}, @@ -13,7 +15,7 @@ pub async fn handler( State(db_connection): State, Path(identifier): Path, ) -> impl IntoResponse { - let post = if let Ok(id) = identifier.parse::() { + if let Ok(id) = identifier.parse::() { select_post_with_i64_id(&db_connection, id).await } else { select_post_with_str_id(&db_connection, &identifier).await @@ -22,9 +24,11 @@ pub async fn handler( } async fn select_post_with_i64_id(db: &PgPool, id: i64) { + dev_consume!(db, id); todo!() } async fn select_post_with_str_id(db: &PgPool, id: &str) { + dev_consume!(db, id); todo!() } diff --git a/backend/server/src/modules/blog/api/mod.rs b/backend/server/src/modules/blog/api/mod.rs index 3092117..5eb1f92 100644 --- a/backend/server/src/modules/blog/api/mod.rs +++ b/backend/server/src/modules/blog/api/mod.rs @@ -1,3 +1,6 @@ pub mod get_page_count; + +// todo: remove exception tag +#[allow(dead_code)] pub mod get_post; // pub mod get_post_list; From 0c6325d5f27e5582a7d5aa1cf2c5a6c44b46a09b Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Sun, 23 Feb 2025 17:39:36 -0800 Subject: [PATCH 09/32] feat(backend) cargo fmt & migration to rust 2024 --- backend/Cargo.toml | 2 +- backend/macros/Cargo.toml | 2 +- backend/macros/src/log.rs | 2 +- backend/markdown/Cargo.toml | 4 ++-- backend/markdown/src/lib.rs | 2 +- backend/server/Cargo.toml | 2 +- backend/server/src/init/logger.rs | 4 ++-- backend/server/src/init/mod.rs | 2 +- backend/server/src/modules/blog/api/get_page_count/mod.rs | 6 +++--- backend/server/src/modules/blog/api/get_post/mod.rs | 2 +- backend/server/src/modules/blog/cron/check_update/mod.rs | 4 ++-- backend/server/src/modules/blog/mod.rs | 4 ++-- backend/server/src/modules/mod.rs | 2 +- backend/server/src/scheduler/mod.rs | 4 ++-- backend/server/src/utils/git.rs | 3 +-- 15 files changed, 22 insertions(+), 23 deletions(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 154acf9..3dfd506 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -resolver = "2" +resolver = "3" members = [ "server", "macros", diff --git a/backend/macros/Cargo.toml b/backend/macros/Cargo.toml index 6291521..2e8a8d5 100644 --- a/backend/macros/Cargo.toml +++ b/backend/macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "macros" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] tracing = "0.1.40" diff --git a/backend/macros/src/log.rs b/backend/macros/src/log.rs index b3aed4e..17bd26c 100644 --- a/backend/macros/src/log.rs +++ b/backend/macros/src/log.rs @@ -44,7 +44,7 @@ mod tests { use std::io; use tracing::Level; use tracing_subscriber::prelude::*; - use tracing_subscriber::{fmt, Registry}; + use tracing_subscriber::{Registry, fmt}; let format = tracing_subscriber::fmt::format() .with_level(true) diff --git a/backend/markdown/Cargo.toml b/backend/markdown/Cargo.toml index e0ca45e..5ca020d 100644 --- a/backend/markdown/Cargo.toml +++ b/backend/markdown/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "markdown" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] } serde = { version = "1.0.209", features = ["derive"] } serde_json = "1.0.128" regex = "1.10.4" -yaml-rust2 = "0.9.0" +yaml-rust2 = "0.10.0" sha2 = "0.10.8" base64 = "0.22.1" thiserror = "2.0.3" diff --git a/backend/markdown/src/lib.rs b/backend/markdown/src/lib.rs index 518d162..545f8c7 100644 --- a/backend/markdown/src/lib.rs +++ b/backend/markdown/src/lib.rs @@ -8,7 +8,7 @@ use std::str::FromStr; use crate::error::Error; use crate::utils::front_matter::{parse_front_matter, split_md_front_matter}; -use base64::{engine::general_purpose, Engine}; +use base64::{Engine, engine::general_purpose}; use serde::Serialize; use serde_json::Value; use sha2::{Digest, Sha256}; diff --git a/backend/server/Cargo.toml b/backend/server/Cargo.toml index 33944ed..4acdf7d 100644 --- a/backend/server/Cargo.toml +++ b/backend/server/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "server" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] macros = { path = "../macros" } diff --git a/backend/server/src/init/logger.rs b/backend/server/src/init/logger.rs index 57ee6be..887c633 100644 --- a/backend/server/src/init/logger.rs +++ b/backend/server/src/init/logger.rs @@ -8,8 +8,8 @@ use dirs::home_dir; use tracing::info; use tracing_appender::{non_blocking, non_blocking::WorkerGuard, rolling}; use tracing_subscriber::{ - filter::LevelFilter, fmt, fmt::format::Writer, fmt::time::FormatTime, fmt::MakeWriter, - layer::SubscriberExt, Layer, Registry, + Layer, Registry, filter::LevelFilter, fmt, fmt::MakeWriter, fmt::format::Writer, + fmt::time::FormatTime, layer::SubscriberExt, }; struct LocalTimer; diff --git a/backend/server/src/init/mod.rs b/backend/server/src/init/mod.rs index 465d530..8710bb0 100644 --- a/backend/server/src/init/mod.rs +++ b/backend/server/src/init/mod.rs @@ -4,7 +4,7 @@ mod misc; use std::{env, panic}; -use crate::modules::{blog::Blog, ModuleTree}; +use crate::modules::{ModuleTree, blog::Blog}; use macros::error_panic; use tracing::info; diff --git a/backend/server/src/modules/blog/api/get_page_count/mod.rs b/backend/server/src/modules/blog/api/get_page_count/mod.rs index c27a427..ce20576 100644 --- a/backend/server/src/modules/blog/api/get_page_count/mod.rs +++ b/backend/server/src/modules/blog/api/get_page_count/mod.rs @@ -1,8 +1,8 @@ use crate::constants::blog::PAGE_SIZE; -use axum::{extract::State, Json}; -use serde_json::{json, Value}; -use sqlx::{query, PgPool, Row}; +use axum::{Json, extract::State}; +use serde_json::{Value, json}; +use sqlx::{PgPool, Row, query}; use tracing::error; const GET_PAGE_COUNT_QUERY: &str = include_str!("post_page_count.sql"); diff --git a/backend/server/src/modules/blog/api/get_post/mod.rs b/backend/server/src/modules/blog/api/get_post/mod.rs index 5e55731..91818db 100644 --- a/backend/server/src/modules/blog/api/get_post/mod.rs +++ b/backend/server/src/modules/blog/api/get_post/mod.rs @@ -2,8 +2,8 @@ use macros::dev_consume; use axum::response::IntoResponse; use axum::{ - extract::{Path, State}, Json, + extract::{Path, State}, }; use serde_json::json; use sqlx::PgPool; diff --git a/backend/server/src/modules/blog/cron/check_update/mod.rs b/backend/server/src/modules/blog/cron/check_update/mod.rs index 4471e04..bc02d9c 100644 --- a/backend/server/src/modules/blog/cron/check_update/mod.rs +++ b/backend/server/src/modules/blog/cron/check_update/mod.rs @@ -15,14 +15,14 @@ use crate::{ use macros::panic_with_log; use git2::{Delta, Repository}; -use sqlx::{Pool, Postgres}; +use sqlx::{PgPool, Pool, Postgres}; use tracing::{error, info, warn, Level}; mod error; mod index; mod utils; -pub fn task(db_connection: &Pool) { +pub fn task(db_connection: &PgPool) { // init let git_work_dir = { let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index 9976b25..bdda532 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -3,6 +3,7 @@ mod cron; use crate::{ modules::{ + Module, blog::{ api::{ get_page_count, @@ -11,12 +12,11 @@ use crate::{ }, // cron::check_update, }, - Module, }, scheduler::tasks::CronTask, }; -use axum::{routing::get, Router}; +use axum::{Router, routing::get}; use sqlx::PgPool; const MOUNT_POINT: &str = "/blog"; diff --git a/backend/server/src/modules/mod.rs b/backend/server/src/modules/mod.rs index dd7e83e..32089d8 100644 --- a/backend/server/src/modules/mod.rs +++ b/backend/server/src/modules/mod.rs @@ -5,7 +5,7 @@ use std::num::ParseIntError; use crate::{ constants::config::server::{default_value, var_name}, - scheduler::{tasks::CronTask, Scheduler}, + scheduler::{Scheduler, tasks::CronTask}, }; use macros::error_panic; diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index edf5b1a..692ab24 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -1,8 +1,8 @@ use std::{ collections::HashMap, sync::{ - atomic::{AtomicBool, Ordering}, - Arc, RwLock, + atomic::{AtomicBool, Ordering}, Arc, + RwLock, }, thread, }; diff --git a/backend/server/src/utils/git.rs b/backend/server/src/utils/git.rs index cf906e6..9c48fc0 100644 --- a/backend/server/src/utils/git.rs +++ b/backend/server/src/utils/git.rs @@ -3,12 +3,11 @@ /// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 /// use git2::{ - build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, + AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, build, }; use std::path::PathBuf; use tracing::{error, warn}; - #[derive(Debug, Eq, PartialEq)] pub struct FileDelta { pub old_path: Option, From 5e8b1ff35d38d159aefbba4a2f0b320358d246bf Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 5 Mar 2025 13:52:07 -0800 Subject: [PATCH 10/32] feat(backend) scheduler rewrite (deadlock for now) --- backend/server/src/modules/blog/mod.rs | 9 +- backend/server/src/modules/mod.rs | 13 +- backend/server/src/scheduler/mod.rs | 171 +++++------------- backend/server/src/scheduler/runner.rs | 152 ++++++++++++++++ backend/server/src/scheduler/task_func.rs | 12 ++ .../server/src/scheduler/tasks/async_basic.rs | 37 ++++ backend/server/src/scheduler/tasks/basic.rs | 7 +- backend/server/src/scheduler/tasks/mod.rs | 5 +- 8 files changed, 267 insertions(+), 139 deletions(-) create mode 100644 backend/server/src/scheduler/runner.rs create mode 100644 backend/server/src/scheduler/tasks/async_basic.rs diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index bdda532..3f17c69 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -3,7 +3,6 @@ mod cron; use crate::{ modules::{ - Module, blog::{ api::{ get_page_count, @@ -12,11 +11,12 @@ use crate::{ }, // cron::check_update, }, + Module, }, - scheduler::tasks::CronTask, + scheduler::tasks::{async_basic::AsyncBasic, CronTask}, }; -use axum::{Router, routing::get}; +use axum::{routing::get, Router}; use sqlx::PgPool; const MOUNT_POINT: &str = "/blog"; @@ -47,6 +47,9 @@ impl Module for Blog { // BasicTask::new(move || check_update::task(&db_handler), "0/15 * * * * *") // .unwrap() // .to_task(), + AsyncBasic::new(|| Box::pin(async { println!("test") }), "* * * * * *") + .unwrap() + .to_task(), ] } } diff --git a/backend/server/src/modules/mod.rs b/backend/server/src/modules/mod.rs index 32089d8..ab1e7dd 100644 --- a/backend/server/src/modules/mod.rs +++ b/backend/server/src/modules/mod.rs @@ -5,11 +5,12 @@ use std::num::ParseIntError; use crate::{ constants::config::server::{default_value, var_name}, - scheduler::{Scheduler, tasks::CronTask}, + scheduler::{tasks::CronTask, Scheduler}, }; use macros::error_panic; use axum::Router; +use tokio::net::TcpListener; use tracing::{info, warn}; // todo: merge `get_server_router` & `get_mount_point`, make Option @@ -51,13 +52,7 @@ impl ModuleTree { Err(_) => default_value::PORT, }; - let listener = match tokio::net::TcpListener::bind(format!( - "{}:{}", - default_value::HOST, - port - )) - .await - { + let listener = match TcpListener::bind(format!("{}:{}", default_value::HOST, port)).await { Ok(listener) => { info!("server started on {}:{}", default_value::HOST, port); listener @@ -77,6 +72,8 @@ impl ModuleTree { app = app.nest(module.get_mount_point(), module.get_server_router()); } + scheduler.start().unwrap(); + axum::serve(listener, app).await.unwrap_or_else(|e| { error_panic!("failed to start axum: {}", e.to_string()); }); diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index 692ab24..67a7b47 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -1,172 +1,94 @@ use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicBool, Ordering}, Arc, - RwLock, - }, - thread, + collections::{HashMap, HashSet}, + sync::{Arc, RwLock}, }; use crate::scheduler::{ - scheduler_error::{ - SchedulerError, SchedulerError::IdAlreadyRunning, SchedulerError::IdNotFound, - }, - sleeper::SkippableSleeper, + runner::Runner, + scheduler_error::{SchedulerError, SchedulerError::IdNotFound}, tasks::CronTask, }; -use tracing::log::error; use uuid::Uuid; +mod runner; pub mod scheduler_error; mod sleeper; mod task_func; pub mod tasks; -struct TaskStatus { - sleeper: Option>, - is_alive: AtomicBool, -} - #[derive(Default)] pub struct Scheduler { - tasks: Arc>>>>, - running: Arc>>>, + running: Arc>>, + parking_pool: Arc>>>>, + runner: Runner, } impl Scheduler { /// Start all tasks in the scheduler pub fn start(&self) -> Result<(), SchedulerError> { - for (k, _v) in self.tasks.read()?.iter() { - self.run_non_blocking(k)? - } - Ok(()) + self.runner.run_all() } /// Terminate all tasks in the scheduler - pub fn halt() {} - - fn run_task( - running: Arc>>>, - id: Uuid, - task: Arc>, - ) -> Result<(), SchedulerError> { - if running.read()?.get(&id).is_some() { - return Err(IdAlreadyRunning); - } else { - running.write()?.insert( - id, - TaskStatus { - sleeper: None, - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - loop { - if let Some(next_execution) = task.get_next_execution() { - // guarantee is alive - let task_status = { - let running_reading_guard = running.read()?; - running_reading_guard.get(&id).cloned().clone() - }; - if let Some(task_status) = task_status { - if !&task_status.is_alive.load(Ordering::SeqCst) { - { - let mut running_lock_guard = running.write()?; - running_lock_guard.remove(&id); - } - return Ok(()); - } else { - let sleeper = Arc::from(SkippableSleeper::from(next_execution)); - { - let mut running_lock_guard = running.write()?; - running_lock_guard.insert( - id, - TaskStatus { - sleeper: Some(sleeper.clone()), - is_alive: AtomicBool::new(true), - } - .into(), - ); - } - sleeper.start(); - task.call(); - } - } - } else { - error!("unexpected error "); - break; - } + pub fn halt(&self) -> Result<(), SchedulerError> { + for id in self.running.read()?.iter() { + let func = &self.runner.terminate_remove(*id)?; + self.running.write()?.remove(id); + self.parking_pool.write()?.insert(*id, func.clone()); } Ok(()) } /// Run a task with `id` pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - Self::run_task(self.running.clone(), id, task.clone()) - } else { - Err(IdNotFound) - } - } - - pub fn run_non_blocking(&self, id: &Uuid) -> Result<(), SchedulerError> { - let id = id.to_owned(); - if let Some(task) = self.tasks.read()?.get(&id) { - let running_clone = self.running.clone(); - let task_clone = task.clone(); - thread::spawn(move || Self::run_task(running_clone, id, task_clone)); + if self.running.read()?.contains(id) { Ok(()) + } else if self.parking_pool.read()?.contains_key(id) { + let func = self.parking_pool.write()?.remove(id).unwrap(); + self.running.write()?.insert(*id); + self.runner.add_with_id(func, id) } else { Err(IdNotFound) } } - fn get_running(&self, id: &Uuid) -> Option> { - let ret = self.running.read().unwrap().get(id).cloned(); - ret - } - /// Stop a task with `id` pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - status.is_alive.store(false, Ordering::SeqCst); - if let Some(s) = &status.sleeper { - s.skip(); - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } + dbg!("stop() called"); + if self.running.read()?.contains(id) { + dbg!("b1"); + let func = self.runner.terminate_remove(*id)?; + dbg!("b1a"); + self.running.write()?.remove(id); + dbg!("b1b"); + self.parking_pool.write()?.insert(*id, func.clone()); + dbg!("b1c"); + Ok(()) + } else if self.parking_pool.read()?.get(id).is_some() { + dbg!("b2"); + Ok(()) + } else { + dbg!("b3"); + Err(IdNotFound) } } - pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { - match self.get_running(id) { - None => Err(IdNotFound), - Some(status) => { - if let Some(s) = &status.sleeper { - s.skip() - } else { - return Err(IdAlreadyRunning); - } - Ok(()) - } - } - } + // pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { + // } pub fn insert(&self, task: Box) -> Result { - let id = Uuid::new_v4(); - self.tasks.write()?.insert(id, Arc::new(task)); + let id = self.runner.add(task.into())?; + self.running.write()?.insert(id); Ok(id) } pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { - tasks.into_iter().map(|t| self.insert(t)).collect() + let ids = self + .runner + .add_list(tasks.into_iter().map(|t| t.into()).collect())?; + self.running.write()?.extend(&ids); + Ok(ids) } pub fn new() -> Self { @@ -187,11 +109,14 @@ mod tests { let scheduler = Scheduler::default(); let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; let id = scheduler.insert(Box::new(task))?; + scheduler.start()?; for i in 1..=2 { println!("Loop {} start", i); println!("Run {}", i); - scheduler.run_non_blocking(&id)?; + scheduler.run(&id)?; + println!("Sleep {}", i); thread::sleep(Duration::from_secs(4)); + println!("Sleep {} finished", i); scheduler.stop(&id)?; println!("Stop {}", i); thread::sleep(Duration::from_secs(2)); diff --git a/backend/server/src/scheduler/runner.rs b/backend/server/src/scheduler/runner.rs new file mode 100644 index 0000000..71c09cd --- /dev/null +++ b/backend/server/src/scheduler/runner.rs @@ -0,0 +1,152 @@ +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, Arc, + RwLock, + }, +}; + +use crate::scheduler::{ + scheduler_error::{SchedulerError, SchedulerError::IdNotFound}, + sleeper::SkippableSleeper, + tasks::CronTask, +}; + +use tokio::runtime::Runtime; +use uuid::Uuid; + +struct TaskStatus { + sleeper: RwLock, + is_running: AtomicBool, + expected_running: AtomicBool, +} + +pub struct TaskEntity { + func: Arc>, + status: TaskStatus, +} + +impl TaskEntity { + pub fn new(func: Arc>) -> Self { + let next_execution: SkippableSleeper = func.get_next_execution().unwrap().into(); + Self { + func, + status: TaskStatus { + sleeper: next_execution.into(), + is_running: false.into(), + expected_running: true.into(), + }, + } + } + + pub async fn run_blocking(&self) { + loop { + if self.status.expected_running.load(Ordering::SeqCst) { + self.status.sleeper.read().unwrap().start(); + self.func.call().await; + if let Some(next_execution) = self.func.get_next_execution() { + *(self.status.sleeper.write().unwrap()) = next_execution.into(); + } else { + self.status.is_running.store(false, Ordering::SeqCst); + break; + } + } else { + break; + } + } + } + + pub fn terminate(&self) { + self.status.expected_running.store(false, Ordering::SeqCst); + self.status.is_running.store(false, Ordering::SeqCst); + self.status.sleeper.read().unwrap().skip(); + } +} + +pub struct Runner { + task_list: RwLock>>, + rt: Runtime, +} + +impl Runner { + pub fn new() -> Self { + Self { + task_list: HashMap::new().into(), + rt: Runtime::new().unwrap(), + } + } + + pub fn run_all(&self) -> Result<(), SchedulerError> { + let list = &self.task_list.read()?; + self.rt.block_on(async { + for (_, entity) in list.iter() { + if !entity.status.is_running.load(Ordering::SeqCst) + && entity.status.expected_running.load(Ordering::SeqCst) + { + entity.status.is_running.store(true, Ordering::SeqCst); + tokio::spawn({ + let entity_clone = entity.clone(); + async move { entity_clone.run_blocking().await } + }); + } + } + }); + Ok(()) + } + + pub fn add_with_id( + &self, + task: Arc>, + id: &Uuid, + ) -> Result<(), SchedulerError> { + self.task_list + .write()? + .insert(*id, TaskEntity::new(task).into()); + Ok(()) + } + + pub fn add(&self, task: Arc>) -> Result { + let id = Uuid::new_v4(); + self.add_with_id(task, &id)?; + Ok(id) + } + + pub fn add_list_with_id( + &self, + tasks: Vec<(&Uuid, Arc>)>, + ) -> Result<(), SchedulerError> { + tasks + .into_iter() + .try_for_each(|(id, task)| self.add_with_id(task, id)) + } + + pub fn add_list( + &self, + tasks: Vec>>, + ) -> Result, SchedulerError> { + tasks + .into_iter() + .map(|task| { + let id = Uuid::new_v4(); + self.add_with_id(task, &id).map(move |_| id) + }) + .collect::, _>>() + } + + pub fn terminate_remove(&self, id: Uuid) -> Result>, SchedulerError> { + match self.task_list.read()?.get(&id) { + None => Err(IdNotFound), + Some(task) => { + task.terminate(); + let func = &self.task_list.write()?.remove(&id).unwrap().func; + Ok(func.clone()) + } + } + } +} + +impl Default for Runner { + fn default() -> Self { + Self::new() + } +} diff --git a/backend/server/src/scheduler/task_func.rs b/backend/server/src/scheduler/task_func.rs index af27ded..ceafe54 100644 --- a/backend/server/src/scheduler/task_func.rs +++ b/backend/server/src/scheduler/task_func.rs @@ -1,3 +1,15 @@ +use std::pin::Pin; + pub trait TaskFunc: Fn() + Send + Sync + 'static {} impl TaskFunc for T where T: Fn() + Send + Sync + Clone + 'static {} + +pub trait AsyncTaskFunc: + Fn() -> Pin + Send>> + Send + Sync + 'static +{ +} + +impl AsyncTaskFunc for T where + T: Fn() -> Pin + Send>> + Send + Sync + Clone + 'static +{ +} diff --git a/backend/server/src/scheduler/tasks/async_basic.rs b/backend/server/src/scheduler/tasks/async_basic.rs new file mode 100644 index 0000000..3de6f37 --- /dev/null +++ b/backend/server/src/scheduler/tasks/async_basic.rs @@ -0,0 +1,37 @@ +use std::{error::Error, pin::Pin, str::FromStr}; + +use crate::{scheduler::task_func::AsyncTaskFunc, scheduler::tasks::CronTask}; + +use chrono::{DateTime, Utc}; +use cron::Schedule; + +pub struct AsyncBasic { + schedule: Schedule, + func: Box, +} + +impl AsyncBasic { + pub fn new(func: F, cron_expr: &str) -> Result> + where + F: AsyncTaskFunc, + { + Ok(AsyncBasic { + schedule: Schedule::from_str(cron_expr)?, + func: Box::new(func), + }) + } + + pub fn to_task(self) -> Box { + Box::new(self) + } +} + +impl CronTask for AsyncBasic { + fn get_next_execution(&self) -> Option> { + self.schedule.upcoming(Utc).next() + } + + fn call(&self) -> Pin + Send)>> { + Box::pin((self.func)()) + } +} diff --git a/backend/server/src/scheduler/tasks/basic.rs b/backend/server/src/scheduler/tasks/basic.rs index 4c69db7..cc02ed7 100644 --- a/backend/server/src/scheduler/tasks/basic.rs +++ b/backend/server/src/scheduler/tasks/basic.rs @@ -1,5 +1,4 @@ -use std::error::Error; -use std::str::FromStr; +use std::{error::Error, pin::Pin, str::FromStr}; use crate::scheduler::{task_func::TaskFunc, tasks::CronTask}; @@ -32,7 +31,7 @@ impl CronTask for BasicTask { self.schedule.upcoming(Utc).next() } - fn call(&self) { - (self.func)() + fn call(&self) -> Pin + Send + '_>> { + Box::pin(async { (self.func)() }) } } diff --git a/backend/server/src/scheduler/tasks/mod.rs b/backend/server/src/scheduler/tasks/mod.rs index 8058bf6..0c58c99 100644 --- a/backend/server/src/scheduler/tasks/mod.rs +++ b/backend/server/src/scheduler/tasks/mod.rs @@ -1,8 +1,11 @@ +use std::pin::Pin; + use chrono::{DateTime, Utc}; +pub mod async_basic; pub mod basic; pub trait CronTask: Send + Sync { fn get_next_execution(&self) -> Option>; - fn call(&self); + fn call(&self) -> Pin + Send + '_>>; } From 2e32297d5e7d376f9d01992605ffcb38b200d0cf Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 5 Mar 2025 23:38:40 -0800 Subject: [PATCH 11/32] feat(backend) fix deadlock issue, code cleanup --- backend/server/src/scheduler/mod.rs | 56 ++++++++++++++++---------- backend/server/src/scheduler/runner.rs | 39 ++++++++++++------ 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index 67a7b47..456dc76 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -19,8 +19,8 @@ pub mod tasks; #[derive(Default)] pub struct Scheduler { - running: Arc>>, - parking_pool: Arc>>>>, + running: RwLock>, + parking_pool: RwLock>>>, runner: Runner, } @@ -47,7 +47,9 @@ impl Scheduler { } else if self.parking_pool.read()?.contains_key(id) { let func = self.parking_pool.write()?.remove(id).unwrap(); self.running.write()?.insert(*id); - self.runner.add_with_id(func, id) + self.runner + .add_with_id(func, id) + .and_then(|_| self.runner.run_id(id)) } else { Err(IdNotFound) } @@ -55,21 +57,14 @@ impl Scheduler { /// Stop a task with `id` pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { - dbg!("stop() called"); if self.running.read()?.contains(id) { - dbg!("b1"); let func = self.runner.terminate_remove(*id)?; - dbg!("b1a"); self.running.write()?.remove(id); - dbg!("b1b"); self.parking_pool.write()?.insert(*id, func.clone()); - dbg!("b1c"); Ok(()) } else if self.parking_pool.read()?.get(id).is_some() { - dbg!("b2"); Ok(()) } else { - dbg!("b3"); Err(IdNotFound) } } @@ -98,18 +93,16 @@ impl Scheduler { #[cfg(test)] mod tests { - use crate::scheduler::tasks::basic::BasicTask; - use crate::scheduler::Scheduler; - use std::error::Error; - use std::thread; - use std::time::Duration; + use std::{error::Error, thread, time::Duration}; - #[test] - fn test_run() -> Result<(), Box> { - let scheduler = Scheduler::default(); - let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; - let id = scheduler.insert(Box::new(task))?; - scheduler.start()?; + use crate::scheduler::{ + tasks::{async_basic::AsyncBasic, basic::BasicTask}, + Scheduler, + }; + + use uuid::Uuid; + + fn execution(scheduler: Scheduler, id: Uuid) -> Result<(), Box> { for i in 1..=2 { println!("Loop {} start", i); println!("Run {}", i); @@ -124,4 +117,25 @@ mod tests { } Ok(()) } + + #[test] + fn test_run() -> Result<(), Box> { + let scheduler = Scheduler::default(); + let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; + let id = scheduler.insert(Box::new(task))?; + scheduler.start()?; + execution(scheduler, id) + } + + #[test] + fn test_run_async() -> Result<(), Box> { + let scheduler = Scheduler::default(); + let task = AsyncBasic::new( + || Box::pin(async { println!("test async print") }), + "* * * * * *", + )?; + let id = scheduler.insert(Box::new(task))?; + scheduler.start()?; + execution(scheduler, id) + } } diff --git a/backend/server/src/scheduler/runner.rs b/backend/server/src/scheduler/runner.rs index 71c09cd..15a3181 100644 --- a/backend/server/src/scheduler/runner.rs +++ b/backend/server/src/scheduler/runner.rs @@ -41,9 +41,10 @@ impl TaskEntity { pub async fn run_blocking(&self) { loop { + self.status.sleeper.read().unwrap().start(); if self.status.expected_running.load(Ordering::SeqCst) { - self.status.sleeper.read().unwrap().start(); self.func.call().await; + if let Some(next_execution) = self.func.get_next_execution() { *(self.status.sleeper.write().unwrap()) = next_execution.into(); } else { @@ -80,20 +81,33 @@ impl Runner { let list = &self.task_list.read()?; self.rt.block_on(async { for (_, entity) in list.iter() { - if !entity.status.is_running.load(Ordering::SeqCst) - && entity.status.expected_running.load(Ordering::SeqCst) - { - entity.status.is_running.store(true, Ordering::SeqCst); - tokio::spawn({ - let entity_clone = entity.clone(); - async move { entity_clone.run_blocking().await } - }); - } + self.run_entity(entity.clone()).await } }); Ok(()) } + pub fn run_id(&self, id: &Uuid) -> Result<(), SchedulerError> { + if let Some(entity) = self.task_list.read()?.get(id).cloned() { + self.rt.block_on(async { self.run_entity(entity).await }); + Ok(()) + } else { + Err(IdNotFound) + } + } + + pub async fn run_entity(&self, entity: Arc) { + if !entity.status.is_running.load(Ordering::SeqCst) + && entity.status.expected_running.load(Ordering::SeqCst) + { + entity.status.is_running.store(true, Ordering::SeqCst); + tokio::spawn({ + let entity_clone = entity.clone(); + async move { entity_clone.run_blocking().await } + }); + } + } + pub fn add_with_id( &self, task: Arc>, @@ -134,11 +148,12 @@ impl Runner { } pub fn terminate_remove(&self, id: Uuid) -> Result>, SchedulerError> { - match self.task_list.read()?.get(&id) { + let mut list = self.task_list.write()?; + match list.get(&id) { None => Err(IdNotFound), Some(task) => { task.terminate(); - let func = &self.task_list.write()?.remove(&id).unwrap().func; + let func = list.remove(&id).unwrap().func.clone(); Ok(func.clone()) } } From dc781bc2ef3523625b028fa0770e740869655420 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Thu, 6 Mar 2025 00:48:35 -0800 Subject: [PATCH 12/32] feat(backend) module rename & import improve --- backend/server/src/modules/blog/api/get_post/mod.rs | 4 ++-- backend/server/src/scheduler/{scheduler_error.rs => error.rs} | 2 +- backend/server/src/scheduler/mod.rs | 4 ++-- backend/server/src/scheduler/runner.rs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename backend/server/src/scheduler/{scheduler_error.rs => error.rs} (94%) diff --git a/backend/server/src/modules/blog/api/get_post/mod.rs b/backend/server/src/modules/blog/api/get_post/mod.rs index 91818db..3bdf2f8 100644 --- a/backend/server/src/modules/blog/api/get_post/mod.rs +++ b/backend/server/src/modules/blog/api/get_post/mod.rs @@ -1,9 +1,9 @@ use macros::dev_consume; -use axum::response::IntoResponse; use axum::{ - Json, extract::{Path, State}, + response::IntoResponse, + Json, }; use serde_json::json; use sqlx::PgPool; diff --git a/backend/server/src/scheduler/scheduler_error.rs b/backend/server/src/scheduler/error.rs similarity index 94% rename from backend/server/src/scheduler/scheduler_error.rs rename to backend/server/src/scheduler/error.rs index 8961bf6..04b3df5 100644 --- a/backend/server/src/scheduler/scheduler_error.rs +++ b/backend/server/src/scheduler/error.rs @@ -4,7 +4,7 @@ use std::{ sync::PoisonError, }; -use crate::scheduler::scheduler_error::SchedulerError::MutexPoison; +use crate::scheduler::error::SchedulerError::MutexPoison; // todo: refactor with thiserror pub enum SchedulerError { diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index 456dc76..67f09a1 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -4,15 +4,15 @@ use std::{ }; use crate::scheduler::{ + error::{SchedulerError, SchedulerError::IdNotFound}, runner::Runner, - scheduler_error::{SchedulerError, SchedulerError::IdNotFound}, tasks::CronTask, }; use uuid::Uuid; +pub mod error; mod runner; -pub mod scheduler_error; mod sleeper; mod task_func; pub mod tasks; diff --git a/backend/server/src/scheduler/runner.rs b/backend/server/src/scheduler/runner.rs index 15a3181..6aca062 100644 --- a/backend/server/src/scheduler/runner.rs +++ b/backend/server/src/scheduler/runner.rs @@ -7,7 +7,7 @@ use std::{ }; use crate::scheduler::{ - scheduler_error::{SchedulerError, SchedulerError::IdNotFound}, + error::{SchedulerError, SchedulerError::IdNotFound}, sleeper::SkippableSleeper, tasks::CronTask, }; From 2045d380c12ebd2d47d96e044df5cd8a090510d9 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Thu, 6 Mar 2025 16:08:00 -0800 Subject: [PATCH 13/32] feat(makefile) cleanup --- Makefile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 0c4e0d9..581ed84 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,7 @@ ifeq ($(ARCH), arm64) ARCH := aarch64 endif -all: release - -release: - docker compose -f prod.compose.yaml build --build-arg ARCH=$(ARCH) - -dev: - docker-compose -f dev.compose.yaml up --build - +.PHONY: up clean: - docker-compose -f prod.compose.yaml -f dev.compose.yaml rm -fsv + echo "todo" From 7cc44522faf6ce7666a0be323dd3d8ff268a472b Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Fri, 7 Mar 2025 23:52:44 -0800 Subject: [PATCH 14/32] feat(backend::scheduler) fix runtime issue --- backend/server/src/scheduler/runner.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/server/src/scheduler/runner.rs b/backend/server/src/scheduler/runner.rs index 6aca062..b3ca00b 100644 --- a/backend/server/src/scheduler/runner.rs +++ b/backend/server/src/scheduler/runner.rs @@ -44,7 +44,6 @@ impl TaskEntity { self.status.sleeper.read().unwrap().start(); if self.status.expected_running.load(Ordering::SeqCst) { self.func.call().await; - if let Some(next_execution) = self.func.get_next_execution() { *(self.status.sleeper.write().unwrap()) = next_execution.into(); } else { @@ -104,7 +103,9 @@ impl Runner { tokio::spawn({ let entity_clone = entity.clone(); async move { entity_clone.run_blocking().await } - }); + }) + .await + .expect("tokio runtime error"); } } From 7b84a63a14877d15bf98b930c440ef301dcf7ff9 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Wed, 12 Mar 2025 18:34:49 -0700 Subject: [PATCH 15/32] feat(backend::scheduler) deadlock spotted --- .../check_update/error.rs | 0 .../check_update/index.rs | 20 +-- .../blog/components/check_update/mod.rs | 78 +++++++++ .../blog/components/check_update/utils.rs | 81 +++++++++ .../server/src/modules/blog/components/mod.rs | 2 + .../src/modules/blog/components/rag/mod.rs | 0 .../src/modules/blog/cron/check_update/mod.rs | 148 ---------------- .../modules/blog/cron/check_update/utils.rs | 88 ---------- backend/server/src/modules/blog/cron/mod.rs | 1 - backend/server/src/modules/blog/mod.rs | 17 +- backend/server/src/modules/mod.rs | 12 +- backend/server/src/scheduler/mod.rs | 158 +++++++++--------- backend/server/src/scheduler/runner.rs | 88 ++++++---- backend/server/src/scheduler/sleeper.rs | 101 +++++------ backend/server/src/utils/git.rs | 9 +- 15 files changed, 375 insertions(+), 428 deletions(-) rename backend/server/src/modules/blog/{cron => components}/check_update/error.rs (100%) rename backend/server/src/modules/blog/{cron => components}/check_update/index.rs (97%) create mode 100644 backend/server/src/modules/blog/components/check_update/mod.rs create mode 100644 backend/server/src/modules/blog/components/check_update/utils.rs create mode 100644 backend/server/src/modules/blog/components/mod.rs create mode 100644 backend/server/src/modules/blog/components/rag/mod.rs delete mode 100644 backend/server/src/modules/blog/cron/check_update/mod.rs delete mode 100644 backend/server/src/modules/blog/cron/check_update/utils.rs delete mode 100644 backend/server/src/modules/blog/cron/mod.rs diff --git a/backend/server/src/modules/blog/cron/check_update/error.rs b/backend/server/src/modules/blog/components/check_update/error.rs similarity index 100% rename from backend/server/src/modules/blog/cron/check_update/error.rs rename to backend/server/src/modules/blog/components/check_update/error.rs diff --git a/backend/server/src/modules/blog/cron/check_update/index.rs b/backend/server/src/modules/blog/components/check_update/index.rs similarity index 97% rename from backend/server/src/modules/blog/cron/check_update/index.rs rename to backend/server/src/modules/blog/components/check_update/index.rs index fabd478..9cf590c 100644 --- a/backend/server/src/modules/blog/cron/check_update/index.rs +++ b/backend/server/src/modules/blog/components/check_update/index.rs @@ -1,12 +1,10 @@ use std::error::Error; use std::path::Path; -use crate::{ - modules::blog::cron::check_update::utils::get_md_file_basename, - modules::blog::cron::check_update::{error::PostIdxError, utils::parse_date}, -}; use markdown::MdFile; +use crate::modules::blog::components::check_update::utils::get_md_file_basename; +use crate::modules::blog::components::check_update::{error::PostIdxError, utils::parse_date}; use sqlx::{query, PgPool, Pool, Postgres, Row}; use tokio::runtime::Runtime; use tracing::{info, warn}; @@ -18,7 +16,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> return Err(PostIdxError::MissingField { field: "title".to_string(), filename: post.filename.clone(), - }) + }); } Some(title) => title.as_str().unwrap_or_else(|| { warn!("invalid title for `{}`", &post.filename); @@ -41,7 +39,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> return Err(PostIdxError::MissingField { field: "category".to_string(), filename: post.filename.clone(), - }) + }); } Some(title) => title.as_str().unwrap_or_else(|| { warn!("invalid category for `{}`", &post.filename); @@ -58,7 +56,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> err_field: last_update.to_string(), filename: post.filename.clone(), msg: e.to_string(), - }) + }); } } } else { @@ -70,7 +68,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> return Err(PostIdxError::MissingField { field: "date".to_string(), filename: post.filename.clone(), - }) + }); } Some(first_update) => match parse_date(first_update.as_str().unwrap()) { Ok(date) => date, @@ -80,7 +78,7 @@ async fn base(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> err_field: first_update.to_string(), filename: post.filename.clone(), msg: e.to_string(), - }) + }); } }, }; @@ -137,7 +135,7 @@ async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> return Err(PostIdxError::MissingField { field: "tags".to_string(), filename: post.filename.clone(), - }) + }); } Some(val) => val, }; @@ -173,7 +171,7 @@ async fn tag(db_connection: &PgPool, post: &MdFile) -> Result<(), PostIdxError> db_table: "Tag".to_string(), id: post_id.to_string(), msg: e.to_string(), - }) + }); } } } diff --git a/backend/server/src/modules/blog/components/check_update/mod.rs b/backend/server/src/modules/blog/components/check_update/mod.rs new file mode 100644 index 0000000..6e9f5dd --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/mod.rs @@ -0,0 +1,78 @@ +use crate::constants::config::general::{default_value, var_name}; +use macros::panic_with_log; +use std::sync::Arc; +use std::{ + env, + path::{Path, PathBuf}, +}; + +use crate::modules::blog::components::check_update::{ + // index::drop_index, + // utils::{expand_path, index_files, search_md}, + utils::expand_path, +}; +use crate::scheduler::task_func::AsyncTaskFunc; +use sqlx::PgPool; +use tracing::{info, warn, Level}; + +// mod error; +// mod index; +mod utils; + +#[derive(Debug)] +struct Args { + git_work_dir: PathBuf, + git_url: String, + git_remote_name: String, + git_remote_branch: String, +} + +pub fn task(db_connection: PgPool) -> impl AsyncTaskFunc { + // init + let git_work_dir = { + let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { + expand_path(default_log_path) + } else { + let fallback_path = expand_path(default_value::WORK_DIR.to_string()); + warn!( + "unset env var `{}`, using default path: {}/blog_git", + var_name::WORK_DIR, + fallback_path + ); + fallback_path + }; + Path::new(&path).join("blog_git") + }; + + // clone & check update + + let git_url = match env::var(var_name::BLOG_GIT_URL) { + Ok(url) => url, + Err(e) => { + panic_with_log!( + Level::ERROR, + "failed to read `{}`: {}", + var_name::BLOG_GIT_URL, + e + ) + } + }; + let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) + .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); + let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) + .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); + + let arc_args = Arc::new(Args { + git_work_dir, + git_url, + git_remote_name, + git_remote_branch, + }); + + move || Box::pin(test(arc_args.clone())) +} + +async fn test(args: Arc) { + info!("test"); + dbg!(&args); +} diff --git a/backend/server/src/modules/blog/components/check_update/utils.rs b/backend/server/src/modules/blog/components/check_update/utils.rs new file mode 100644 index 0000000..9f00fe5 --- /dev/null +++ b/backend/server/src/modules/blog/components/check_update/utils.rs @@ -0,0 +1,81 @@ +use crate::constants::{ + config::general::{default_value, var_name}, + time::DEFAULT_DATE_FORMAT, +}; + +// use crate::modules::blog::components::check_update::index::index_file; +use chrono::{DateTime, FixedOffset, ParseResult}; +use dirs::home_dir; +use tracing::warn; + +// pub fn search_md(entry: PathBuf) -> Vec { +// if entry.ends_with(".git") { +// return vec![]; +// } +// if entry.is_dir() { +// if let Ok(files) = entry.read_dir() { +// files +// .flat_map(|file| { +// let file_path = file.unwrap().path(); +// search_md(file_path) +// }) +// .collect() +// } else { +// vec![] +// } +// } else if entry.extension().is_some_and(|ext| ext == "md") { +// vec![entry] +// } else { +// vec![] +// } +// } + +// pub fn index_files(files: &[PathBuf], db_conn: &Pool) { +// let rt = Runtime::new().unwrap(); +// rt.block_on(async { +// let files_clone = files.to_owned(); +// let handlers = files_clone +// .into_iter() +// .map(|file: PathBuf| { +// let conn_clone = db_conn.clone(); +// spawn(async move { +// if let Err(e) = index_file(&file, &conn_clone).await { +// error!("failed to index markdown file `{}`: {}", file.display(), e); +// } +// }) +// }) +// .collect::>(); +// +// block_on(join_all(handlers)); +// }); +// } + +pub fn expand_path(path: String) -> String { + if path.starts_with('~') { + home_dir() + .map(|dir| { + dir.join(path.strip_prefix("~/").unwrap()) + .into_os_string() + .into_string() + .unwrap() + }) + .unwrap_or_else(|| { + warn!( + "failed to parse env var `{}`, using default value `{}`", + var_name::WORK_DIR, + default_value::WORK_DIR + ); + default_value::WORK_DIR.to_string() + }) + } else { + path + } +} + +pub fn parse_date(date_str: &str) -> ParseResult> { + DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) +} + +// pub fn get_md_file_basename(f_name: &str) -> String { +// f_name.replace(".md", "") +// } diff --git a/backend/server/src/modules/blog/components/mod.rs b/backend/server/src/modules/blog/components/mod.rs new file mode 100644 index 0000000..2683687 --- /dev/null +++ b/backend/server/src/modules/blog/components/mod.rs @@ -0,0 +1,2 @@ +pub mod check_update; +mod rag; diff --git a/backend/server/src/modules/blog/components/rag/mod.rs b/backend/server/src/modules/blog/components/rag/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/backend/server/src/modules/blog/cron/check_update/mod.rs b/backend/server/src/modules/blog/cron/check_update/mod.rs deleted file mode 100644 index bc02d9c..0000000 --- a/backend/server/src/modules/blog/cron/check_update/mod.rs +++ /dev/null @@ -1,148 +0,0 @@ -use std::{ - env, - fs::remove_dir_all, - path::{Path, PathBuf}, -}; - -use crate::{ - constants::config::general::{default_value, var_name}, - modules::blog::cron::check_update::{ - index::drop_index, - utils::{expand_path, get_md_file_basename, index_files, search_md}, - }, - utils::git, -}; -use macros::panic_with_log; - -use git2::{Delta, Repository}; -use sqlx::{PgPool, Pool, Postgres}; -use tracing::{error, info, warn, Level}; - -mod error; -mod index; -mod utils; - -pub fn task(db_connection: &PgPool) { - // init - let git_work_dir = { - let path = if let Ok(default_log_path) = env::var(var_name::WORK_DIR) { - expand_path(default_log_path) - } else { - let fallback_path = expand_path(default_value::WORK_DIR.to_string()); - warn!( - "unset env var `{}`, using default path: {}/blog_git", - var_name::WORK_DIR, - fallback_path - ); - fallback_path - }; - Path::new(&path).join("blog_git") - }; - - // clone & check update - - let git_url = match env::var(var_name::BLOG_GIT_URL) { - Ok(url) => url, - Err(e) => { - panic_with_log!( - Level::ERROR, - "failed to read `{}`: {}", - var_name::BLOG_GIT_URL, - e - ) - } - }; - let git_remote_name = env::var(var_name::BLOG_GIT_REMOTE_NAME) - .unwrap_or(default_value::BLOG_GIT_REMOTE_NAME.to_string()); - let git_remote_branch = env::var(var_name::BLOG_GIT_REMOTE_BRANCH) - .unwrap_or(default_value::BLOG_GIT_REMOTE_BRANCH.to_string()); - - match Repository::open(&git_work_dir) { - Ok(repo) => { - // prevent git remote url mismatch (clean cache) - match repo.find_remote(&git_remote_name) { - Ok(remote) => { - if remote.url().is_some_and(|url| url != git_url) { - remove_dir_all(&git_work_dir).unwrap(); - warn!("blog content git({}) url mismatch, reinitialized", &git_url); - return; - } - } - Err(e) => { - remove_dir_all(&git_work_dir).unwrap(); - error!( - "unexpected error on indexing blog content git({}), reinitialized: {}", - &git_url, e - ); - } - } - - match git::sync(&git_remote_name, &git_remote_branch, &repo) { - Ok(updates) => { - if !updates.is_empty() { - // incremental update - info!( - "blog content git({}) update detected: {:?}", - &git_url, updates - ); - - let mut update_md: Vec = vec![]; - - for update in updates { - if [Delta::Deleted, Delta::Untracked].contains(&update.status) { - if let Some(path) = &update.old_path { - if drop_index( - &get_md_file_basename( - path.file_name().unwrap().to_str().unwrap(), - ), - db_connection, - ) - .is_ok() - { - info!( - "blog content git({}) file `{}` is detected as deleted/untracked, index dropped", - &git_url, - &path.display() - ); - } else { - error!( - "blog content git({}) file `{}` is detected as deleted/untracked, failed to drop index", - &git_url, - &path.display() - ) - } - } - } else if let Some(path) = &update.new_path { - if path.components().any(|comp| comp.as_os_str() == "posts") - && path.extension().is_some_and(|ext| ext == "md") - { - update_md.push(path.to_path_buf()); - } - } - } - - index_files(&update_md, db_connection); - } - } - Err(e) => { - error!( - "failed to sync blog content git({}) updates: {}", - &git_url, e - ); - } - } - } - Err(_) => { - match Repository::clone(&git_url, &git_work_dir) { - Ok(_repo) => { - // full repo index - let files = search_md(git_work_dir.join("posts")); - index_files(&files, db_connection); - } - Err(e) => { - error!("failed to clone blog content git({}): {}", &git_url, e); - } - } - } - }; -} diff --git a/backend/server/src/modules/blog/cron/check_update/utils.rs b/backend/server/src/modules/blog/cron/check_update/utils.rs deleted file mode 100644 index 85086ca..0000000 --- a/backend/server/src/modules/blog/cron/check_update/utils.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::path::PathBuf; - -use crate::{ - constants::{ - config::general::{default_value, var_name}, - time::DEFAULT_DATE_FORMAT, - }, - modules::blog::cron::check_update::index::index_file, -}; - -use chrono::{DateTime, FixedOffset, ParseResult}; -use dirs::home_dir; -use futures::{executor::block_on, future::join_all}; -use sqlx::{Pool, Postgres}; -use tokio::{runtime::Runtime, spawn}; -use tracing::{error, warn}; - -pub fn search_md(entry: PathBuf) -> Vec { - if entry.ends_with(".git") { - return vec![]; - } - if entry.is_dir() { - if let Ok(files) = entry.read_dir() { - files - .flat_map(|file| { - let file_path = file.unwrap().path(); - search_md(file_path) - }) - .collect() - } else { - vec![] - } - } else if entry.extension().is_some_and(|ext| ext == "md") { - vec![entry] - } else { - vec![] - } -} - -pub fn index_files(files: &[PathBuf], db_conn: &Pool) { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let files_clone = files.to_owned(); - let handlers = files_clone - .into_iter() - .map(|file: PathBuf| { - let conn_clone = db_conn.clone(); - spawn(async move { - if let Err(e) = index_file(&file, &conn_clone).await { - error!("failed to index markdown file `{}`: {}", file.display(), e); - } - }) - }) - .collect::>(); - - block_on(join_all(handlers)); - }); -} - -pub fn expand_path(path: String) -> String { - if path.starts_with('~') { - home_dir() - .map(|dir| { - dir.join(path.strip_prefix("~/").unwrap()) - .into_os_string() - .into_string() - .unwrap() - }) - .unwrap_or_else(|| { - warn!( - "failed to parse env var `{}`, using default value `{}`", - var_name::WORK_DIR, - default_value::WORK_DIR - ); - default_value::WORK_DIR.to_string() - }) - } else { - path - } -} - -pub fn parse_date(date_str: &str) -> ParseResult> { - DateTime::parse_from_str(format!("{}T120000", date_str).as_str(), DEFAULT_DATE_FORMAT) -} - -pub fn get_md_file_basename(f_name: &str) -> String { - f_name.replace(".md", "") -} diff --git a/backend/server/src/modules/blog/cron/mod.rs b/backend/server/src/modules/blog/cron/mod.rs deleted file mode 100644 index f5d8ad7..0000000 --- a/backend/server/src/modules/blog/cron/mod.rs +++ /dev/null @@ -1 +0,0 @@ -// pub mod check_update; diff --git a/backend/server/src/modules/blog/mod.rs b/backend/server/src/modules/blog/mod.rs index 3f17c69..213e816 100644 --- a/backend/server/src/modules/blog/mod.rs +++ b/backend/server/src/modules/blog/mod.rs @@ -1,5 +1,5 @@ mod api; -mod cron; +mod components; use crate::{ modules::{ @@ -13,9 +13,12 @@ use crate::{ }, Module, }, - scheduler::tasks::{async_basic::AsyncBasic, CronTask}, + scheduler::tasks::CronTask, }; +use crate::modules::blog::components::check_update; +use crate::scheduler::tasks::async_basic::AsyncBasic; +use crate::scheduler::tasks::basic::BasicTask; use axum::{routing::get, Router}; use sqlx::PgPool; @@ -42,12 +45,12 @@ impl Module for Blog { } fn get_cron_tasks(&self) -> Vec> { - // let db_handler = self.db_handler.clone(); + let db_handler = self.db_handler.clone(); vec![ - // BasicTask::new(move || check_update::task(&db_handler), "0/15 * * * * *") - // .unwrap() - // .to_task(), - AsyncBasic::new(|| Box::pin(async { println!("test") }), "* * * * * *") + AsyncBasic::new(check_update::task(db_handler.clone()), "* * * * * *") + .unwrap() + .to_task(), + BasicTask::new(|| println!("test lol"), "* * * * * *") .unwrap() .to_task(), ] diff --git a/backend/server/src/modules/mod.rs b/backend/server/src/modules/mod.rs index ab1e7dd..3ff1c0a 100644 --- a/backend/server/src/modules/mod.rs +++ b/backend/server/src/modules/mod.rs @@ -1,7 +1,6 @@ pub mod blog; -use std::env; -use std::num::ParseIntError; +use std::{env, num::ParseIntError, thread}; use crate::{ constants::config::server::{default_value, var_name}, @@ -10,6 +9,7 @@ use crate::{ use macros::error_panic; use axum::Router; +use futures::executor::block_on; use tokio::net::TcpListener; use tracing::{info, warn}; @@ -68,11 +68,15 @@ impl ModuleTree { }; for module in &self.modules { - let _ = scheduler.insert_list(module.get_cron_tasks()); + let _ = scheduler.insert_list(module.get_cron_tasks()).await; app = app.nest(module.get_mount_point(), module.get_server_router()); } - scheduler.start().unwrap(); + let _ = thread::spawn(move || { + let fut = scheduler.start(); + block_on(fut).unwrap() + }) + .join(); axum::serve(listener, app).await.unwrap_or_else(|e| { error_panic!("failed to start axum: {}", e.to_string()); diff --git a/backend/server/src/scheduler/mod.rs b/backend/server/src/scheduler/mod.rs index 67f09a1..63913c3 100644 --- a/backend/server/src/scheduler/mod.rs +++ b/backend/server/src/scheduler/mod.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - sync::{Arc, RwLock}, + sync::Arc, }; use crate::scheduler::{ @@ -9,12 +9,13 @@ use crate::scheduler::{ tasks::CronTask, }; +use tokio::sync::RwLock; use uuid::Uuid; pub mod error; mod runner; mod sleeper; -mod task_func; +pub mod task_func; pub mod tasks; #[derive(Default)] @@ -26,43 +27,42 @@ pub struct Scheduler { impl Scheduler { /// Start all tasks in the scheduler - pub fn start(&self) -> Result<(), SchedulerError> { - self.runner.run_all() + pub async fn start(&self) -> Result<(), SchedulerError> { + self.runner.run_all().await } /// Terminate all tasks in the scheduler - pub fn halt(&self) -> Result<(), SchedulerError> { - for id in self.running.read()?.iter() { - let func = &self.runner.terminate_remove(*id)?; - self.running.write()?.remove(id); - self.parking_pool.write()?.insert(*id, func.clone()); + pub async fn halt(&self) -> Result<(), SchedulerError> { + for id in self.running.read().await.iter() { + let func = &self.runner.terminate_remove(*id).await?; + self.running.write().await.remove(id); + self.parking_pool.write().await.insert(*id, func.clone()); } Ok(()) } /// Run a task with `id` - pub fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { - if self.running.read()?.contains(id) { + pub async fn run(&self, id: &Uuid) -> Result<(), SchedulerError> { + if self.running.read().await.contains(id) { Ok(()) - } else if self.parking_pool.read()?.contains_key(id) { - let func = self.parking_pool.write()?.remove(id).unwrap(); - self.running.write()?.insert(*id); - self.runner - .add_with_id(func, id) - .and_then(|_| self.runner.run_id(id)) + } else if self.parking_pool.read().await.contains_key(id) { + let func = self.parking_pool.write().await.remove(id).unwrap(); + self.running.write().await.insert(*id); + self.runner.add_with_id(func, id).await?; + self.runner.run_id(id).await } else { Err(IdNotFound) } } /// Stop a task with `id` - pub fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { - if self.running.read()?.contains(id) { - let func = self.runner.terminate_remove(*id)?; - self.running.write()?.remove(id); - self.parking_pool.write()?.insert(*id, func.clone()); + pub async fn stop(&self, id: &Uuid) -> Result<(), SchedulerError> { + if self.running.read().await.contains(id) { + let func = self.runner.terminate_remove(*id).await?; + self.running.write().await.remove(id); + self.parking_pool.write().await.insert(*id, func.clone()); Ok(()) - } else if self.parking_pool.read()?.get(id).is_some() { + } else if self.parking_pool.read().await.get(id).is_some() { Ok(()) } else { Err(IdNotFound) @@ -72,17 +72,21 @@ impl Scheduler { // pub fn now(&self, id: &Uuid) -> Result<(), SchedulerError> { // } - pub fn insert(&self, task: Box) -> Result { - let id = self.runner.add(task.into())?; - self.running.write()?.insert(id); + pub async fn insert(&self, task: Box) -> Result { + let id = self.runner.add(task.into()).await?; + self.running.write().await.insert(id); Ok(id) } - pub fn insert_list(&self, tasks: Vec>) -> Result, SchedulerError> { + pub async fn insert_list( + &self, + tasks: Vec>, + ) -> Result, SchedulerError> { let ids = self .runner - .add_list(tasks.into_iter().map(|t| t.into()).collect())?; - self.running.write()?.extend(&ids); + .add_list(tasks.into_iter().map(|t| t.into()).collect()) + .await?; + self.running.write().await.extend(&ids); Ok(ids) } @@ -91,51 +95,51 @@ impl Scheduler { } } -#[cfg(test)] -mod tests { - use std::{error::Error, thread, time::Duration}; - - use crate::scheduler::{ - tasks::{async_basic::AsyncBasic, basic::BasicTask}, - Scheduler, - }; - - use uuid::Uuid; - - fn execution(scheduler: Scheduler, id: Uuid) -> Result<(), Box> { - for i in 1..=2 { - println!("Loop {} start", i); - println!("Run {}", i); - scheduler.run(&id)?; - println!("Sleep {}", i); - thread::sleep(Duration::from_secs(4)); - println!("Sleep {} finished", i); - scheduler.stop(&id)?; - println!("Stop {}", i); - thread::sleep(Duration::from_secs(2)); - println!("Loop {} finished", i); - } - Ok(()) - } - - #[test] - fn test_run() -> Result<(), Box> { - let scheduler = Scheduler::default(); - let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; - let id = scheduler.insert(Box::new(task))?; - scheduler.start()?; - execution(scheduler, id) - } - - #[test] - fn test_run_async() -> Result<(), Box> { - let scheduler = Scheduler::default(); - let task = AsyncBasic::new( - || Box::pin(async { println!("test async print") }), - "* * * * * *", - )?; - let id = scheduler.insert(Box::new(task))?; - scheduler.start()?; - execution(scheduler, id) - } -} +// #[cfg(test)] +// mod tests { +// use std::{error::Error, thread, time::Duration}; +// +// use crate::scheduler::{ +// tasks::{async_basic::AsyncBasic, basic::BasicTask}, +// Scheduler, +// }; +// +// use uuid::Uuid; +// +// async fn execution(scheduler: Scheduler, id: Uuid) -> Result<(), Box> { +// for i in 1..=2 { +// println!("Loop {} start", i); +// println!("Run {}", i); +// scheduler.run(&id).await?; +// println!("Sleep {}", i); +// thread::sleep(Duration::from_secs(4)); +// println!("Sleep {} finished", i); +// scheduler.stop(&id)?; +// println!("Stop {}", i); +// thread::sleep(Duration::from_secs(2)); +// println!("Loop {} finished", i); +// } +// Ok(()) +// } +// +// #[tokio::test] +// async fn test_run() -> Result<(), Box> { +// let scheduler = Scheduler::default(); +// let task = BasicTask::new(|| println!("test print"), "* * * * * *")?; +// let id = scheduler.insert(Box::new(task)).await?; +// scheduler.start().await?; +// execution(scheduler, id) +// } +// +// #[tokio::test] +// async fn test_run_async() -> Result<(), Box> { +// let scheduler = Scheduler::default(); +// let task = AsyncBasic::new( +// || Box::pin(async { println!("test async print") }), +// "* * * * * *", +// )?; +// let id = scheduler.insert(Box::new(task)).await?; +// scheduler.start().await?; +// execution(scheduler, id) +// } +// } diff --git a/backend/server/src/scheduler/runner.rs b/backend/server/src/scheduler/runner.rs index b3ca00b..69dbe2c 100644 --- a/backend/server/src/scheduler/runner.rs +++ b/backend/server/src/scheduler/runner.rs @@ -1,20 +1,22 @@ -use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicBool, Ordering}, Arc, - RwLock, - }, -}; - use crate::scheduler::{ error::{SchedulerError, SchedulerError::IdNotFound}, sleeper::SkippableSleeper, tasks::CronTask, }; +use futures::future; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; use tokio::runtime::Runtime; +use tokio::sync::RwLock; use uuid::Uuid; +#[derive(Debug)] struct TaskStatus { sleeper: RwLock, is_running: AtomicBool, @@ -40,13 +42,22 @@ impl TaskEntity { } pub async fn run_blocking(&self) { + dbg!("entered"); + dbg!("chkpt 1"); loop { - self.status.sleeper.read().unwrap().start(); + dbg!("chkpt 2"); + // deadlock spotted + self.status.sleeper.read().await.start().await; + dbg!("chkpt 3"); if self.status.expected_running.load(Ordering::SeqCst) { + dbg!("chkpt 4"); self.func.call().await; + dbg!("chkpt 5"); if let Some(next_execution) = self.func.get_next_execution() { - *(self.status.sleeper.write().unwrap()) = next_execution.into(); + dbg!("chkpt 6a"); + *(self.status.sleeper.write().await) = next_execution.into(); } else { + dbg!("chkpt 6b"); self.status.is_running.store(false, Ordering::SeqCst); break; } @@ -56,10 +67,10 @@ impl TaskEntity { } } - pub fn terminate(&self) { + pub async fn terminate(&self) { self.status.expected_running.store(false, Ordering::SeqCst); self.status.is_running.store(false, Ordering::SeqCst); - self.status.sleeper.read().unwrap().skip(); + self.status.sleeper.read().await.skip(); } } @@ -76,8 +87,8 @@ impl Runner { } } - pub fn run_all(&self) -> Result<(), SchedulerError> { - let list = &self.task_list.read()?; + pub async fn run_all(&self) -> Result<(), SchedulerError> { + let list = &self.task_list.read().await; self.rt.block_on(async { for (_, entity) in list.iter() { self.run_entity(entity.clone()).await @@ -86,8 +97,8 @@ impl Runner { Ok(()) } - pub fn run_id(&self, id: &Uuid) -> Result<(), SchedulerError> { - if let Some(entity) = self.task_list.read()?.get(id).cloned() { + pub async fn run_id(&self, id: &Uuid) -> Result<(), SchedulerError> { + if let Some(entity) = self.task_list.read().await.get(id).cloned() { self.rt.block_on(async { self.run_entity(entity).await }); Ok(()) } else { @@ -100,60 +111,67 @@ impl Runner { && entity.status.expected_running.load(Ordering::SeqCst) { entity.status.is_running.store(true, Ordering::SeqCst); - tokio::spawn({ + let fut = { let entity_clone = entity.clone(); async move { entity_clone.run_blocking().await } - }) - .await - .expect("tokio runtime error"); + }; + dbg!("spawn"); + tokio::spawn(fut); } } - pub fn add_with_id( + pub async fn add_with_id( &self, task: Arc>, id: &Uuid, ) -> Result<(), SchedulerError> { self.task_list - .write()? + .write() + .await .insert(*id, TaskEntity::new(task).into()); Ok(()) } - pub fn add(&self, task: Arc>) -> Result { + pub async fn add(&self, task: Arc>) -> Result { let id = Uuid::new_v4(); - self.add_with_id(task, &id)?; + self.add_with_id(task, &id).await?; Ok(id) } - pub fn add_list_with_id( + pub async fn add_list_with_id( &self, tasks: Vec<(&Uuid, Arc>)>, ) -> Result<(), SchedulerError> { - tasks + let fut = tasks .into_iter() - .try_for_each(|(id, task)| self.add_with_id(task, id)) + .map(async |(id, entity)| self.add_with_id(entity, id).await) + .collect::>(); + future::join_all(fut).await.into_iter().collect() } - pub fn add_list( + pub async fn add_list( &self, tasks: Vec>>, ) -> Result, SchedulerError> { - tasks + let fut = tasks .into_iter() - .map(|task| { + .map(async |task| { let id = Uuid::new_v4(); - self.add_with_id(task, &id).map(move |_| id) + self.add_with_id(task, &id).await.map(move |_| id) }) - .collect::, _>>() + .collect::>(); + future::join_all(fut).await.into_iter().collect() } - pub fn terminate_remove(&self, id: Uuid) -> Result>, SchedulerError> { - let mut list = self.task_list.write()?; + pub async fn terminate_remove( + &self, + id: Uuid, + ) -> Result>, SchedulerError> { + let mut list = self.task_list.write().await; match list.get(&id) { None => Err(IdNotFound), Some(task) => { - task.terminate(); + task.terminate().await; let func = list.remove(&id).unwrap().func.clone(); Ok(func.clone()) } diff --git a/backend/server/src/scheduler/sleeper.rs b/backend/server/src/scheduler/sleeper.rs index abc085a..93b913f 100644 --- a/backend/server/src/scheduler/sleeper.rs +++ b/backend/server/src/scheduler/sleeper.rs @@ -1,80 +1,81 @@ -use std::{ - sync::{Arc, Condvar, Mutex}, - thread, - time::Duration, -}; - use chrono::{DateTime, Utc}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{sync::Arc, time::Duration}; +use tokio::sync::Notify; -#[derive(Clone, Debug, Default)] +#[derive(Debug, Clone)] pub struct SkippableSleeper { - pub duration: Duration, - state: Arc<(Mutex, Condvar)>, + duration: Duration, + skipped: Arc, + notify: Arc, } impl SkippableSleeper { pub fn new(duration: Duration) -> Self { - SkippableSleeper { + Self { duration, - state: Arc::new((Mutex::new(false), Condvar::new())), + skipped: Arc::new(AtomicBool::new(false)), + notify: Arc::new(Notify::new()), } } - // Blocks until the duration has passed or skip() is called - pub fn start(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - let state_clone = Arc::clone(&self.state); - let duration_clone = self.duration; - thread::spawn(move || { - thread::sleep(duration_clone); - let (lock, cvar) = &*state_clone; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); - }); - while !*received { - received = cvar.wait(received).unwrap(); + pub async fn start(&self) { + if self.skipped.load(Ordering::Acquire) { + return; + } + + let sleep = tokio::time::sleep(self.duration); + let notify = self.notify.notified(); + + tokio::select! { + _ = sleep => {} + _ = notify => { + self.skipped.store(true, Ordering::Release); + } } } - // Skip the blocking delay pub fn skip(&self) { - let (lock, cvar) = &*self.state; - let mut received = lock.lock().unwrap(); - *received = true; - cvar.notify_one(); + self.notify.notify_waiters(); } } impl From> for SkippableSleeper { fn from(value: DateTime) -> Self { - let from_utc_timestamp = value.timestamp() as u64; - let curr_utc_timestamp = Utc::now().timestamp() as u64; - SkippableSleeper::new(Duration::from_secs(from_utc_timestamp - curr_utc_timestamp)) + let now = Utc::now(); + let duration = value.signed_duration_since(now); + let duration = duration.to_std().unwrap_or(Duration::ZERO); + Self::new(duration) } } #[cfg(test)] -mod test { - use crate::scheduler::sleeper::SkippableSleeper; - use std::thread; - use std::time::Duration; +mod tests { + use super::*; + use tokio::time::{sleep, Duration}; - #[test] - fn usage() { - let sleeper = SkippableSleeper::new(Duration::new(10, 0)); - thread::spawn({ - let sleeper = sleeper.clone(); - move || { - println!("Sleeping..."); - sleeper.start(); - println!("Woke up!"); - } + #[tokio::test] + async fn skip_sleeper() { + let sleeper = SkippableSleeper::new(Duration::from_secs(10)); + let sleeper_clone = sleeper.clone(); + + let handle = tokio::spawn(async move { + println!("Sleeping..."); + sleeper_clone.start().await; + println!("Woke up!"); }); - println!("Start Sleeping..."); - thread::sleep(Duration::from_secs(4)); + + println!("Start waiting..."); + sleep(Duration::from_secs(4)).await; println!("Skipping sleep..."); sleeper.skip(); + + handle.await.unwrap(); + } + + #[tokio::test] + async fn normal_sleeper() { + let sleeper = SkippableSleeper::new(Duration::from_secs(4)); + sleeper.start().await; } } diff --git a/backend/server/src/utils/git.rs b/backend/server/src/utils/git.rs index 9c48fc0..94b00cc 100644 --- a/backend/server/src/utils/git.rs +++ b/backend/server/src/utils/git.rs @@ -3,11 +3,12 @@ /// https://github.com/rust-lang/git2-rs/commit/f3b87baed1e33d6c2d94fe1fa6aa6503a071d837 /// use git2::{ - AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, build, + build, AnnotatedCommit, AutotagOption, Delta, FetchOptions, Reference, Remote, Repository, }; use std::path::PathBuf; use tracing::{error, warn}; + #[derive(Debug, Eq, PartialEq)] pub struct FileDelta { pub old_path: Option, @@ -15,7 +16,6 @@ pub struct FileDelta { pub status: Delta, } -#[allow(dead_code)] fn do_fetch<'a>( repo: &'a Repository, refs: &[&str], @@ -30,7 +30,6 @@ fn do_fetch<'a>( repo.reference_to_annotated_commit(&fetch_head) } -#[allow(dead_code)] fn fast_forward( repo: &Repository, lb: &mut Reference, @@ -48,7 +47,6 @@ fn fast_forward( Ok(()) } -#[allow(dead_code)] fn normal_merge( repo: &Repository, local: &AnnotatedCommit, @@ -85,7 +83,6 @@ fn normal_merge( Ok(()) } -#[allow(dead_code)] fn do_merge<'a>( repo: &'a Repository, remote_branch: &str, @@ -122,7 +119,6 @@ fn do_merge<'a>( calculate_diff(repo, &head_commit, &fetch_commit) } -#[allow(dead_code)] fn calculate_diff( repo: &Repository, local: &AnnotatedCommit, @@ -153,7 +149,6 @@ fn calculate_diff( Ok(updated_files.into_iter().collect()) } -#[allow(dead_code)] pub fn sync( remote_name: &str, remote_branch: &str, From d54996b69045ba3c3b295a5488e6322da65b77d1 Mon Sep 17 00:00:00 2001 From: Lumither Tao Date: Thu, 13 Mar 2025 16:16:50 -0700 Subject: [PATCH 16/32] feat(client) add client --- client/.gitignore | 11 +++++ client/README.md | 30 ++++++++++++ client/cli | 27 +++++++++++ client/requirements.txt | 4 ++ client/templates/post.py | 87 ++++++++++++++++++++++++++++++++++ client/templates/utils.py | 17 +++++++ client/utils/ansi.py | 31 ++++++++++++ client/utils/autocompletion.py | 46 ++++++++++++++++++ client/utils/cache.py | 49 +++++++++++++++++++ client/utils/index.py | 27 +++++++++++ client/utils/lang.py | 13 +++++ 11 files changed, 342 insertions(+) create mode 100644 client/.gitignore create mode 100644 client/README.md create mode 100755 client/cli create mode 100644 client/requirements.txt create mode 100644 client/templates/post.py create mode 100644 client/templates/utils.py create mode 100644 client/utils/ansi.py create mode 100644 client/utils/autocompletion.py create mode 100644 client/utils/cache.py create mode 100644 client/utils/index.py create mode 100644 client/utils/lang.py diff --git a/client/.gitignore b/client/.gitignore new file mode 100644 index 0000000..1c461db --- /dev/null +++ b/client/.gitignore @@ -0,0 +1,11 @@ +.venv/ + +.idea/ + +.DS_store + +__pycache__ + +.cache + +content/* diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..76fc27b --- /dev/null +++ b/client/README.md @@ -0,0 +1,30 @@ +## Module Structure + +```shell +. +├── README.md +├── cli +├── content +│ ├── pages +│ │ └── about +│ │ └── .md +│ └── posts +│ └── +│ └── +│ ├── assets +│ │ └── +│ ├── translate_.md +│ └── index.md +├── requirements.txt +├── templates +│ ├──