From 681cc0f59d61212aaf800df58afcafe08e3806ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20W=C3=B6lfer?= Date: Sat, 17 Jan 2026 21:58:29 +0100 Subject: [PATCH] Cleaned up project structure --- .gitea/workflows/test.yaml | 11 ++ Cargo.lock | 12 ++ Cargo.toml | 1 + idea.md | 1 - src/{worldsdc => fetching}/caching.rs | 0 src/fetching/mod.rs | 39 +++++ src/{worldsdc => fetching}/scoringdance.rs | 4 +- src/fetching/types.rs | 67 +++++++++ src/fetching/worldsdc.rs | 61 ++++++++ src/main.rs | 30 +++- src/updater.rs | 22 +-- src/watchdog.rs | 27 ++-- src/wikiinfo.rs | 7 +- src/worldsdc/mod.rs | 133 ------------------ .../2025-08-12_emeline.json | 0 .../2025-10-02_polina.html | 0 .../2026-01-07_robert.html | 0 17 files changed, 253 insertions(+), 162 deletions(-) create mode 100644 .gitea/workflows/test.yaml delete mode 100644 idea.md rename src/{worldsdc => fetching}/caching.rs (100%) create mode 100644 src/fetching/mod.rs rename src/{worldsdc => fetching}/scoringdance.rs (98%) create mode 100644 src/fetching/types.rs create mode 100644 src/fetching/worldsdc.rs delete mode 100644 src/worldsdc/mod.rs rename emeline.json => test_data/2025-08-12_emeline.json (100%) rename polina.html => test_data/2025-10-02_polina.html (100%) rename robert-2026-01-07.html => test_data/2026-01-07_robert.html (100%) diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml new file mode 100644 index 0000000..49fbff1 --- /dev/null +++ b/.gitea/workflows/test.yaml @@ -0,0 +1,11 @@ +name: Rust + +on: [push, pull_request] + +jobs: + build_and_test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + - run: cargo test --all-features diff --git a/Cargo.lock b/Cargo.lock index fd8d4ea..c3ea671 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,6 +245,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -3192,6 +3203,7 @@ dependencies = [ name = "teachertracker-rs" version = "0.1.4" dependencies = [ + "async-trait", "chrono", "clap", "futures", diff --git a/Cargo.toml b/Cargo.toml index 71299e6..95ddbeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,4 @@ thiserror = "2.0.12" tokio = { version = "1.46.1", features = ["rt"] } tracing = { version = "0.1.41", default-features = false, features = ["std"] } tracing-subscriber = "0.3.19" +async-trait = "0.1.79" diff --git a/idea.md b/idea.md deleted file mode 100644 index 3749747..0000000 --- a/idea.md +++ /dev/null @@ -1 +0,0 @@ -https://dancing.thasky.one/api.php?action=query&format=json&list=querypage&formatversion=2&qppage=Wantedpages diff --git a/src/worldsdc/caching.rs b/src/fetching/caching.rs similarity index 100% rename from src/worldsdc/caching.rs rename to src/fetching/caching.rs diff --git a/src/fetching/mod.rs b/src/fetching/mod.rs new file mode 100644 index 0000000..f6882a0 --- /dev/null +++ b/src/fetching/mod.rs @@ -0,0 +1,39 @@ +use crate::dance_info::DanceInfo; +use crate::fetching::types::DanceInfoError; + +mod scoringdance; +mod worldsdc; +pub mod types; + +use async_trait::async_trait; +use std::sync::Arc; + +#[async_trait] +pub trait WsdcFetcher: Send + Sync { + async fn fetch(&self, id: u32) -> Result; +} + +pub struct WorldsdcFetcher; +pub struct ScoringDanceFetcher; + +#[async_trait] +impl WsdcFetcher for WorldsdcFetcher { + async fn fetch(&self, id: u32) -> Result { + worldsdc::fetch_wsdc_info_wsdc(id).await + } +} + +#[async_trait] +impl WsdcFetcher for ScoringDanceFetcher { + async fn fetch(&self, id: u32) -> Result { + scoringdance::fetch_wsdc_info_scoring_dance(id).await + } +} + +/// Convenience alias for a shared, dynamic fetcher +pub type DynWsdcFetcher = Arc; + +/// Back-compat helper that uses the `WorldsdcFetcher`. +pub async fn fetch_wsdc_info(id: u32) -> Result { + WorldsdcFetcher.fetch(id).await +} diff --git a/src/worldsdc/scoringdance.rs b/src/fetching/scoringdance.rs similarity index 98% rename from src/worldsdc/scoringdance.rs rename to src/fetching/scoringdance.rs index 5812229..bd43afb 100644 --- a/src/worldsdc/scoringdance.rs +++ b/src/fetching/scoringdance.rs @@ -6,7 +6,7 @@ use scraper::{ElementRef, Html, Selector}; use crate::{ app_signature, dance_info::{CompState, DanceInfo, DanceRank, DanceRole}, - worldsdc::DanceInfoError, + fetching::DanceInfoError, }; #[derive(thiserror::Error, Debug)] pub enum ScoringParseError { @@ -180,7 +180,7 @@ fn parse_info(html: &str) -> Result { #[test] fn test_parse_table() { - dbg!(parse_info(include_str!("../../robert-2026-01-07.html"))); + dbg!(parse_info(include_str!("../../test_data/2026-01-07_robert.html"))); } pub async fn fetch_wsdc_info_scoring_dance(id: u32) -> Result { diff --git a/src/fetching/types.rs b/src/fetching/types.rs new file mode 100644 index 0000000..d623f06 --- /dev/null +++ b/src/fetching/types.rs @@ -0,0 +1,67 @@ +use crate::dance_info::{CompState, DanceInfo, DanceRank, DanceRole}; + +#[derive(thiserror::Error, Debug)] +pub enum DanceInfoError { + #[error("Failed to build client: {0}")] + ClientBuild(reqwest::Error), + #[error("Failed to build request: {0}")] + RequestBuild(reqwest::Error), + #[error("Request error: {0:#?}")] + Request(reqwest::Error), + #[error("Failed to parse response: {0}")] + JsonParse(reqwest::Error), + #[error("Failed to parse html: {0}")] + HtmlParse(#[from] super::scoringdance::ScoringParseError), +} + +#[derive(serde::Deserialize, Debug)] +enum OptionalDanceRank { + #[serde(rename = "N/A")] + NotAvailable, + #[serde(untagged)] + Rank(DanceRank), +} + +#[derive(serde::Deserialize, Debug)] +enum OptionalDancePoints { + #[serde(rename = "N/A")] + NotAvailable, + #[serde(untagged)] + Points(u16), +} + +#[derive(serde::Deserialize, Debug)] +pub struct DanceInfoParser { + pub dancer_first: String, + pub dancer_last: String, + pub short_dominate_role: DanceRole, + #[allow(dead_code)] + pub short_non_dominate_role: DanceRole, + pub dominate_role_highest_level_points: u16, + pub dominate_role_highest_level: DanceRank, + pub non_dominate_role_highest_level_points: OptionalDancePoints, + pub non_dominate_role_highest_level: OptionalDanceRank, +} + +impl From for DanceInfo { + fn from(value: DanceInfoParser) -> Self { + let non_dominant_role_comp = if let OptionalDanceRank::Rank(r) = + value.non_dominate_role_highest_level + && let OptionalDancePoints::Points(l) = value.non_dominate_role_highest_level_points + { + Some(CompState { rank: r, points: l }) + } else { + None + }; + Self { + firstname: value.dancer_first, + lastname: value.dancer_last, + dominant_role: value.short_dominate_role, + dominant_role_comp: CompState { + rank: value.dominate_role_highest_level, + points: value.dominate_role_highest_level_points, + }, + non_dominant_role_comp, + } + } +} diff --git a/src/fetching/worldsdc.rs b/src/fetching/worldsdc.rs new file mode 100644 index 0000000..a33bb6f --- /dev/null +++ b/src/fetching/worldsdc.rs @@ -0,0 +1,61 @@ +use std::collections::HashMap; + +use crate::{ + app_signature, + dance_info::{CompState, DanceInfo, DanceRank, DanceRole}, + fetching::{DanceInfoError, types::DanceInfoParser, scoringdance::{self, fetch_wsdc_info_scoring_dance}}, +}; +use reqwest::ClientBuilder; + +pub async fn fetch_wsdc_info_wsdc(id: u32) -> Result { + let client = ClientBuilder::new() + .user_agent(app_signature()) + .build() + .map_err(DanceInfoError::ClientBuild)?; + + let mut params = HashMap::new(); + + let url = if cfg!(test) { + // "https://o5grQU3Y.free.beeceptor.com/lookup2020/find" + "http://localhost:8000" + } else { + "https://points.worldsdc.com/lookup2020/find" + }; + params.insert("num", id.to_string()); + let request = client + .request(reqwest::Method::POST, url) + .form(¶ms) + .build() + .map_err(DanceInfoError::RequestBuild)?; + let response = client + .execute(request) + .await + .map_err(DanceInfoError::Request)?; + + let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?; + Ok(x.into()) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, reason = "Allow unwrap in tests")] + use crate::fetching::fetch_wsdc_info; + + #[test] + #[ignore = "Only run when the mock api is setup"] + fn test_fetch_wsdc() { + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(o) => o, + Err(e) => { + tracing::error!("Could not start runtime: {e}"); + return; + } + }; + let x = rt.block_on(fetch_wsdc_info(7)); + dbg!(&x); + x.unwrap(); + } +} diff --git a/src/main.rs b/src/main.rs index 40a17ce..f3f5bae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,13 +27,14 @@ use tracing::level_filters::LevelFilter; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; use crate::watchdog::{update_wanted_ids, watch_wanted}; +use crate::fetching::{DynWsdcFetcher, WorldsdcFetcher, ScoringDanceFetcher}; mod dance_info; +mod fetching; mod updater; mod watchdog; mod wikiinfo; mod wikipage; -mod worldsdc; #[allow(dead_code)] #[allow(clippy::print_stdout, reason = "We want to print here")] @@ -107,11 +108,22 @@ fn init_sentry() -> Option { use clap::{Parser, Subcommand}; #[derive(Parser)] -#[command(name = "myapp")] -#[command(about = "A simple CLI app with subcommands", long_about = None)] +#[command(name = "teachertracking")] +#[command(about = "MediaWiki Bot to keep West Coast Swing Teacher Scores updated", long_about = None)] struct Cli { #[command(subcommand)] command: Option, + + #[clap(value_enum)] + #[arg(default_value_t)] + backend: WsdcPointsBackend, +} + +#[derive(clap::ValueEnum, Debug, Clone, Default)] +enum WsdcPointsBackend { + ScoringDance, + #[default] + WorldSDC, } #[derive(Subcommand)] @@ -132,10 +144,16 @@ fn main() -> Result<(), AppError> { let bot = rt.block_on(Bot::from_path(Path::new("./mwbot.toml")))?; + // Build a dynamic fetcher based on CLI selection + let fetcher: DynWsdcFetcher = match cli.backend { + WsdcPointsBackend::ScoringDance => std::sync::Arc::new(ScoringDanceFetcher {}), + WsdcPointsBackend::WorldSDC => std::sync::Arc::new(WorldsdcFetcher {}), + }; + match &cli.command { Some(Commands::Missing) => { rt.block_on(async { - let wanted = wikiinfo::wanted_ids(bot.clone()).await; + let wanted = wikiinfo::wanted_ids(bot.clone(), fetcher.clone()).await; tracing::info!( "Missing ids: {}", wanted @@ -145,7 +163,7 @@ fn main() -> Result<(), AppError> { .collect::>() .join("\n") ); - update_wanted_ids(&wanted, &[]).await; + update_wanted_ids(&wanted, &[], fetcher.clone()).await; }); return Ok(()); @@ -156,7 +174,7 @@ fn main() -> Result<(), AppError> { reason = "This is a false positive I think, I just want to loop infinitely on two futures" )] rt.block_on(async { - futures::join!(watch_wanted(bot.clone()), updater::update_wsdc(bot)) + futures::join!(watch_wanted(bot.clone(), fetcher.clone()), updater::update_wsdc(bot, fetcher.clone())) }); } } diff --git a/src/updater.rs b/src/updater.rs index a06db76..ad854e1 100644 --- a/src/updater.rs +++ b/src/updater.rs @@ -5,32 +5,38 @@ use rand::seq::SliceRandom as _; use tokio::time::sleep; use crate::{watchdog::generate_page, wikiinfo::index_wsdc_ids}; +use crate::fetching::DynWsdcFetcher; -pub async fn update_wsdc(bot: Bot) -> ! { +pub async fn update_wsdc(bot: Bot, fetcher: DynWsdcFetcher) -> ! { loop { - update_all_teachers(&bot).await; + update_all_teachers(&bot, fetcher.clone()).await; } } /// Updates all teachers once -async fn update_all_teachers(bot: &Bot) { - let mut l = index_wsdc_ids(bot).await; +async fn update_all_teachers(bot: &Bot, fetcher: DynWsdcFetcher) { + let mut l = index_wsdc_ids(bot, fetcher.clone()).await; l.shuffle(&mut rand::rng()); tracing::info!("We have to update {} pages", l.len()); let wait_duration = Duration::from_hours(6); for (index, page) in l { - process_page(wait_duration, index, page).await; + process_page(wait_duration, index, page, fetcher.clone()).await; } tracing::info!("Updates all pages"); } -#[tracing::instrument(skip(page, wait_duration))] -async fn process_page(wait_duration: Duration, index: u32, page: mwbot::Page) { +#[tracing::instrument(skip(page, wait_duration, fetcher))] +async fn process_page( + wait_duration: Duration, + index: u32, + page: mwbot::Page, + fetcher: DynWsdcFetcher, +) { tracing::info!("Next up"); sleep(wait_duration).await; - match generate_page(index, page).await { + match generate_page(index, page, fetcher).await { Ok(()) => (), Err(err) => { tracing::error!("Error updating: {err}"); diff --git a/src/watchdog.rs b/src/watchdog.rs index 76a7b33..15ad066 100644 --- a/src/watchdog.rs +++ b/src/watchdog.rs @@ -2,8 +2,9 @@ use std::time::Duration; use crate::app_signature; use crate::wikipage::InfoCompileError; -use crate::worldsdc::DanceInfoError; -use crate::{wikiinfo::wanted_ids, wikipage::page_from_info, worldsdc::fetch_wsdc_info}; +use crate::fetching::types::DanceInfoError; +use crate::{wikiinfo::wanted_ids, wikipage::page_from_info}; +use crate::fetching::DynWsdcFetcher; use mwbot::SaveOptions; use mwbot::{Bot, Page}; @@ -31,27 +32,31 @@ impl Ticker { /// Continuously monitors teacher IDs that do not have a corresponding teacher WSDC page, ignoring those that fail processing. #[tracing::instrument(skip_all)] -pub async fn watch_wanted(bot: Bot) -> ! { +pub async fn watch_wanted(bot: Bot, fetcher: DynWsdcFetcher) -> ! { let mut ignored_ids = vec![]; let mut heartbeat_ticker = Ticker::new(120); loop { if heartbeat_ticker.tick() { tracing::info!(failed_id_count = ignored_ids.len(), "Watchdog check..."); } - let wanted = wanted_ids(bot.clone()).await; - let new_ignored = update_wanted_ids(&wanted, &ignored_ids).await; + let wanted = wanted_ids(bot.clone(), fetcher.clone()).await; + let new_ignored = update_wanted_ids(&wanted, &ignored_ids, fetcher.clone()).await; ignored_ids.extend(new_ignored); tokio::time::sleep(Duration::from_secs(30)).await; } } -pub async fn update_wanted_ids(wanted: &[(u32, Page)], ignored_ids: &[u32]) -> Vec { +pub async fn update_wanted_ids( + wanted: &[(u32, Page)], + ignored_ids: &[u32], + fetcher: DynWsdcFetcher, +) -> Vec { let mut new_ignored = vec![]; for (id, page) in wanted.iter().filter(|(x, _)| !ignored_ids.contains(x)) { let span = tracing::info_span!("update", id); let _enter = span.enter(); - if let Err(e) = generate_page(*id, page.clone()).await { + if let Err(e) = generate_page(*id, page.clone(), fetcher.clone()).await { tracing::error!("{e}"); new_ignored.push(*id); } @@ -71,9 +76,13 @@ pub enum GeneratePageError { Save(#[from] mwbot::Error), } -pub async fn generate_page(id: u32, page: mwbot::Page) -> Result<(), GeneratePageError> { +pub async fn generate_page( + id: u32, + page: mwbot::Page, + fetcher: DynWsdcFetcher, +) -> Result<(), GeneratePageError> { tracing::info!("Generating page for {id}"); - let info = fetch_wsdc_info(id).await?; + let info = fetcher.fetch(id).await?; let code = page_from_info(info)?; diff --git a/src/wikiinfo.rs b/src/wikiinfo.rs index ac61d50..22ae03d 100644 --- a/src/wikiinfo.rs +++ b/src/wikiinfo.rs @@ -2,9 +2,10 @@ use mwbot::{ Bot, Page, generators::{Generator, querypage::QueryPage, search::Search}, }; +use crate::fetching::DynWsdcFetcher; -pub async fn wanted_ids(bot: Bot) -> Vec<(u32, Page)> { - let mut gene = QueryPage::new("Wantedpages").generate(&bot); +pub async fn wanted_ids(_bot: Bot, _fetcher: DynWsdcFetcher) -> Vec<(u32, Page)> { + let mut gene = QueryPage::new("Wantedpages").generate(&_bot); let mut result = vec![]; while let Some(x) = gene.recv().await { let p = match x { @@ -63,7 +64,7 @@ fn parse_wsdc_page_name(name: &str) -> Result { // } #[allow(dead_code)] -pub async fn index_wsdc_ids(bot: &Bot) -> Vec<(u32, Page)> { +pub async fn index_wsdc_ids(bot: &Bot, _fetcher: DynWsdcFetcher) -> Vec<(u32, Page)> { let mut gene = Search::new("WSDC/").generate(bot); let mut result = vec![]; while let Some(x) = gene.recv().await { diff --git a/src/worldsdc/mod.rs b/src/worldsdc/mod.rs deleted file mode 100644 index 9033d60..0000000 --- a/src/worldsdc/mod.rs +++ /dev/null @@ -1,133 +0,0 @@ -use std::collections::HashMap; - -use crate::{ - app_signature, - dance_info::{CompState, DanceInfo, DanceRank, DanceRole}, - worldsdc::scoringdance::fetch_wsdc_info_scoring_dance, -}; -use reqwest::ClientBuilder; -mod scoringdance; - -pub async fn fetch_wsdc_info_wsdc(id: u32) -> Result { - let client = ClientBuilder::new() - .user_agent(app_signature()) - .build() - .map_err(DanceInfoError::ClientBuild)?; - - let mut params = HashMap::new(); - - let url = if cfg!(test) { - // "https://o5grQU3Y.free.beeceptor.com/lookup2020/find" - "http://localhost:8000" - } else { - "https://points.worldsdc.com/lookup2020/find" - }; - params.insert("num", id.to_string()); - let request = client - .request(reqwest::Method::POST, url) - .form(¶ms) - .build() - .map_err(DanceInfoError::RequestBuild)?; - let response = client - .execute(request) - .await - .map_err(DanceInfoError::Request)?; - - let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?; - Ok(x.into()) -} - -pub async fn fetch_wsdc_info(id: u32) -> Result { - // fetch_wsdc_info_scoring_dance(id).await - fetch_wsdc_info_wsdc(id).await -} - -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used, reason = "Allow unwrap in tests")] - use crate::worldsdc::fetch_wsdc_info; - - #[test] - #[ignore = "Only run when the mock api is setup"] - fn test_fetch_wsdc() { - let rt = match tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - { - Ok(o) => o, - Err(e) => { - tracing::error!("Could not start runtime: {e}"); - return; - } - }; - let x = rt.block_on(fetch_wsdc_info(7)); - dbg!(&x); - x.unwrap(); - } -} - -#[derive(thiserror::Error, Debug)] -pub enum DanceInfoError { - #[error("Failed to build client: {0}")] - ClientBuild(reqwest::Error), - #[error("Failed to build request: {0}")] - RequestBuild(reqwest::Error), - #[error("Request error: {0:#?}")] - Request(reqwest::Error), - #[error("Failed to parse response: {0}")] - JsonParse(reqwest::Error), - #[error("Failed to parse html: {0}")] - HtmlParse(#[from] scoringdance::ScoringParseError), -} - -#[derive(serde::Deserialize, Debug)] -enum OptionalDanceRank { - #[serde(rename = "N/A")] - NotAvailable, - #[serde(untagged)] - Rank(DanceRank), -} - -#[derive(serde::Deserialize, Debug)] -enum OptionalDancePoints { - #[serde(rename = "N/A")] - NotAvailable, - #[serde(untagged)] - Points(u16), -} - -#[derive(serde::Deserialize, Debug)] -struct DanceInfoParser { - pub dancer_first: String, - pub dancer_last: String, - pub short_dominate_role: DanceRole, - #[allow(dead_code)] - pub short_non_dominate_role: DanceRole, - pub dominate_role_highest_level_points: u16, - pub dominate_role_highest_level: DanceRank, - pub non_dominate_role_highest_level_points: OptionalDancePoints, - pub non_dominate_role_highest_level: OptionalDanceRank, -} - -impl From for DanceInfo { - fn from(value: DanceInfoParser) -> Self { - let non_dominant_role_comp = if let OptionalDanceRank::Rank(r) = - value.non_dominate_role_highest_level - && let OptionalDancePoints::Points(l) = value.non_dominate_role_highest_level_points - { - Some(CompState { rank: r, points: l }) - } else { - None - }; - Self { - firstname: value.dancer_first, - lastname: value.dancer_last, - dominant_role: value.short_dominate_role, - dominant_role_comp: CompState { - rank: value.dominate_role_highest_level, - points: value.dominate_role_highest_level_points, - }, - non_dominant_role_comp, - } - } -} diff --git a/emeline.json b/test_data/2025-08-12_emeline.json similarity index 100% rename from emeline.json rename to test_data/2025-08-12_emeline.json diff --git a/polina.html b/test_data/2025-10-02_polina.html similarity index 100% rename from polina.html rename to test_data/2025-10-02_polina.html diff --git a/robert-2026-01-07.html b/test_data/2026-01-07_robert.html similarity index 100% rename from robert-2026-01-07.html rename to test_data/2026-01-07_robert.html