Fix tracing span, using official mwbot crate

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
-name = "dancing-bot-teachers"
-version = "0.1.0"
+name = "teachertracker-rs"
+version = "0.1.3"
 edition = "2024"
 authors = ["Lukas Wölfer <coding@thasky.one>"]
 description = "A MediaWiki bot that updates score information of teachers"
@@ -11,7 +11,11 @@ keywords = ["mediawiki", "bot", "teacher", "score", "automation"]
 categories = ["web-programming", "api-bindings", "automation"]
 
 [dependencies]
-mwbot = { git = "https://gitlab.wikimedia.org/repos/mwbot-rs/mwbot.git", rev = "05cbb12188f18e2da710de158d89a9a4f1b42689", default-features = false, features = ["generators", "mwbot_derive"] }
+chrono = "0.4.41"
+futures = "0.3.31"
+# mwbot = { git = "https://gitlab.wikimedia.org/repos/mwbot-rs/mwbot.git", rev = "05cbb12188f18e2da710de158d89a9a4f1b42689", default-features = false, features = ["generators", "mwbot_derive"] }
+mwbot = { version = "0.7.0", default-features = false, features = ["generators", "mwbot_derive"] }
+rand = "0.9.2"
 reqwest = "0.12.22"
 serde = { version = "1.0.219", features = ["derive"] }
 thiserror = "2.0.12"

src/dance_info.rs

@@ -1,5 +1,3 @@
-use std::collections::HashMap;
-
 #[derive(serde::Deserialize, Debug, PartialEq, Eq)]
 pub enum DanceRole {
     Leader,
@@ -48,40 +46,13 @@ impl DanceRank {
     }
 }
 
-#[derive(serde::Deserialize, Debug)]
-enum OptionalDanceRank {
-    #[serde(rename = "N/A")]
-    NotAvailable,
-    #[serde(untagged)]
-    Rank(DanceRank),
-}
-
-#[derive(serde::Deserialize, Debug)]
-enum OptionalDancePoints {
-    #[serde(rename = "N/A")]
-    NotAvailable,
-    #[serde(untagged)]
-    Points(u16),
-}
-
-#[derive(serde::Deserialize, Debug)]
-struct DanceInfoParser {
-    pub dancer_first: String,
-    pub dancer_last: String,
-    pub short_dominate_role: DanceRole,
-    #[allow(dead_code)]
-    pub short_non_dominate_role: DanceRole,
-    pub dominate_role_highest_level_points: u16,
-    pub dominate_role_highest_level: DanceRank,
-    pub non_dominate_role_highest_level_points: OptionalDancePoints,
-    pub non_dominate_role_highest_level: OptionalDanceRank,
-}
-
 #[derive(Debug)]
 pub struct CompState {
     pub rank: DanceRank,
     pub points: u16,
 }
+
+#[derive(Debug)]
 pub struct DanceInfo {
     pub firstname: String,
     pub lastname: String,
@@ -94,63 +65,4 @@ impl DanceInfo {
     pub fn name(&self) -> String {
         format!("{} {}", self.firstname, self.lastname)
     }
-
-    #[allow(dead_code)]
-    pub const fn non_dominant_role(&self) -> DanceRole {
-        self.dominant_role.other()
-    }
-}
-
-impl From<DanceInfoParser> for DanceInfo {
-    fn from(value: DanceInfoParser) -> Self {
-        let non_dominant_role_comp = if let OptionalDanceRank::Rank(r) =
-            value.non_dominate_role_highest_level
-            && let OptionalDancePoints::Points(l) = value.non_dominate_role_highest_level_points
-        {
-            Some(CompState { rank: r, points: l })
-        } else {
-            None
-        };
-        Self {
-            firstname: value.dancer_first,
-            lastname: value.dancer_last,
-            dominant_role: value.short_dominate_role,
-            dominant_role_comp: CompState {
-                rank: value.dominate_role_highest_level,
-                points: value.dominate_role_highest_level_points,
-            },
-            non_dominant_role_comp,
-        }
-    }
-}
-
-#[derive(thiserror::Error, Debug)]
-pub enum DanceInfoError {
-    #[error("Failed to build client: {0}")]
-    ClientBuild(reqwest::Error),
-    #[error("Request error: {0}")]
-    Request(reqwest::Error),
-    #[error("Failed to parse response: {0}")]
-    JsonParse(reqwest::Error),
-}
-
-pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
-    let client = reqwest::ClientBuilder::new()
-        .build()
-        .map_err(DanceInfoError::ClientBuild)?;
-
-    let mut params = HashMap::new();
-    params.insert("q", id.to_string());
-    let response = client
-        .request(
-            reqwest::Method::POST,
-            "https://points.worldsdc.com/lookup2020/find",
-        )
-        .form(&params)
-        .send()
-        .await
-        .map_err(DanceInfoError::Request)?;
-
-    let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
-    Ok(x.into())
 }

src/main.rs

@@ -13,17 +13,22 @@
     reason = "Disable this for most of the time, enable this for cleanup later"
 )]
 
+#![feature(never_type)]
+
 use mwbot::{
-    Bot,
+    Bot, ConfigError,
     generators::{Generator, SortDirection, categories::CategoryMemberSort},
 };
-use std::{error::Error, path::Path};
+use std::path::Path;
 
 use crate::watchdog::watch_wanted;
 
 mod dance_info;
+mod updater;
 mod watchdog;
 mod wikiinfo;
+mod wikipage;
+mod worldsdc;
 
 #[allow(dead_code)]
 #[allow(clippy::print_stdout, reason = "We want to print here")]
@@ -38,31 +43,37 @@ fn list_teacher_pages(bot: &Bot) -> tokio::sync::mpsc::Receiver<Result<mwbot::Pa
     pages.generate(bot)
 }
 
-fn main() -> Result<(), Box<dyn Error>> {
+#[must_use]
+pub fn app_signature() -> String {
+    format!("{} [{}]", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"))
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum AppError {
+    #[error("Runtime error: {0}")]
+    RuntimeError(#[from] std::io::Error),
+    #[error("Bot initialization error: {0}")]
+    BotError(#[from] ConfigError),
+}
+
+fn main() -> Result<(), AppError> {
     tracing_subscriber::fmt()
         .with_level(true)
         .with_max_level(tracing::Level::INFO)
         .init();
+    tracing::info!("Starting {}", app_signature());
 
-    let rt = match tokio::runtime::Builder::new_current_thread()
+    let rt = tokio::runtime::Builder::new_current_thread()
         .enable_all()
-        .build()
-    {
-        Ok(o) => o,
-        Err(e) => {
-            tracing::error!("Could not start runtime: {e}");
-            return Ok(());
-        }
-    };
-    rt.block_on(async {
-        let bot = match Bot::from_path(Path::new("./mwbot.toml")).await {
-            Ok(x) => x,
-            Err(e) => {
-                dbg!(e);
-                return Ok(());
-            }
-        };
-        watch_wanted(bot).await;
-        Ok(())
-    })
+        .build()?;
+
+    let bot = rt.block_on(Bot::from_path(Path::new("./mwbot.toml")))?;
+
+    #[allow(
+        unreachable_code,
+        reason = "This is a false positive I think, I just want to loop infinitely on two futures"
+    )]
+    rt.block_on(async { futures::join!(watch_wanted(bot.clone()), updater::update_wsdc(bot)) });
+
+    Ok(())
 }

src/updater.rs

@@ -0,0 +1,39 @@
+use std::time::Duration;
+
+use mwbot::Bot;
+use rand::seq::SliceRandom as _;
+use tokio::time::sleep;
+
+use crate::{watchdog::generate_page, wikiinfo::index_wsdc_ids};
+
+pub async fn update_wsdc(bot: Bot) -> ! {
+    loop {
+        update_all_teachers(&bot).await;
+    }
+}
+
+/// Updates all teachers once
+async fn update_all_teachers(bot: &Bot) {
+    let mut l = index_wsdc_ids(bot).await;
+    l.shuffle(&mut rand::rng());
+    tracing::info!("We have to update {} pages", l.len());
+    let wait_duration = Duration::from_hours(6);
+
+    for (index, page) in l {
+        process_page(wait_duration, index, page).await;
+    }
+    tracing::info!("Updates all pages");
+}
+
+#[tracing::instrument(skip(page, wait_duration))]
+async fn process_page(wait_duration: Duration, index: u32, page: mwbot::Page) {
+    tracing::info!("Next up");
+    sleep(wait_duration).await;
+
+    match generate_page(index, page).await {
+        Ok(()) => (),
+        Err(err) => {
+            tracing::error!("Error updating: {err}");
+        }
+    }
+}

src/watchdog.rs

@@ -1,106 +1,92 @@
 use std::time::Duration;
 
-use crate::{
-    dance_info::{DanceInfo, fetch_wsdc_info},
-    wikiinfo::wanted_ids,
-};
-use mwbot::{
-    Bot,
-    parsoid::{self, Template, Wikicode, map::IndexMap},
-};
-use mwbot::{SaveOptions, parsoid::WikinodeIterator};
-use tracing::Level;
+use crate::app_signature;
+use crate::wikipage::InfoCompileError;
+use crate::worldsdc::DanceInfoError;
+use crate::{wikiinfo::wanted_ids, wikipage::page_from_info, worldsdc::fetch_wsdc_info};
+use mwbot::SaveOptions;
+use mwbot::{Bot, Page};
 
-#[derive(thiserror::Error, Debug)]
-enum InfoCompileError {
-    #[error("Could not compile wikipage: {0}")]
-    CompileError(#[from] parsoid::Error),
+pub struct Ticker {
+    count: usize,
+    max: usize,
 }
 
-fn page_from_info(info: DanceInfo) -> Result<Wikicode, InfoCompileError> {
-    let mut params = IndexMap::new();
-    params.insert("name".to_string(), info.name());
-    params.insert(
-        "dominant_role".to_string(),
-        info.dominant_role.as_str().to_string(),
-    );
-    params.insert(
-        "allowed_rank".to_string(),
-        info.dominant_role_comp.rank.as_str().to_string(),
-    );
-    params.insert(
-        "dominant_rank".to_string(),
-        info.dominant_role_comp.rank.as_str().to_string(),
-    );
-    params.insert(
-        "dominant_points".to_string(),
-        info.dominant_role_comp.points.to_string(),
-    );
-    if let Some(u) = info.non_dominant_role_comp {
-        params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string());
-        params.insert("non_dominant_points".to_string(), u.points.to_string());
+impl Ticker {
+    pub const fn new(max: usize) -> Self {
+        Self { count: 0, max }
     }
-    let t = Template::new("Template:WSDCBox", &params)?;
-    let result = Wikicode::new("");
-    result.append(&t);
-    Ok(result)
-}
 
-pub async fn watch_wanted(bot: Bot) {
-    let span = tracing::span!(Level::INFO, "wanted_watchdog");
-    let _enter = span.enter();
-
-    let mut ignored_ids = vec![];
-    loop {
-        if ignored_ids.is_empty() {
-            tracing::info!("Watchdog check...");
+    /// Returns `true` if the ticker has "ticked" (i.e., reached `max` and reset).
+    pub const fn tick(&mut self) -> bool {
+        self.count += 1;
+        if self.count >= self.max {
+            self.count = 0;
+            true
         } else {
-            tracing::info!("Watchdog check [{} failed ids]...", ignored_ids.len());
-        }
-        let wanted = wanted_ids(bot.clone()).await;
-        let mut new_ignored = vec![];
-        for (id, page) in wanted.into_iter().filter(|(x, _)| ignored_ids.contains(x)) {
-            if !generate_page(id, page).await {
-                new_ignored.push(id);
-            }
-        }
-        if !new_ignored.is_empty() {
-            ignored_ids.extend(new_ignored);
-        }
-        tokio::time::sleep(Duration::from_secs(30)).await;
-    }
-}
-
-async fn generate_page(id: u32, page: mwbot::Page) -> bool {
-    tracing::info!("Taking care of {id}");
-    let info = match fetch_wsdc_info(id).await {
-        Ok(o) => o,
-        Err(e) => {
-            tracing::error!("Error fetching wsdc info for {id}: {e}");
-            return false;
-        }
-    };
-    let code = match page_from_info(info) {
-        Ok(o) => o,
-        Err(e) => {
-            tracing::error!("Creating wikicode for {id}: {e}");
-            return false;
-        }
-    };
-
-    match page
-        .save(
-            code,
-            &SaveOptions::summary("Created WSDC info from worldsdc.com")
-                .mark_as_bot(true)
-                .mark_as_minor(false),
-        )
-        .await
-    {
-        Ok(_) => true,
-        Err(e) => {
-            tracing::error!("Could not save page for {id}: {e}");
             false
         }
     }
 }
+
+/// Continuously monitors teacher IDs that do not have a corresponding teacher WSDC page, ignoring those that fail processing.
+#[tracing::instrument(skip_all)]
+pub async fn watch_wanted(bot: Bot) -> ! {
+    let mut ignored_ids = vec![];
+    let mut heartbeat_ticker = Ticker::new(120);
+    loop {
+        if heartbeat_ticker.tick() {
+            tracing::info!(failed_id_count = ignored_ids.len(), "Watchdog check...");
+        }
+        let wanted = wanted_ids(bot.clone()).await;
+        let new_ignored = update_wanted_ids(&wanted, &ignored_ids).await;
+        ignored_ids.extend(new_ignored);
+        tokio::time::sleep(Duration::from_secs(30)).await;
+    }
+}
+
+async fn update_wanted_ids(wanted: &[(u32, Page)], ignored_ids: &[u32]) -> Vec<u32> {
+    let mut new_ignored = vec![];
+
+    for (id, page) in wanted.iter().filter(|(x, _)| ignored_ids.contains(x)) {
+        let span = tracing::info_span!("update", id);
+        let _enter = span.enter();
+        if let Err(e) = generate_page(*id, page.clone()).await {
+            tracing::error!("{e}");
+            new_ignored.push(*id);
+        }
+    }
+    new_ignored
+}
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum GeneratePageError {
+    #[error("Error fetching WSDC info for {0}")]
+    Fetch(#[from] DanceInfoError),
+    #[error("Error creating wikicode for {0}")]
+    Wikicode(#[from] InfoCompileError),
+    #[error("Error saving page for {0}")]
+    Save(#[from] mwbot::Error),
+}
+
+pub async fn generate_page(id: u32, page: mwbot::Page) -> Result<(), GeneratePageError> {
+    tracing::info!("Generating page for {id}");
+    let info = fetch_wsdc_info(id).await?;
+
+    let code = page_from_info(info)?;
+
+    page.save(
+        code,
+        &SaveOptions::summary(&format!(
+            "Created WSDC info from worldsdc.com -- {}",
+            app_signature()
+        ))
+        .mark_as_bot(true)
+        .mark_as_minor(false),
+    )
+    .await?;
+
+    Ok(())
+}

src/wikiinfo.rs

@@ -42,8 +42,20 @@ fn parse_wsdc_page_name(name: &str) -> Result<u32, TitleParseError> {
     }
 }
 
+// fn get_wsdc_page_date(bot: &Bot, page: &Page) -> Option<SystemTime> {
+// todo!();
+// let prefix = "Updated-On: ";
+// page.filter_comments()
+//     .iter()
+//     .filter_map(|x| {
+//         let c = x.text_contents();
+//         if c.starts_with(prefix) { Some(c) } else { None }
+//     })
+//     .map(|x| x.trim_start_matches(prefix).parse::<u64>());
+// }
+
 #[allow(dead_code)]
-pub async fn index_wsdc_ids(bot: &Bot) -> Vec<u32> {
+pub async fn index_wsdc_ids(bot: &Bot) -> Vec<(u32, Page)> {
     let mut gene = Search::new("WSDC/").generate(bot);
     let mut result = vec![];
     while let Some(x) = gene.recv().await {
@@ -55,7 +67,7 @@ pub async fn index_wsdc_ids(bot: &Bot) -> Vec<u32> {
             }
         };
         if let Ok(n) = parse_wsdc_page_name(p.title()) {
-            result.push(n);
+            result.push((n, p));
         }
     }
     result

src/wikipage.rs

@@ -0,0 +1,38 @@
+use crate::dance_info::DanceInfo;
+use mwbot::parsoid::WikinodeIterator;
+use mwbot::parsoid::{self, Template, Wikicode, map::IndexMap};
+
+#[derive(thiserror::Error, Debug)]
+pub enum InfoCompileError {
+    #[error("Could not compile wikipage: {0}")]
+    CompileError(#[from] parsoid::Error),
+}
+
+pub fn page_from_info(info: DanceInfo) -> Result<Wikicode, InfoCompileError> {
+    let mut params = IndexMap::new();
+    params.insert("name".to_string(), info.name());
+    params.insert(
+        "dominant_role".to_string(),
+        info.dominant_role.as_str().to_string(),
+    );
+    params.insert(
+        "allowed_rank".to_string(),
+        info.dominant_role_comp.rank.as_str().to_string(),
+    );
+    params.insert(
+        "dominant_rank".to_string(),
+        info.dominant_role_comp.rank.as_str().to_string(),
+    );
+    params.insert(
+        "dominant_points".to_string(),
+        info.dominant_role_comp.points.to_string(),
+    );
+    if let Some(u) = info.non_dominant_role_comp {
+        params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string());
+        params.insert("non_dominant_points".to_string(), u.points.to_string());
+    }
+    let t = Template::new("Template:WSDCBox", &params)?;
+    let result = Wikicode::new("");
+    result.append(&t);
+    Ok(result)
+}

src/worldsdc/caching.rs

@@ -0,0 +1,48 @@
+use std::{collections::HashMap, path::Path};
+
+use reqwest::{Client, ClientBuilder};
+
+use crate::{dance_info::DanceInfo, worldsdc::DanceInfoError};
+
+use super::DanceInfoParser;
+
+struct CachingFetcher {
+    hitcache: Vec<(u32, String)>,
+    errorcache: Vec<(u32, String)>,
+    client: Client,
+}
+
+#[derive(thiserror::Error, Debug)]
+enum CachingFetcherCreationError {
+    #[error("Could not create client: {0}")]
+    ClientError(#[from] reqwest::Error),
+}
+
+impl CachingFetcher {
+    pub fn new(cachepath: &Path) -> Result<Self, CachingFetcherCreationError> {
+        let client = ClientBuilder::new().build()?;
+
+        Ok(Self {
+            hitcache: vec![],
+            errorcache: vec![],
+            client,
+        })
+    }
+    pub async fn fetch(&mut self, id: u32) -> Result<DanceInfo, DanceInfoError> {
+        let mut params = HashMap::new();
+        params.insert("q", id.to_string());
+        let response = self
+            .client
+            .request(
+                reqwest::Method::POST,
+                "https://points.worldsdc.com/lookup2020/find",
+            )
+            .form(&params)
+            .send()
+            .await
+            .map_err(DanceInfoError::Request)?;
+
+        let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
+        Ok(x.into())
+    }
+}

src/worldsdc/mod.rs

@@ -0,0 +1,126 @@
+use std::collections::HashMap;
+
+use reqwest::ClientBuilder;
+
+use crate::{
+    app_signature,
+    dance_info::{CompState, DanceInfo, DanceRank, DanceRole},
+};
+
+// mod caching;
+pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
+    let client = ClientBuilder::new()
+        .user_agent(app_signature())
+        .build()
+        .map_err(DanceInfoError::ClientBuild)?;
+
+    let mut params = HashMap::new();
+
+    let url = if cfg!(test) {
+        // "https://o5grQU3Y.free.beeceptor.com/lookup2020/find"
+        "http://localhost:8000"
+    } else {
+        "https://points.worldsdc.com/lookup2020/find"
+    };
+    params.insert("num", id.to_string());
+    let request = client
+        .request(reqwest::Method::POST, url)
+        .form(&params)
+        .build()
+        .map_err(DanceInfoError::RequestBuild)?;
+    let response = client
+        .execute(request)
+        .await
+        .map_err(DanceInfoError::Request)?;
+
+    let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
+    Ok(x.into())
+}
+
+#[cfg(test)]
+mod tests {
+    #![allow(clippy::unwrap_used, reason = "Allow unwrap in tests")]
+    use crate::worldsdc::fetch_wsdc_info;
+
+    #[test]
+    #[ignore = "Only run when the mock api is setup"]
+    fn test_fetch_wsdc() {
+        let rt = match tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+        {
+            Ok(o) => o,
+            Err(e) => {
+                tracing::error!("Could not start runtime: {e}");
+                return;
+            }
+        };
+        let x = rt.block_on(fetch_wsdc_info(7));
+        dbg!(&x);
+        x.unwrap();
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum DanceInfoError {
+    #[error("Failed to build client: {0}")]
+    ClientBuild(reqwest::Error),
+    #[error("Failed to build request: {0}")]
+    RequestBuild(reqwest::Error),
+    #[error("Request error: {0}")]
+    Request(reqwest::Error),
+    #[error("Failed to parse response: {0}")]
+    JsonParse(reqwest::Error),
+}
+
+#[derive(serde::Deserialize, Debug)]
+enum OptionalDanceRank {
+    #[serde(rename = "N/A")]
+    NotAvailable,
+    #[serde(untagged)]
+    Rank(DanceRank),
+}
+
+#[derive(serde::Deserialize, Debug)]
+enum OptionalDancePoints {
+    #[serde(rename = "N/A")]
+    NotAvailable,
+    #[serde(untagged)]
+    Points(u16),
+}
+
+#[derive(serde::Deserialize, Debug)]
+struct DanceInfoParser {
+    pub dancer_first: String,
+    pub dancer_last: String,
+    pub short_dominate_role: DanceRole,
+    #[allow(dead_code)]
+    pub short_non_dominate_role: DanceRole,
+    pub dominate_role_highest_level_points: u16,
+    pub dominate_role_highest_level: DanceRank,
+    pub non_dominate_role_highest_level_points: OptionalDancePoints,
+    pub non_dominate_role_highest_level: OptionalDanceRank,
+}
+
+impl From<DanceInfoParser> for DanceInfo {
+    fn from(value: DanceInfoParser) -> Self {
+        let non_dominant_role_comp = if let OptionalDanceRank::Rank(r) =
+            value.non_dominate_role_highest_level
+            && let OptionalDancePoints::Points(l) = value.non_dominate_role_highest_level_points
+        {
+            Some(CompState { rank: r, points: l })
+        } else {
+            None
+        };
+        Self {
+            firstname: value.dancer_first,
+            lastname: value.dancer_last,
+            dominant_role: value.short_dominate_role,
+            dominant_role_comp: CompState {
+                rank: value.dominate_role_highest_level,
+                points: value.dominate_role_highest_level_points,
+            },
+            non_dominant_role_comp,
+        }
+    }
+}