First working version of creation and update watchdog

Cargo.lock

@@ -279,7 +279,9 @@ name = "dancing-bot-teachers"
 version = "0.1.0"
 dependencies = [
  "chrono",
+ "futures",
  "mwbot",
+ "rand 0.9.2",
  "reqwest",
  "serde",
  "thiserror 2.0.12",
@@ -497,7 +499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -571,6 +573,21 @@ dependencies = [
  "new_debug_unreachable",
 ]
 
+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.31"
@@ -578,6 +595,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
 dependencies = [
  "futures-core",
+ "futures-sink",
 ]
 
 [[package]]
@@ -586,6 +604,34 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
+[[package]]
+name = "futures-executor"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.31"
@@ -604,10 +650,16 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
+ "futures-channel",
  "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
  "futures-task",
+ "memchr",
  "pin-project-lite",
  "pin-utils",
+ "slab",
 ]
 
 [[package]]
@@ -1751,7 +1803,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2037,7 +2089,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2420,7 +2472,7 @@ dependencies = [
  "getrandom 0.3.3",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2992,7 +3044,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]

Cargo.toml

@@ -12,7 +12,9 @@ categories = ["web-programming", "api-bindings", "automation"]
 
 [dependencies]
 chrono = "0.4.41"
+futures = "0.3.31"
 mwbot = { git = "https://gitlab.wikimedia.org/repos/mwbot-rs/mwbot.git", rev = "05cbb12188f18e2da710de158d89a9a4f1b42689", default-features = false, features = ["generators", "mwbot_derive"] }
+rand = "0.9.2"
 reqwest = "0.12.22"
 serde = { version = "1.0.219", features = ["derive"] }
 thiserror = "2.0.12"

src/main.rs

@@ -22,9 +22,10 @@ use std::{error::Error, path::Path};
 use crate::watchdog::watch_wanted;
 
 mod dance_info;
+mod updater;
 mod watchdog;
 mod wikiinfo;
-mod task_schedule;
+mod wikipage;
 mod worldsdc;
 
 #[allow(dead_code)]
@@ -40,7 +41,7 @@ fn list_teacher_pages(bot: &Bot) -> tokio::sync::mpsc::Receiver<Result<mwbot::Pa
     pages.generate(bot)
 }
 
-fn main() -> Result<(), Box<dyn Error>> {
+fn main() {
     tracing_subscriber::fmt()
         .with_level(true)
         .with_max_level(tracing::Level::INFO)
@@ -53,7 +54,7 @@ fn main() -> Result<(), Box<dyn Error>> {
         Ok(o) => o,
         Err(e) => {
             tracing::error!("Could not start runtime: {e}");
-            return Ok(());
+            return;
         }
     };
     rt.block_on(async {
@@ -61,10 +62,9 @@ fn main() -> Result<(), Box<dyn Error>> {
             Ok(x) => x,
             Err(e) => {
                 dbg!(e);
-                return Ok(());
+                return;
             }
         };
-        watch_wanted(bot).await;
+        futures::join!(watch_wanted(bot.clone()), updater::update_wsdc(bot));
-        Ok(())
+    });
-    })
 }

src/task_schedule.rs

@@ -1,11 +0,0 @@
-#[derive(Default, Debug)]
-struct UpdateQueue {
-    exists: Vec<(u32, std::time::SystemTime)>,
-    create: Vec<u32>,
-}
-
-impl UpdateQueue {
-    pub fn add_exist(&mut self, wsdc_id: u32, update_on: std::time::SystemTime) {}
-    pub fn add_create(&mut self, wsdc_id: u32) {}
-    pub async fn run(&mut self) {}
-}

src/updater.rs

@@ -0,0 +1,24 @@
+use std::time::Duration;
+
+use mwbot::Bot;
+use rand::seq::SliceRandom as _;
+
+use crate::{watchdog::generate_page, wikiinfo::index_wsdc_ids};
+
+pub async fn update_wsdc(bot: Bot) -> ! {
+    loop {
+        let mut l = index_wsdc_ids(&bot).await;
+        l.shuffle(&mut rand::rng());
+        tracing::info!("We have to update {} pages", l.len());
+        let wait_duration = Duration::from_secs(6 * 3600);
+        for (index, page) in l {
+            tracing::info!("Next up: {index}");
+            tokio::time::sleep(wait_duration).await;
+            if generate_page(index, page).await {
+                tracing::info!("Updated {index}");
+            } else {
+                tracing::error!("Error updating {index}");
+            }
+        }
+    }
+}

src/watchdog.rs

@@ -1,49 +1,11 @@
 use std::time::Duration;
 
-use crate::{dance_info::DanceInfo, wikiinfo::wanted_ids, worldsdc::fetch_wsdc_info};
+use crate::{wikiinfo::wanted_ids, wikipage::page_from_info, worldsdc::fetch_wsdc_info};
-use mwbot::{
+use mwbot::Bot;
-    Bot,
+use mwbot::SaveOptions;
-    parsoid::{self, Template, Wikicode, map::IndexMap},
-};
-use mwbot::{SaveOptions, parsoid::WikinodeIterator};
 use tracing::Level;
 
-#[derive(thiserror::Error, Debug)]
+pub async fn watch_wanted(bot: Bot) -> ! {
-enum InfoCompileError {
-    #[error("Could not compile wikipage: {0}")]
-    CompileError(#[from] parsoid::Error),
-}
-
-fn page_from_info(info: DanceInfo) -> Result<Wikicode, InfoCompileError> {
-    let mut params = IndexMap::new();
-    params.insert("name".to_string(), info.name());
-    params.insert(
-        "dominant_role".to_string(),
-        info.dominant_role.as_str().to_string(),
-    );
-    params.insert(
-        "allowed_rank".to_string(),
-        info.dominant_role_comp.rank.as_str().to_string(),
-    );
-    params.insert(
-        "dominant_rank".to_string(),
-        info.dominant_role_comp.rank.as_str().to_string(),
-    );
-    params.insert(
-        "dominant_points".to_string(),
-        info.dominant_role_comp.points.to_string(),
-    );
-    if let Some(u) = info.non_dominant_role_comp {
-        params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string());
-        params.insert("non_dominant_points".to_string(), u.points.to_string());
-    }
-    let t = Template::new("Template:WSDCBox", &params)?;
-    let result = Wikicode::new("");
-    result.append(&t);
-    Ok(result)
-}
-
-pub async fn watch_wanted(bot: Bot) {
     let span = tracing::span!(Level::INFO, "wanted_watchdog");
     let _enter = span.enter();
 
@@ -68,8 +30,8 @@ pub async fn watch_wanted(bot: Bot) {
     }
 }
 
-async fn generate_page(id: u32, page: mwbot::Page) -> bool {
+pub async fn generate_page(id: u32, page: mwbot::Page) -> bool {
-    tracing::info!("Taking care of {id}");
+    tracing::info!("Generating page for {id}");
     let info = match fetch_wsdc_info(id).await {
         Ok(o) => o,
         Err(e) => {

src/wikiinfo.rs

@@ -1,9 +1,6 @@
-use std::time::SystemTime;
-
 use mwbot::{
     Bot, Page,
     generators::{Generator, querypage::QueryPage, search::Search},
-    parsoid::{Wikicode, WikinodeIterator},
 };
 
 pub async fn wanted_ids(bot: Bot) -> Vec<(u32, Page)> {
@@ -45,15 +42,16 @@ fn parse_wsdc_page_name(name: &str) -> Result<u32, TitleParseError> {
     }
 }
 
-// fn get_wsdc_page_date(page: &Wikicode) -> Option<SystemTime> {
+// fn get_wsdc_page_date(bot: &Bot, page: &Page) -> Option<SystemTime> {
-//     let prefix = "Updated-On: ";
+// todo!();
-//     page.filter_comments()
+// let prefix = "Updated-On: ";
-//         .iter()
+// page.filter_comments()
-//         .filter_map(|x| {
+//     .iter()
-//             let c = x.text_contents();
+//     .filter_map(|x| {
-//             if c.starts_with(prefix) { Some(c) } else { None }
+//         let c = x.text_contents();
-//         })
+//         if c.starts_with(prefix) { Some(c) } else { None }
-//         .map(|x| x.trim_start_matches(prefix).parse::<u64>());
+//     })
+//     .map(|x| x.trim_start_matches(prefix).parse::<u64>());
 // }
 
 #[allow(dead_code)]

src/wikipage.rs

@@ -0,0 +1,38 @@
+use crate::dance_info::DanceInfo;
+use mwbot::parsoid::WikinodeIterator;
+use mwbot::parsoid::{self, Template, Wikicode, map::IndexMap};
+
+#[derive(thiserror::Error, Debug)]
+pub enum InfoCompileError {
+    #[error("Could not compile wikipage: {0}")]
+    CompileError(#[from] parsoid::Error),
+}
+
+pub fn page_from_info(info: DanceInfo) -> Result<Wikicode, InfoCompileError> {
+    let mut params = IndexMap::new();
+    params.insert("name".to_string(), info.name());
+    params.insert(
+        "dominant_role".to_string(),
+        info.dominant_role.as_str().to_string(),
+    );
+    params.insert(
+        "allowed_rank".to_string(),
+        info.dominant_role_comp.rank.as_str().to_string(),
+    );
+    params.insert(
+        "dominant_rank".to_string(),
+        info.dominant_role_comp.rank.as_str().to_string(),
+    );
+    params.insert(
+        "dominant_points".to_string(),
+        info.dominant_role_comp.points.to_string(),
+    );
+    if let Some(u) = info.non_dominant_role_comp {
+        params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string());
+        params.insert("non_dominant_points".to_string(), u.points.to_string());
+    }
+    let t = Template::new("Template:WSDCBox", &params)?;
+    let result = Wikicode::new("");
+    result.append(&t);
+    Ok(result)
+}

src/worldsdc/caching.rs

@@ -0,0 +1,48 @@
+use std::{collections::HashMap, path::Path};
+
+use reqwest::{Client, ClientBuilder};
+
+use crate::{dance_info::DanceInfo, worldsdc::DanceInfoError};
+
+use super::DanceInfoParser;
+
+struct CachingFetcher {
+    hitcache: Vec<(u32, String)>,
+    errorcache: Vec<(u32, String)>,
+    client: Client,
+}
+
+#[derive(thiserror::Error, Debug)]
+enum CachingFetcherCreationError {
+    #[error("Could not create client: {0}")]
+    ClientError(#[from] reqwest::Error),
+}
+
+impl CachingFetcher {
+    pub fn new(cachepath: &Path) -> Result<Self, CachingFetcherCreationError> {
+        let client = ClientBuilder::new().build()?;
+
+        Ok(Self {
+            hitcache: vec![],
+            errorcache: vec![],
+            client,
+        })
+    }
+    pub async fn fetch(&mut self, id: u32) -> Result<DanceInfo, DanceInfoError> {
+        let mut params = HashMap::new();
+        params.insert("q", id.to_string());
+        let response = self
+            .client
+            .request(
+                reqwest::Method::POST,
+                "https://points.worldsdc.com/lookup2020/find",
+            )
+            .form(&params)
+            .send()
+            .await
+            .map_err(DanceInfoError::Request)?;
+
+        let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
+        Ok(x.into())
+    }
+}

src/worldsdc/mod.rs

@@ -1,9 +1,10 @@
-use std::{collections::HashMap, path::Path};
+use std::collections::HashMap;
 
-use reqwest::{Client, ClientBuilder};
+use reqwest::ClientBuilder;
 
 use crate::dance_info::{CompState, DanceInfo, DanceRank, DanceRole};
 
+// mod caching;
 pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
     let client = ClientBuilder::new()
         .build()
@@ -25,47 +26,6 @@ pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
     Ok(x.into())
 }
 
-struct CachingFetcher {
-    hitcache: Vec<(u32, String)>,
-    errorcache: Vec<(u32, String)>,
-    client: Client,
-}
-
-#[derive(thiserror::Error, Debug)]
-enum CachingFetcherCreationError {
-    #[error("Could not create client: {0}")]
-    ClientError(#[from] reqwest::Error),
-}
-
-impl CachingFetcher {
-    pub fn new(cachepath: &Path) -> Result<Self, CachingFetcherCreationError> {
-        let client = ClientBuilder::new().build()?;
-
-        Ok(Self {
-            hitcache: vec![],
-            errorcache: vec![],
-            client,
-        })
-    }
-    pub async fn fetch(&mut self, id: u32) -> Result<DanceInfo, DanceInfoError> {
-        let mut params = HashMap::new();
-        params.insert("q", id.to_string());
-        let response = self
-            .client
-            .request(
-                reqwest::Method::POST,
-                "https://points.worldsdc.com/lookup2020/find",
-            )
-            .form(&params)
-            .send()
-            .await
-            .map_err(DanceInfoError::Request)?;
-
-        let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
-        Ok(x.into())
-    }
-}
-
 #[derive(thiserror::Error, Debug)]
 pub enum DanceInfoError {
     #[error("Failed to build client: {0}")]