diff --git a/Cargo.lock b/Cargo.lock index 49726e4..b5b94c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,7 +279,9 @@ name = "dancing-bot-teachers" version = "0.1.0" dependencies = [ "chrono", + "futures", "mwbot", + "rand 0.9.2", "reqwest", "serde", "thiserror 2.0.12", @@ -497,7 +499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -571,6 +573,21 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -578,6 +595,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -586,6 +604,34 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -604,10 +650,16 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1751,7 +1803,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2037,7 +2089,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2420,7 +2472,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2992,7 +3044,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6dfc860..ed3209c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,9 @@ categories = ["web-programming", "api-bindings", "automation"] [dependencies] chrono = "0.4.41" +futures = "0.3.31" mwbot = { git = "https://gitlab.wikimedia.org/repos/mwbot-rs/mwbot.git", rev = "05cbb12188f18e2da710de158d89a9a4f1b42689", default-features = false, features = ["generators", "mwbot_derive"] } +rand = "0.9.2" reqwest = "0.12.22" serde = { version = "1.0.219", features = ["derive"] } thiserror = "2.0.12" diff --git a/src/main.rs b/src/main.rs index 1ce1911..ee3ddd6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,9 +22,10 @@ use std::{error::Error, path::Path}; use crate::watchdog::watch_wanted; mod dance_info; +mod updater; mod watchdog; mod wikiinfo; -mod task_schedule; +mod wikipage; mod worldsdc; #[allow(dead_code)] @@ -40,7 +41,7 @@ fn list_teacher_pages(bot: &Bot) -> tokio::sync::mpsc::Receiver Result<(), Box> { +fn main() { tracing_subscriber::fmt() .with_level(true) .with_max_level(tracing::Level::INFO) @@ -53,7 +54,7 @@ fn main() -> Result<(), Box> { Ok(o) => o, Err(e) => { tracing::error!("Could not start runtime: {e}"); - return Ok(()); + return; } }; rt.block_on(async { @@ -61,10 +62,9 @@ fn main() -> Result<(), Box> { Ok(x) => x, Err(e) => { dbg!(e); - return Ok(()); + return; } }; - watch_wanted(bot).await; - Ok(()) - }) + futures::join!(watch_wanted(bot.clone()), updater::update_wsdc(bot)); + }); } diff --git a/src/task_schedule.rs b/src/task_schedule.rs deleted file mode 100644 index 6511369..0000000 --- a/src/task_schedule.rs +++ /dev/null @@ -1,11 +0,0 @@ -#[derive(Default, Debug)] -struct UpdateQueue { - exists: Vec<(u32, std::time::SystemTime)>, - create: Vec, -} - -impl UpdateQueue { - pub fn add_exist(&mut self, wsdc_id: u32, update_on: std::time::SystemTime) {} - pub fn add_create(&mut self, wsdc_id: u32) {} - pub async fn run(&mut self) {} -} diff --git a/src/updater.rs b/src/updater.rs new file mode 100644 index 0000000..6ae0dc4 --- /dev/null +++ b/src/updater.rs @@ -0,0 +1,24 @@ +use std::time::Duration; + +use mwbot::Bot; +use rand::seq::SliceRandom as _; + +use crate::{watchdog::generate_page, wikiinfo::index_wsdc_ids}; + +pub async fn update_wsdc(bot: Bot) -> ! { + loop { + let mut l = index_wsdc_ids(&bot).await; + l.shuffle(&mut rand::rng()); + tracing::info!("We have to update {} pages", l.len()); + let wait_duration = Duration::from_secs(6 * 3600); + for (index, page) in l { + tracing::info!("Next up: {index}"); + tokio::time::sleep(wait_duration).await; + if generate_page(index, page).await { + tracing::info!("Updated {index}"); + } else { + tracing::error!("Error updating {index}"); + } + } + } +} diff --git a/src/watchdog.rs b/src/watchdog.rs index dc4632f..922f36e 100644 --- a/src/watchdog.rs +++ b/src/watchdog.rs @@ -1,49 +1,11 @@ use std::time::Duration; -use crate::{dance_info::DanceInfo, wikiinfo::wanted_ids, worldsdc::fetch_wsdc_info}; -use mwbot::{ - Bot, - parsoid::{self, Template, Wikicode, map::IndexMap}, -}; -use mwbot::{SaveOptions, parsoid::WikinodeIterator}; +use crate::{wikiinfo::wanted_ids, wikipage::page_from_info, worldsdc::fetch_wsdc_info}; +use mwbot::Bot; +use mwbot::SaveOptions; use tracing::Level; -#[derive(thiserror::Error, Debug)] -enum InfoCompileError { - #[error("Could not compile wikipage: {0}")] - CompileError(#[from] parsoid::Error), -} - -fn page_from_info(info: DanceInfo) -> Result { - let mut params = IndexMap::new(); - params.insert("name".to_string(), info.name()); - params.insert( - "dominant_role".to_string(), - info.dominant_role.as_str().to_string(), - ); - params.insert( - "allowed_rank".to_string(), - info.dominant_role_comp.rank.as_str().to_string(), - ); - params.insert( - "dominant_rank".to_string(), - info.dominant_role_comp.rank.as_str().to_string(), - ); - params.insert( - "dominant_points".to_string(), - info.dominant_role_comp.points.to_string(), - ); - if let Some(u) = info.non_dominant_role_comp { - params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string()); - params.insert("non_dominant_points".to_string(), u.points.to_string()); - } - let t = Template::new("Template:WSDCBox", ¶ms)?; - let result = Wikicode::new(""); - result.append(&t); - Ok(result) -} - -pub async fn watch_wanted(bot: Bot) { +pub async fn watch_wanted(bot: Bot) -> ! { let span = tracing::span!(Level::INFO, "wanted_watchdog"); let _enter = span.enter(); @@ -68,8 +30,8 @@ pub async fn watch_wanted(bot: Bot) { } } -async fn generate_page(id: u32, page: mwbot::Page) -> bool { - tracing::info!("Taking care of {id}"); +pub async fn generate_page(id: u32, page: mwbot::Page) -> bool { + tracing::info!("Generating page for {id}"); let info = match fetch_wsdc_info(id).await { Ok(o) => o, Err(e) => { diff --git a/src/wikiinfo.rs b/src/wikiinfo.rs index d68a666..f1ce260 100644 --- a/src/wikiinfo.rs +++ b/src/wikiinfo.rs @@ -1,9 +1,6 @@ -use std::time::SystemTime; - use mwbot::{ Bot, Page, generators::{Generator, querypage::QueryPage, search::Search}, - parsoid::{Wikicode, WikinodeIterator}, }; pub async fn wanted_ids(bot: Bot) -> Vec<(u32, Page)> { @@ -45,15 +42,16 @@ fn parse_wsdc_page_name(name: &str) -> Result { } } -// fn get_wsdc_page_date(page: &Wikicode) -> Option { -// let prefix = "Updated-On: "; -// page.filter_comments() -// .iter() -// .filter_map(|x| { -// let c = x.text_contents(); -// if c.starts_with(prefix) { Some(c) } else { None } -// }) -// .map(|x| x.trim_start_matches(prefix).parse::()); +// fn get_wsdc_page_date(bot: &Bot, page: &Page) -> Option { +// todo!(); +// let prefix = "Updated-On: "; +// page.filter_comments() +// .iter() +// .filter_map(|x| { +// let c = x.text_contents(); +// if c.starts_with(prefix) { Some(c) } else { None } +// }) +// .map(|x| x.trim_start_matches(prefix).parse::()); // } #[allow(dead_code)] diff --git a/src/wikipage.rs b/src/wikipage.rs new file mode 100644 index 0000000..af0e756 --- /dev/null +++ b/src/wikipage.rs @@ -0,0 +1,38 @@ +use crate::dance_info::DanceInfo; +use mwbot::parsoid::WikinodeIterator; +use mwbot::parsoid::{self, Template, Wikicode, map::IndexMap}; + +#[derive(thiserror::Error, Debug)] +pub enum InfoCompileError { + #[error("Could not compile wikipage: {0}")] + CompileError(#[from] parsoid::Error), +} + +pub fn page_from_info(info: DanceInfo) -> Result { + let mut params = IndexMap::new(); + params.insert("name".to_string(), info.name()); + params.insert( + "dominant_role".to_string(), + info.dominant_role.as_str().to_string(), + ); + params.insert( + "allowed_rank".to_string(), + info.dominant_role_comp.rank.as_str().to_string(), + ); + params.insert( + "dominant_rank".to_string(), + info.dominant_role_comp.rank.as_str().to_string(), + ); + params.insert( + "dominant_points".to_string(), + info.dominant_role_comp.points.to_string(), + ); + if let Some(u) = info.non_dominant_role_comp { + params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string()); + params.insert("non_dominant_points".to_string(), u.points.to_string()); + } + let t = Template::new("Template:WSDCBox", ¶ms)?; + let result = Wikicode::new(""); + result.append(&t); + Ok(result) +} diff --git a/src/worldsdc/caching.rs b/src/worldsdc/caching.rs new file mode 100644 index 0000000..15ef5e5 --- /dev/null +++ b/src/worldsdc/caching.rs @@ -0,0 +1,48 @@ +use std::{collections::HashMap, path::Path}; + +use reqwest::{Client, ClientBuilder}; + +use crate::{dance_info::DanceInfo, worldsdc::DanceInfoError}; + +use super::DanceInfoParser; + +struct CachingFetcher { + hitcache: Vec<(u32, String)>, + errorcache: Vec<(u32, String)>, + client: Client, +} + +#[derive(thiserror::Error, Debug)] +enum CachingFetcherCreationError { + #[error("Could not create client: {0}")] + ClientError(#[from] reqwest::Error), +} + +impl CachingFetcher { + pub fn new(cachepath: &Path) -> Result { + let client = ClientBuilder::new().build()?; + + Ok(Self { + hitcache: vec![], + errorcache: vec![], + client, + }) + } + pub async fn fetch(&mut self, id: u32) -> Result { + let mut params = HashMap::new(); + params.insert("q", id.to_string()); + let response = self + .client + .request( + reqwest::Method::POST, + "https://points.worldsdc.com/lookup2020/find", + ) + .form(¶ms) + .send() + .await + .map_err(DanceInfoError::Request)?; + + let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?; + Ok(x.into()) + } +} diff --git a/src/worldsdc/mod.rs b/src/worldsdc/mod.rs index b5f1691..3631bb9 100644 --- a/src/worldsdc/mod.rs +++ b/src/worldsdc/mod.rs @@ -1,9 +1,10 @@ -use std::{collections::HashMap, path::Path}; +use std::collections::HashMap; -use reqwest::{Client, ClientBuilder}; +use reqwest::ClientBuilder; use crate::dance_info::{CompState, DanceInfo, DanceRank, DanceRole}; +// mod caching; pub async fn fetch_wsdc_info(id: u32) -> Result { let client = ClientBuilder::new() .build() @@ -25,47 +26,6 @@ pub async fn fetch_wsdc_info(id: u32) -> Result { Ok(x.into()) } -struct CachingFetcher { - hitcache: Vec<(u32, String)>, - errorcache: Vec<(u32, String)>, - client: Client, -} - -#[derive(thiserror::Error, Debug)] -enum CachingFetcherCreationError { - #[error("Could not create client: {0}")] - ClientError(#[from] reqwest::Error), -} - -impl CachingFetcher { - pub fn new(cachepath: &Path) -> Result { - let client = ClientBuilder::new().build()?; - - Ok(Self { - hitcache: vec![], - errorcache: vec![], - client, - }) - } - pub async fn fetch(&mut self, id: u32) -> Result { - let mut params = HashMap::new(); - params.insert("q", id.to_string()); - let response = self - .client - .request( - reqwest::Method::POST, - "https://points.worldsdc.com/lookup2020/find", - ) - .form(¶ms) - .send() - .await - .map_err(DanceInfoError::Request)?; - - let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?; - Ok(x.into()) - } -} - #[derive(thiserror::Error, Debug)] pub enum DanceInfoError { #[error("Failed to build client: {0}")]