lukas/dancing-teacher-info
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First working version of creation and update watchdog

Browse Source
This commit is contained in:
Lukas Wölfer
2025-07-24 19:02:10 +02:00
parent 29e85397a7
commit 8172d4c769
10 changed files with 195 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
Cargo.lock generated
View File

@@ -279,7 +279,9 @@ name = "dancing-bot-teachers"
version = "0.1.0"
dependencies = [
"chrono",
"futures",
"mwbot",
"rand 0.9.2",
"reqwest",
"serde",
"thiserror 2.0.12",
@@ -497,7 +499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -571,6 +573,21 @@ dependencies = [
"new_debug_unreachable",
]
[[package]]
name = "futures"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.31"
@@ -578,6 +595,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
@@ -586,6 +604,34 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
[[package]]
name = "futures-executor"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-macro"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
]
[[package]]
name = "futures-sink"
version = "0.3.31"
@@ -604,10 +650,16 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
@@ -1751,7 +1803,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -2037,7 +2089,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -2420,7 +2472,7 @@ dependencies = [
"getrandom 0.3.3",
"once_cell",
"rustix",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -2992,7 +3044,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]

2
Cargo.toml
View File

@@ -12,7 +12,9 @@ categories = ["web-programming", "api-bindings", "automation"]
[dependencies]
chrono = "0.4.41"
futures = "0.3.31"
mwbot = { git = "https://gitlab.wikimedia.org/repos/mwbot-rs/mwbot.git", rev = "05cbb12188f18e2da710de158d89a9a4f1b42689", default-features = false, features = ["generators", "mwbot_derive"] }
rand = "0.9.2"
reqwest = "0.12.22"
serde = { version = "1.0.219", features = ["derive"] }
thiserror = "2.0.12"

14
src/main.rs
View File

@@ -22,9 +22,10 @@ use std::{error::Error, path::Path};
use crate::watchdog::watch_wanted;
mod dance_info;
mod updater;
mod watchdog;
mod wikiinfo;
mod task_schedule;
mod wikipage;
mod worldsdc;
#[allow(dead_code)]
@@ -40,7 +41,7 @@ fn list_teacher_pages(bot: &Bot) -> tokio::sync::mpsc::Receiver<Result<mwbot::Pa
pages.generate(bot)
}
fn main() -> Result<(), Box<dyn Error>> {
fn main() {
tracing_subscriber::fmt()
.with_level(true)
.with_max_level(tracing::Level::INFO)
@@ -53,7 +54,7 @@ fn main() -> Result<(), Box<dyn Error>> {
Ok(o) => o,
Err(e) => {
tracing::error!("Could not start runtime: {e}");
return Ok(());
return;
}
};
rt.block_on(async {
@@ -61,10 +62,9 @@ fn main() -> Result<(), Box<dyn Error>> {
Ok(x) => x,
Err(e) => {
dbg!(e);
return Ok(());
return;
}
};
watch_wanted(bot).await;
Ok(())
})
futures::join!(watch_wanted(bot.clone()), updater::update_wsdc(bot));
});
}

11
src/task_schedule.rs
View File

@@ -1,11 +0,0 @@
#[derive(Default, Debug)]
struct UpdateQueue {
exists: Vec<(u32, std::time::SystemTime)>,
create: Vec<u32>,
}
impl UpdateQueue {
pub fn add_exist(&mut self, wsdc_id: u32, update_on: std::time::SystemTime) {}
pub fn add_create(&mut self, wsdc_id: u32) {}
pub async fn run(&mut self) {}
}

24
src/updater.rs Normal file
View File

@@ -0,0 +1,24 @@
use std::time::Duration;
use mwbot::Bot;
use rand::seq::SliceRandom as _;
use crate::{watchdog::generate_page, wikiinfo::index_wsdc_ids};
pub async fn update_wsdc(bot: Bot) -> ! {
loop {
let mut l = index_wsdc_ids(&bot).await;
l.shuffle(&mut rand::rng());
tracing::info!("We have to update {} pages", l.len());
let wait_duration = Duration::from_secs(6 * 3600);
for (index, page) in l {
tracing::info!("Next up: {index}");
tokio::time::sleep(wait_duration).await;
if generate_page(index, page).await {
tracing::info!("Updated {index}");
} else {
tracing::error!("Error updating {index}");
}
}
}
}

50
src/watchdog.rs
View File

@@ -1,49 +1,11 @@
use std::time::Duration;
use crate::{dance_info::DanceInfo, wikiinfo::wanted_ids, worldsdc::fetch_wsdc_info};
use mwbot::{
Bot,
parsoid::{self, Template, Wikicode, map::IndexMap},
};
use mwbot::{SaveOptions, parsoid::WikinodeIterator};
use crate::{wikiinfo::wanted_ids, wikipage::page_from_info, worldsdc::fetch_wsdc_info};
use mwbot::Bot;
use mwbot::SaveOptions;
use tracing::Level;
#[derive(thiserror::Error, Debug)]
enum InfoCompileError {
#[error("Could not compile wikipage: {0}")]
CompileError(#[from] parsoid::Error),
}
fn page_from_info(info: DanceInfo) -> Result<Wikicode, InfoCompileError> {
let mut params = IndexMap::new();
params.insert("name".to_string(), info.name());
params.insert(
"dominant_role".to_string(),
info.dominant_role.as_str().to_string(),
);
params.insert(
"allowed_rank".to_string(),
info.dominant_role_comp.rank.as_str().to_string(),
);
params.insert(
"dominant_rank".to_string(),
info.dominant_role_comp.rank.as_str().to_string(),
);
params.insert(
"dominant_points".to_string(),
info.dominant_role_comp.points.to_string(),
);
if let Some(u) = info.non_dominant_role_comp {
params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string());
params.insert("non_dominant_points".to_string(), u.points.to_string());
}
let t = Template::new("Template:WSDCBox", &params)?;
let result = Wikicode::new("");
result.append(&t);
Ok(result)
}
pub async fn watch_wanted(bot: Bot) {
pub async fn watch_wanted(bot: Bot) -> ! {
let span = tracing::span!(Level::INFO, "wanted_watchdog");
let _enter = span.enter();
@@ -68,8 +30,8 @@ pub async fn watch_wanted(bot: Bot) {
}
}
async fn generate_page(id: u32, page: mwbot::Page) -> bool {
tracing::info!("Taking care of {id}");
pub async fn generate_page(id: u32, page: mwbot::Page) -> bool {
tracing::info!("Generating page for {id}");
let info = match fetch_wsdc_info(id).await {
Ok(o) => o,
Err(e) => {

22
src/wikiinfo.rs
View File

@@ -1,9 +1,6 @@
use std::time::SystemTime;
use mwbot::{
Bot, Page,
generators::{Generator, querypage::QueryPage, search::Search},
parsoid::{Wikicode, WikinodeIterator},
};
pub async fn wanted_ids(bot: Bot) -> Vec<(u32, Page)> {
@@ -45,15 +42,16 @@ fn parse_wsdc_page_name(name: &str) -> Result<u32, TitleParseError> {
}
}
// fn get_wsdc_page_date(page: &Wikicode) -> Option<SystemTime> {
// let prefix = "Updated-On: ";
// page.filter_comments()
// .iter()
// .filter_map(|x| {
// let c = x.text_contents();
// if c.starts_with(prefix) { Some(c) } else { None }
// })
// .map(|x| x.trim_start_matches(prefix).parse::<u64>());
// fn get_wsdc_page_date(bot: &Bot, page: &Page) -> Option<SystemTime> {
// todo!();
// let prefix = "Updated-On: ";
// page.filter_comments()
// .iter()
// .filter_map(|x| {
// let c = x.text_contents();
// if c.starts_with(prefix) { Some(c) } else { None }
// })
// .map(|x| x.trim_start_matches(prefix).parse::<u64>());
// }
#[allow(dead_code)]

38
src/wikipage.rs Normal file
View File

@@ -0,0 +1,38 @@
use crate::dance_info::DanceInfo;
use mwbot::parsoid::WikinodeIterator;
use mwbot::parsoid::{self, Template, Wikicode, map::IndexMap};
#[derive(thiserror::Error, Debug)]
pub enum InfoCompileError {
#[error("Could not compile wikipage: {0}")]
CompileError(#[from] parsoid::Error),
}
pub fn page_from_info(info: DanceInfo) -> Result<Wikicode, InfoCompileError> {
let mut params = IndexMap::new();
params.insert("name".to_string(), info.name());
params.insert(
"dominant_role".to_string(),
info.dominant_role.as_str().to_string(),
);
params.insert(
"allowed_rank".to_string(),
info.dominant_role_comp.rank.as_str().to_string(),
);
params.insert(
"dominant_rank".to_string(),
info.dominant_role_comp.rank.as_str().to_string(),
);
params.insert(
"dominant_points".to_string(),
info.dominant_role_comp.points.to_string(),
);
if let Some(u) = info.non_dominant_role_comp {
params.insert("non_dominant_rank".to_string(), u.rank.as_str().to_string());
params.insert("non_dominant_points".to_string(), u.points.to_string());
}
let t = Template::new("Template:WSDCBox", &params)?;
let result = Wikicode::new("");
result.append(&t);
Ok(result)
}

48
src/worldsdc/caching.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{collections::HashMap, path::Path};
use reqwest::{Client, ClientBuilder};
use crate::{dance_info::DanceInfo, worldsdc::DanceInfoError};
use super::DanceInfoParser;
struct CachingFetcher {
hitcache: Vec<(u32, String)>,
errorcache: Vec<(u32, String)>,
client: Client,
}
#[derive(thiserror::Error, Debug)]
enum CachingFetcherCreationError {
#[error("Could not create client: {0}")]
ClientError(#[from] reqwest::Error),
}
impl CachingFetcher {
pub fn new(cachepath: &Path) -> Result<Self, CachingFetcherCreationError> {
let client = ClientBuilder::new().build()?;
Ok(Self {
hitcache: vec![],
errorcache: vec![],
client,
})
}
pub async fn fetch(&mut self, id: u32) -> Result<DanceInfo, DanceInfoError> {
let mut params = HashMap::new();
params.insert("q", id.to_string());
let response = self
.client
.request(
reqwest::Method::POST,
"https://points.worldsdc.com/lookup2020/find",
)
.form(&params)
.send()
.await
.map_err(DanceInfoError::Request)?;
let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
Ok(x.into())
}
}

46
src/worldsdc/mod.rs
View File

@@ -1,9 +1,10 @@
use std::{collections::HashMap, path::Path};
use std::collections::HashMap;
use reqwest::{Client, ClientBuilder};
use reqwest::ClientBuilder;
use crate::dance_info::{CompState, DanceInfo, DanceRank, DanceRole};
// mod caching;
pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
let client = ClientBuilder::new()
.build()
@@ -25,47 +26,6 @@ pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
Ok(x.into())
}
struct CachingFetcher {
hitcache: Vec<(u32, String)>,
errorcache: Vec<(u32, String)>,
client: Client,
}
#[derive(thiserror::Error, Debug)]
enum CachingFetcherCreationError {
#[error("Could not create client: {0}")]
ClientError(#[from] reqwest::Error),
}
impl CachingFetcher {
pub fn new(cachepath: &Path) -> Result<Self, CachingFetcherCreationError> {
let client = ClientBuilder::new().build()?;
Ok(Self {
hitcache: vec![],
errorcache: vec![],
client,
})
}
pub async fn fetch(&mut self, id: u32) -> Result<DanceInfo, DanceInfoError> {
let mut params = HashMap::new();
params.insert("q", id.to_string());
let response = self
.client
.request(
reqwest::Method::POST,
"https://points.worldsdc.com/lookup2020/find",
)
.form(&params)
.send()
.await
.map_err(DanceInfoError::Request)?;
let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
Ok(x.into())
}
}
#[derive(thiserror::Error, Debug)]
pub enum DanceInfoError {
#[error("Failed to build client: {0}")]