Worked on safer fetching

Cargo.lock

@@ -161,8 +161,10 @@ checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
+ "js-sys",
  "num-traits",
  "serde",
+ "wasm-bindgen",
  "windows-link",
 ]
 
@@ -276,6 +278,7 @@ dependencies = [
 name = "dancing-bot-teachers"
 version = "0.1.0"
 dependencies = [
+ "chrono",
  "mwbot",
  "reqwest",
  "serde",

Cargo.toml

@@ -11,6 +11,7 @@ keywords = ["mediawiki", "bot", "teacher", "score", "automation"]
 categories = ["web-programming", "api-bindings", "automation"]
 
 [dependencies]
+chrono = "0.4.41"
 mwbot = { git = "https://gitlab.wikimedia.org/repos/mwbot-rs/mwbot.git", rev = "05cbb12188f18e2da710de158d89a9a4f1b42689", default-features = false, features = ["generators", "mwbot_derive"] }
 reqwest = "0.12.22"
 serde = { version = "1.0.219", features = ["derive"] }

src/dance_info.rs

@@ -1,5 +1,3 @@
-use std::collections::HashMap;
-
 #[derive(serde::Deserialize, Debug, PartialEq, Eq)]
 pub enum DanceRole {
     Leader,
@@ -48,35 +46,6 @@ impl DanceRank {
     }
 }
 
-#[derive(serde::Deserialize, Debug)]
-enum OptionalDanceRank {
-    #[serde(rename = "N/A")]
-    NotAvailable,
-    #[serde(untagged)]
-    Rank(DanceRank),
-}
-
-#[derive(serde::Deserialize, Debug)]
-enum OptionalDancePoints {
-    #[serde(rename = "N/A")]
-    NotAvailable,
-    #[serde(untagged)]
-    Points(u16),
-}
-
-#[derive(serde::Deserialize, Debug)]
-struct DanceInfoParser {
-    pub dancer_first: String,
-    pub dancer_last: String,
-    pub short_dominate_role: DanceRole,
-    #[allow(dead_code)]
-    pub short_non_dominate_role: DanceRole,
-    pub dominate_role_highest_level_points: u16,
-    pub dominate_role_highest_level: DanceRank,
-    pub non_dominate_role_highest_level_points: OptionalDancePoints,
-    pub non_dominate_role_highest_level: OptionalDanceRank,
-}
-
 #[derive(Debug)]
 pub struct CompState {
     pub rank: DanceRank,
@@ -94,63 +63,4 @@ impl DanceInfo {
     pub fn name(&self) -> String {
         format!("{} {}", self.firstname, self.lastname)
     }
-
-    #[allow(dead_code)]
-    pub const fn non_dominant_role(&self) -> DanceRole {
-        self.dominant_role.other()
-    }
-}
-
-impl From<DanceInfoParser> for DanceInfo {
-    fn from(value: DanceInfoParser) -> Self {
-        let non_dominant_role_comp = if let OptionalDanceRank::Rank(r) =
-            value.non_dominate_role_highest_level
-            && let OptionalDancePoints::Points(l) = value.non_dominate_role_highest_level_points
-        {
-            Some(CompState { rank: r, points: l })
-        } else {
-            None
-        };
-        Self {
-            firstname: value.dancer_first,
-            lastname: value.dancer_last,
-            dominant_role: value.short_dominate_role,
-            dominant_role_comp: CompState {
-                rank: value.dominate_role_highest_level,
-                points: value.dominate_role_highest_level_points,
-            },
-            non_dominant_role_comp,
-        }
-    }
-}
-
-#[derive(thiserror::Error, Debug)]
-pub enum DanceInfoError {
-    #[error("Failed to build client: {0}")]
-    ClientBuild(reqwest::Error),
-    #[error("Request error: {0}")]
-    Request(reqwest::Error),
-    #[error("Failed to parse response: {0}")]
-    JsonParse(reqwest::Error),
-}
-
-pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
-    let client = reqwest::ClientBuilder::new()
-        .build()
-        .map_err(DanceInfoError::ClientBuild)?;
-
-    let mut params = HashMap::new();
-    params.insert("q", id.to_string());
-    let response = client
-        .request(
-            reqwest::Method::POST,
-            "https://points.worldsdc.com/lookup2020/find",
-        )
-        .form(&params)
-        .send()
-        .await
-        .map_err(DanceInfoError::Request)?;
-
-    let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
-    Ok(x.into())
 }

src/main.rs

@@ -24,6 +24,8 @@ use crate::watchdog::watch_wanted;
 mod dance_info;
 mod watchdog;
 mod wikiinfo;
+mod task_schedule;
+mod worldsdc;
 
 #[allow(dead_code)]
 #[allow(clippy::print_stdout, reason = "We want to print here")]

src/task_schedule.rs

@@ -0,0 +1,11 @@
+#[derive(Default, Debug)]
+struct UpdateQueue {
+    exists: Vec<(u32, std::time::SystemTime)>,
+    create: Vec<u32>,
+}
+
+impl UpdateQueue {
+    pub fn add_exist(&mut self, wsdc_id: u32, update_on: std::time::SystemTime) {}
+    pub fn add_create(&mut self, wsdc_id: u32) {}
+    pub async fn run(&mut self) {}
+}

src/watchdog.rs

@@ -1,9 +1,6 @@
 use std::time::Duration;
 
-use crate::{
+use crate::{dance_info::DanceInfo, wikiinfo::wanted_ids, worldsdc::fetch_wsdc_info};
-    dance_info::{DanceInfo, fetch_wsdc_info},
-    wikiinfo::wanted_ids,
-};
 use mwbot::{
     Bot,
     parsoid::{self, Template, Wikicode, map::IndexMap},

src/wikiinfo.rs

@@ -1,6 +1,9 @@
+use std::time::SystemTime;
+
 use mwbot::{
     Bot, Page,
     generators::{Generator, querypage::QueryPage, search::Search},
+    parsoid::{Wikicode, WikinodeIterator},
 };
 
 pub async fn wanted_ids(bot: Bot) -> Vec<(u32, Page)> {
@@ -42,8 +45,19 @@ fn parse_wsdc_page_name(name: &str) -> Result<u32, TitleParseError> {
     }
 }
 
+// fn get_wsdc_page_date(page: &Wikicode) -> Option<SystemTime> {
+//     let prefix = "Updated-On: ";
+//     page.filter_comments()
+//         .iter()
+//         .filter_map(|x| {
+//             let c = x.text_contents();
+//             if c.starts_with(prefix) { Some(c) } else { None }
+//         })
+//         .map(|x| x.trim_start_matches(prefix).parse::<u64>());
+// }
+
 #[allow(dead_code)]
-pub async fn index_wsdc_ids(bot: &Bot) -> Vec<u32> {
+pub async fn index_wsdc_ids(bot: &Bot) -> Vec<(u32, Page)> {
     let mut gene = Search::new("WSDC/").generate(bot);
     let mut result = vec![];
     while let Some(x) = gene.recv().await {
@@ -55,7 +69,7 @@ pub async fn index_wsdc_ids(bot: &Bot) -> Vec<u32> {
             }
         };
         if let Ok(n) = parse_wsdc_page_name(p.title()) {
-            result.push(n);
+            result.push((n, p));
         }
     }
     result

src/worldsdc/mod.rs

@@ -0,0 +1,129 @@
+use std::{collections::HashMap, path::Path};
+
+use reqwest::{Client, ClientBuilder};
+
+use crate::dance_info::{CompState, DanceInfo, DanceRank, DanceRole};
+
+pub async fn fetch_wsdc_info(id: u32) -> Result<DanceInfo, DanceInfoError> {
+    let client = ClientBuilder::new()
+        .build()
+        .map_err(DanceInfoError::ClientBuild)?;
+
+    let mut params = HashMap::new();
+    params.insert("q", id.to_string());
+    let response = client
+        .request(
+            reqwest::Method::POST,
+            "https://points.worldsdc.com/lookup2020/find",
+        )
+        .form(&params)
+        .send()
+        .await
+        .map_err(DanceInfoError::Request)?;
+
+    let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
+    Ok(x.into())
+}
+
+struct CachingFetcher {
+    hitcache: Vec<(u32, String)>,
+    errorcache: Vec<(u32, String)>,
+    client: Client,
+}
+
+#[derive(thiserror::Error, Debug)]
+enum CachingFetcherCreationError {
+    #[error("Could not create client: {0}")]
+    ClientError(#[from] reqwest::Error),
+}
+
+impl CachingFetcher {
+    pub fn new(cachepath: &Path) -> Result<Self, CachingFetcherCreationError> {
+        let client = ClientBuilder::new().build()?;
+
+        Ok(Self {
+            hitcache: vec![],
+            errorcache: vec![],
+            client,
+        })
+    }
+    pub async fn fetch(&mut self, id: u32) -> Result<DanceInfo, DanceInfoError> {
+        let mut params = HashMap::new();
+        params.insert("q", id.to_string());
+        let response = self
+            .client
+            .request(
+                reqwest::Method::POST,
+                "https://points.worldsdc.com/lookup2020/find",
+            )
+            .form(&params)
+            .send()
+            .await
+            .map_err(DanceInfoError::Request)?;
+
+        let x: DanceInfoParser = response.json().await.map_err(DanceInfoError::JsonParse)?;
+        Ok(x.into())
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum DanceInfoError {
+    #[error("Failed to build client: {0}")]
+    ClientBuild(reqwest::Error),
+    #[error("Request error: {0}")]
+    Request(reqwest::Error),
+    #[error("Failed to parse response: {0}")]
+    JsonParse(reqwest::Error),
+}
+
+#[derive(serde::Deserialize, Debug)]
+enum OptionalDanceRank {
+    #[serde(rename = "N/A")]
+    NotAvailable,
+    #[serde(untagged)]
+    Rank(DanceRank),
+}
+
+#[derive(serde::Deserialize, Debug)]
+enum OptionalDancePoints {
+    #[serde(rename = "N/A")]
+    NotAvailable,
+    #[serde(untagged)]
+    Points(u16),
+}
+
+#[derive(serde::Deserialize, Debug)]
+struct DanceInfoParser {
+    pub dancer_first: String,
+    pub dancer_last: String,
+    pub short_dominate_role: DanceRole,
+    #[allow(dead_code)]
+    pub short_non_dominate_role: DanceRole,
+    pub dominate_role_highest_level_points: u16,
+    pub dominate_role_highest_level: DanceRank,
+    pub non_dominate_role_highest_level_points: OptionalDancePoints,
+    pub non_dominate_role_highest_level: OptionalDanceRank,
+}
+
+impl From<DanceInfoParser> for DanceInfo {
+    fn from(value: DanceInfoParser) -> Self {
+        let non_dominant_role_comp = if let OptionalDanceRank::Rank(r) =
+            value.non_dominate_role_highest_level
+            && let OptionalDancePoints::Points(l) = value.non_dominate_role_highest_level_points
+        {
+            Some(CompState { rank: r, points: l })
+        } else {
+            None
+        };
+        Self {
+            firstname: value.dancer_first,
+            lastname: value.dancer_last,
+            dominant_role: value.short_dominate_role,
+            dominant_role_comp: CompState {
+                rank: value.dominate_role_highest_level,
+                points: value.dominate_role_highest_level_points,
+            },
+            non_dominant_role_comp,
+        }
+    }
+}