Compare commits

...

2 commits

Author SHA1 Message Date
8b6daaab3a
vesion bump 2022-08-29 11:44:49 +02:00
6bc4f69cd5
use custom user-agent 2022-08-29 11:44:35 +02:00
5 changed files with 19 additions and 15 deletions

2
Cargo.lock generated
View file

@ -69,7 +69,7 @@ checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
[[package]] [[package]]
name = "cal8tor" name = "cal8tor"
version = "0.1.0" version = "0.1.1"
dependencies = [ dependencies = [
"chrono", "chrono",
"clap", "clap",

View file

@ -1,6 +1,6 @@
[package] [package]
name = "cal8tor" name = "cal8tor"
version = "0.1.0" version = "0.1.1"
authors = ["Mylloon"] authors = ["Mylloon"]
edition = "2021" edition = "2021"
description = "Extractor of the calendar of the IT degree of university Paris 8" description = "Extractor of the calendar of the IT degree of university Paris 8"

View file

@ -3,8 +3,10 @@ use regex::{Captures, Regex};
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use std::collections::HashMap; use std::collections::HashMap;
pub async fn info() -> HashMap<usize, Vec<(DateTime<Utc>, i64)>> { pub async fn info(user_agent: &str) -> HashMap<usize, Vec<(DateTime<Utc>, i64)>> {
let document = get_webpage().await.expect("Can't reach info website."); let document = get_webpage(user_agent)
.await
.expect("Can't reach info website.");
// Selectors // Selectors
let sel_ul = Selector::parse("ul").unwrap(); let sel_ul = Selector::parse("ul").unwrap();
@ -58,14 +60,11 @@ pub async fn info() -> HashMap<usize, Vec<(DateTime<Utc>, i64)>> {
} }
/// Get info webpage /// Get info webpage
async fn get_webpage() -> Result<Html, Box<dyn std::error::Error>> { async fn get_webpage(user_agent: &str) -> Result<Html, Box<dyn std::error::Error>> {
let url = "https://informatique.up8.edu/licence-iv/edt"; let url = "https://informatique.up8.edu/licence-iv/edt";
// We don't use reqwest::get() but a client with a custom user-agent // Use custom User-Agent
// in order to avoid getting rate limit let client = reqwest::Client::builder().user_agent(user_agent).build()?;
let client = reqwest::Client::builder()
.user_agent("bypass-rate_limit")
.build()?;
let html = client.get(url).send().await?.text().await?; let html = client.get(url).send().await?.text().await?;
// Panic on error // Panic on error

View file

@ -51,16 +51,18 @@ async fn main() {
None => "", None => "",
}; };
let user_agent = format!("cal8tor/{}", env!("CARGO_PKG_VERSION"));
println!( println!(
"Récupération de l'emploi du temps des L{}{}{}...", "Récupération de l'emploi du temps des L{}{}{}...",
year, year,
seperator, seperator,
letter.unwrap_or_default().to_uppercase() letter.unwrap_or_default().to_uppercase()
); );
let timetable = timetable::timetable(year, args.semester, letter).await; let timetable = timetable::timetable(year, args.semester, letter, &user_agent).await;
println!("Récupération des informations par rapport à l'année..."); println!("Récupération des informations par rapport à l'année...");
let info = info::info().await; let info = info::info(&user_agent).await;
if args.export.is_some() { if args.export.is_some() {
// Export the calendar // Export the calendar

View file

@ -15,10 +15,11 @@ pub async fn timetable(
year: i8, year: i8,
semester_opt: Option<i8>, semester_opt: Option<i8>,
letter: Option<char>, letter: Option<char>,
user_agent: &str,
) -> (Vec<String>, (usize, Vec<models::Day>)) { ) -> (Vec<String>, (usize, Vec<models::Day>)) {
let semester = get_semester(semester_opt, letter); let semester = get_semester(semester_opt, letter);
let document = get_webpage(year, semester, letter) let document = get_webpage(year, semester, letter, user_agent)
.await .await
.expect("Can't reach timetable website."); .expect("Can't reach timetable website.");
@ -126,6 +127,7 @@ async fn get_webpage(
year: i8, year: i8,
semester: i8, semester: i8,
letter: Option<char>, letter: Option<char>,
user_agent: &str,
) -> Result<Html, Box<dyn std::error::Error>> { ) -> Result<Html, Box<dyn std::error::Error>> {
let url = { let url = {
let panic_semester_message = "Unknown semester."; let panic_semester_message = "Unknown semester.";
@ -168,8 +170,9 @@ async fn get_webpage(
} }
}; };
// Get raw html // Use custom User-Agent
let html = reqwest::get(&url).await?.text().await?; let client = reqwest::Client::builder().user_agent(user_agent).build()?;
let html = client.get(&url).send().await?.text().await?;
// Panic on error // Panic on error
crate::utils::check_errors(&html, &url); crate::utils::check_errors(&html, &url);