Compare commits

...

3 commits

Author SHA1 Message Date
974d5e402a
fix: get name by url and not by title 2022-09-03 14:41:16 +02:00
01911ebb4f
use custom user agent 2022-09-03 14:26:50 +02:00
d0c904181c
add verbose 2022-09-03 14:16:11 +02:00
3 changed files with 32 additions and 11 deletions

View file

@ -1,15 +1,17 @@
/// Download all the posts /// Download all the posts
pub async fn download_posts( pub async fn download_posts(posts: (String, Vec<String>), dir: &str, download_special_files: bool) {
posts: (String, Vec<String>),
dir: String,
download_special_files: bool,
) {
// Create folder, silently ignore if already exists // Create folder, silently ignore if already exists
std::fs::create_dir(&dir).unwrap_or_default(); std::fs::create_dir(&dir).unwrap_or_default();
// Define client with custom user-agent
let client = reqwest::Client::builder()
.user_agent(format!("prose_dl/{}", env!("CARGO_PKG_VERSION")))
.build()
.unwrap();
// Download all the posts // Download all the posts
for post in posts.1 { for post in posts.1 {
download(&posts.0, &dir, post, "md").await; download(&posts.0, dir, post, "md", &client).await;
} }
// Check if specials files need to be downloaded // Check if specials files need to be downloaded
@ -20,17 +22,25 @@ pub async fn download_posts(
]; ];
for file in special_files { for file in special_files {
download(&posts.0, &dir, file.0, file.1).await; download(&posts.0, dir, file.0, file.1, &client).await;
} }
} }
} }
/// Download a file from the raw endpoint /// Download a file from the raw endpoint
async fn download(url: &String, output_dir: &String, post_name: String, extension: &str) { async fn download(
url: &String,
output_dir: &str,
post_name: String,
extension: &str,
client: &reqwest::Client,
) {
// Endpoint name // Endpoint name
let endpoint = "raw"; let endpoint = "raw";
let data = reqwest::get(format!("{}/{}/{}", url, endpoint, post_name)) let data = client
.get(format!("{}/{}/{}", url, endpoint, post_name))
.send()
.await .await
.unwrap() .unwrap()
.text() .text()

View file

@ -37,6 +37,7 @@ async fn main() {
let cli = Cli::parse(); let cli = Cli::parse();
// Retrieve user's posts // Retrieve user's posts
println!("Retrieving posts...");
let posts = parse::get_posts( let posts = parse::get_posts(
cli.scheme.to_lowercase(), cli.scheme.to_lowercase(),
cli.username.to_lowercase(), cli.username.to_lowercase(),
@ -51,5 +52,7 @@ async fn main() {
}; };
// Download the posts // Download the posts
download::download_posts(posts, directory, cli.special_files).await; println!("Downloads posts...");
download::download_posts(posts, &directory, cli.special_files).await;
println!("Download completed in {}/ folder.", directory);
} }

View file

@ -17,7 +17,15 @@ pub async fn get_posts(scheme: String, username: String, domain: String) -> (Str
// Get the name of them and push them into the vector // Get the name of them and push them into the vector
let mut posts = Vec::new(); let mut posts = Vec::new();
for link in raw_posts.select(&Selector::parse("a").unwrap()) { for link in raw_posts.select(&Selector::parse("a").unwrap()) {
posts.push(link.inner_html()); posts.push(
link.value()
.attr("href")
.unwrap()
.split('/')
.last()
.unwrap()
.to_owned(),
);
} }
// Return the vector // Return the vector