Compare commits
No commits in common. "974d5e402a0ec4f49b3e1c4f582c91d414096828" and "93ff79d96a7c6edc4c7d12d0f5783d76e2415bd5" have entirely different histories.
974d5e402a
...
93ff79d96a
3 changed files with 11 additions and 32 deletions
|
@ -1,17 +1,15 @@
|
||||||
/// Download all the posts
|
/// Download all the posts
|
||||||
pub async fn download_posts(posts: (String, Vec<String>), dir: &str, download_special_files: bool) {
|
pub async fn download_posts(
|
||||||
|
posts: (String, Vec<String>),
|
||||||
|
dir: String,
|
||||||
|
download_special_files: bool,
|
||||||
|
) {
|
||||||
// Create folder, silently ignore if already exists
|
// Create folder, silently ignore if already exists
|
||||||
std::fs::create_dir(&dir).unwrap_or_default();
|
std::fs::create_dir(&dir).unwrap_or_default();
|
||||||
|
|
||||||
// Define client with custom user-agent
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.user_agent(format!("prose_dl/{}", env!("CARGO_PKG_VERSION")))
|
|
||||||
.build()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Download all the posts
|
// Download all the posts
|
||||||
for post in posts.1 {
|
for post in posts.1 {
|
||||||
download(&posts.0, dir, post, "md", &client).await;
|
download(&posts.0, &dir, post, "md").await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if specials files need to be downloaded
|
// Check if specials files need to be downloaded
|
||||||
|
@ -22,25 +20,17 @@ pub async fn download_posts(posts: (String, Vec<String>), dir: &str, download_sp
|
||||||
];
|
];
|
||||||
|
|
||||||
for file in special_files {
|
for file in special_files {
|
||||||
download(&posts.0, dir, file.0, file.1, &client).await;
|
download(&posts.0, &dir, file.0, file.1).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Download a file from the raw endpoint
|
/// Download a file from the raw endpoint
|
||||||
async fn download(
|
async fn download(url: &String, output_dir: &String, post_name: String, extension: &str) {
|
||||||
url: &String,
|
|
||||||
output_dir: &str,
|
|
||||||
post_name: String,
|
|
||||||
extension: &str,
|
|
||||||
client: &reqwest::Client,
|
|
||||||
) {
|
|
||||||
// Endpoint name
|
// Endpoint name
|
||||||
let endpoint = "raw";
|
let endpoint = "raw";
|
||||||
|
|
||||||
let data = client
|
let data = reqwest::get(format!("{}/{}/{}", url, endpoint, post_name))
|
||||||
.get(format!("{}/{}/{}", url, endpoint, post_name))
|
|
||||||
.send()
|
|
||||||
.await
|
.await
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.text()
|
.text()
|
||||||
|
|
|
@ -37,7 +37,6 @@ async fn main() {
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
|
||||||
// Retrieve user's posts
|
// Retrieve user's posts
|
||||||
println!("Retrieving posts...");
|
|
||||||
let posts = parse::get_posts(
|
let posts = parse::get_posts(
|
||||||
cli.scheme.to_lowercase(),
|
cli.scheme.to_lowercase(),
|
||||||
cli.username.to_lowercase(),
|
cli.username.to_lowercase(),
|
||||||
|
@ -52,7 +51,5 @@ async fn main() {
|
||||||
};
|
};
|
||||||
|
|
||||||
// Download the posts
|
// Download the posts
|
||||||
println!("Downloads posts...");
|
download::download_posts(posts, directory, cli.special_files).await;
|
||||||
download::download_posts(posts, &directory, cli.special_files).await;
|
|
||||||
println!("Download completed in {}/ folder.", directory);
|
|
||||||
}
|
}
|
||||||
|
|
10
src/parse.rs
10
src/parse.rs
|
@ -17,15 +17,7 @@ pub async fn get_posts(scheme: String, username: String, domain: String) -> (Str
|
||||||
// Get the name of them and push them into the vector
|
// Get the name of them and push them into the vector
|
||||||
let mut posts = Vec::new();
|
let mut posts = Vec::new();
|
||||||
for link in raw_posts.select(&Selector::parse("a").unwrap()) {
|
for link in raw_posts.select(&Selector::parse("a").unwrap()) {
|
||||||
posts.push(
|
posts.push(link.inner_html());
|
||||||
link.value()
|
|
||||||
.attr("href")
|
|
||||||
.unwrap()
|
|
||||||
.split('/')
|
|
||||||
.last()
|
|
||||||
.unwrap()
|
|
||||||
.to_owned(),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return the vector
|
// Return the vector
|
||||||
|
|
Reference in a new issue