From 67ddba74dd25bac2b7ab650db8551678ac333386 Mon Sep 17 00:00:00 2001 From: Mylloon Date: Tue, 24 Aug 2021 01:43:26 +0200 Subject: [PATCH] Update Usage --- README.md | 6 +++--- main.py | 38 ++++++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index abe0916..31fe019 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## Scrap apps from Mobilism -Do `git clone https://gitlab.com/Mylloon/mobilismScrap.git && cd mobilismScrap && python3 main.py `. +Do `git clone https://gitlab.com/Mylloon/mobilismScrap.git && cd mobilismScrap && python3 main.py [user] [pass] `. -Or create `.env` file inside the repo folder with `PSEUDO_MOBILISM`, `PASSWORD_MOBILISM` and `APP_MOBILISM` variables. +Or create `.env` file inside the repo folder with `PSEUDO_MOBILISM`, `PASSWORD_MOBILISM` variables. -You can also add `DEBUG_MOBILISM` variable to add some verboses. +You can also add `DEBUG_MOBILISM` variable to add some verboses (only to `.env` file). diff --git a/main.py b/main.py index 23d326c..ae108c1 100644 --- a/main.py +++ b/main.py @@ -33,11 +33,10 @@ class Scraper: raise ValueError(self.errorFormat(message = "Cookie containing the SID not found.")) if self.debug: print("SID retrieval done,", end = " ") - if self.debug: print("connection attempt...", end = " ") + if self.debug: print("Connection attempt...") reponse = session.post(f"{self.url}/ucp.php", data = self.loginData, params = {"mode": "login"}) # connect to the forum using credentials if reponse.status_code != 200: raise ConnectionRefusedError(self.errorFormat(code = reponse.status_code, message = "Unable to connect")) - if self.debug: print("Connection done.") reponse = session.get(f"{self.url}/index.php", cookies = reponse.cookies, params = {"sid": self.loginData["sid"]}) # back to index page if reponse.status_code != 200: @@ -48,6 +47,8 @@ class Scraper: def search(self, session) -> list: if self.debug: print("Going to search page...", end = " ") reponse = session.get(f"{self.url}/search.php", params = {"keywords": self.requested_app, "sr": "topics", "sf": "titleonly"}) + if "Sorry but you are not permitted to use the search system. If you're not logged in please" in reponse.text: + raise ConnectionError(self.errorFormat(message = "Connection failed, check credentials")) if reponse.status_code != 200: raise ConnectionError(self.errorFormat(code = reponse.status_code, message = "Impossible to make the search")) @@ -99,18 +100,27 @@ def save(elements): f.write("\n") print(f"{taille} éléments ont étés enrengistés dans le fichier {filename}.") - if __name__ == "__main__": argv = argv[1:] - if len(argv) >= 3 and len(argv) <= 4: - save(Scraper(*argv).work()) - else: + if len(argv) < 1: + print("No App to retrieve.") + exit(1) + load_dotenv() + try: try: - load_dotenv() - try: - debug = environ["DEBUG_MOBILISM"].lower() in ("yes", "true", "1") - except: - debug = False - save(Scraper(environ["PSEUDO_MOBILISM"], environ["PASSWORD_MOBILISM"], environ["APP_MOBILISM"], debug).work()) - except KeyError: - print('Please fill in the username and password (with ") by args or with .env file.') + debug = environ["DEBUG_MOBILISM"].lower() in ("yes", "true", "1") + except: + debug = False + try: + pseudoMobilism = environ["PSEUDO_MOBILISM"] + passwordMobilism = environ["PASSWORD_MOBILISM"] + except: + if len(argv) >= 3: + pseudoMobilism = argv[0] + passwordMobilism = argv[1] + argv = argv[-2:] + else: + raise KeyError + save(Scraper(pseudoMobilism, passwordMobilism, " ".join([n for n in argv]), debug).work()) + except KeyError: + print('Please fill in the username and password (with quotes) by args or with .env file and give an app to retrieve.')