Anri/mobilismScrap
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Wiki Activity

Update Usage

Browse source
This commit is contained in:
Mylloon 2021-08-24 01:43:26 +02:00
parent 75c026d936
commit 67ddba74dd
2 changed files with 27 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
README.md
View file

@ -1,7 +1,7 @@
## Scrap apps from Mobilism ## Scrap apps from Mobilism
Do `git clone https://gitlab.com/Mylloon/mobilismScrap.git && cd mobilismScrap && python3 main.py <user> <pass> <app>`. Do `git clone https://gitlab.com/Mylloon/mobilismScrap.git && cd mobilismScrap && python3 main.py [user] [pass] <app>`.
Or create `.env` file inside the repo folder with `PSEUDO_MOBILISM`, `PASSWORD_MOBILISM` and `APP_MOBILISM` variables. Or create `.env` file inside the repo folder with `PSEUDO_MOBILISM`, `PASSWORD_MOBILISM` variables.
You can also add `DEBUG_MOBILISM` variable to add some verboses. You can also add `DEBUG_MOBILISM` variable to add some verboses (only to `.env` file).

28
main.py
View file

@ -33,11 +33,10 @@ class Scraper:
raise ValueError(self.errorFormat(message = "Cookie containing the SID not found.")) raise ValueError(self.errorFormat(message = "Cookie containing the SID not found."))
if self.debug: print("SID retrieval done,", end = " ") if self.debug: print("SID retrieval done,", end = " ")
if self.debug: print("connection attempt...", end = " ") if self.debug: print("Connection attempt...")
reponse = session.post(f"{self.url}/ucp.php", data = self.loginData, params = {"mode": "login"}) # connect to the forum using credentials reponse = session.post(f"{self.url}/ucp.php", data = self.loginData, params = {"mode": "login"}) # connect to the forum using credentials
if reponse.status_code != 200: if reponse.status_code != 200:
raise ConnectionRefusedError(self.errorFormat(code = reponse.status_code, message = "Unable to connect")) raise ConnectionRefusedError(self.errorFormat(code = reponse.status_code, message = "Unable to connect"))
if self.debug: print("Connection done.")
reponse = session.get(f"{self.url}/index.php", cookies = reponse.cookies, params = {"sid": self.loginData["sid"]}) # back to index page reponse = session.get(f"{self.url}/index.php", cookies = reponse.cookies, params = {"sid": self.loginData["sid"]}) # back to index page
if reponse.status_code != 200: if reponse.status_code != 200:
@ -48,6 +47,8 @@ class Scraper:
def search(self, session) -> list: def search(self, session) -> list:
if self.debug: print("Going to search page...", end = " ") if self.debug: print("Going to search page...", end = " ")
reponse = session.get(f"{self.url}/search.php", params = {"keywords": self.requested_app, "sr": "topics", "sf": "titleonly"}) reponse = session.get(f"{self.url}/search.php", params = {"keywords": self.requested_app, "sr": "topics", "sf": "titleonly"})
if "Sorry but you are not permitted to use the search system. If you're not logged in please" in reponse.text:
raise ConnectionError(self.errorFormat(message = "Connection failed, check credentials"))
if reponse.status_code != 200: if reponse.status_code != 200:
raise ConnectionError(self.errorFormat(code = reponse.status_code, message = "Impossible to make the search")) raise ConnectionError(self.errorFormat(code = reponse.status_code, message = "Impossible to make the search"))
@ -99,18 +100,27 @@ def save(elements):
f.write("\n") f.write("\n")
print(f"{taille} éléments ont étés enrengistés dans le fichier {filename}.") print(f"{taille} éléments ont étés enrengistés dans le fichier {filename}.")
if __name__ == "__main__": if __name__ == "__main__":
argv = argv[1:] argv = argv[1:]
if len(argv) >= 3 and len(argv) <= 4: if len(argv) < 1:
save(Scraper(*argv).work()) print("No App to retrieve.")
else: exit(1)
try:
load_dotenv() load_dotenv()
try:
try: try:
debug = environ["DEBUG_MOBILISM"].lower() in ("yes", "true", "1") debug = environ["DEBUG_MOBILISM"].lower() in ("yes", "true", "1")
except: except:
debug = False debug = False
save(Scraper(environ["PSEUDO_MOBILISM"], environ["PASSWORD_MOBILISM"], environ["APP_MOBILISM"], debug).work()) try:
pseudoMobilism = environ["PSEUDO_MOBILISM"]
passwordMobilism = environ["PASSWORD_MOBILISM"]
except:
if len(argv) >= 3:
pseudoMobilism = argv[0]
passwordMobilism = argv[1]
argv = argv[-2:]
else:
raise KeyError
save(Scraper(pseudoMobilism, passwordMobilism, " ".join([n for n in argv]), debug).work())
except KeyError: except KeyError:
print('Please fill in the username and password (with ") by args or with .env file.') print('Please fill in the username and password (with quotes) by args or with .env file and give an app to retrieve.')