Anri/mobilismScrap
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Wiki Activity

Update Usage

Browse source
This commit is contained in:
Mylloon 2021-08-24 01:43:26 +02:00
parent 75c026d936
commit 67ddba74dd
2 changed files with 27 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
README.md
View file

@ -1,7 +1,7 @@
## Scrap apps from Mobilism ## Scrap apps from Mobilism
Do `git clone https://gitlab.com/Mylloon/mobilismScrap.git && cd mobilismScrap && python3 main.py <user> <pass> <app>`. Do `git clone https://gitlab.com/Mylloon/mobilismScrap.git && cd mobilismScrap && python3 main.py [user] [pass] <app>`.
Or create `.env` file inside the repo folder with `PSEUDO_MOBILISM`, `PASSWORD_MOBILISM` and `APP_MOBILISM` variables. Or create `.env` file inside the repo folder with `PSEUDO_MOBILISM`, `PASSWORD_MOBILISM` variables.
You can also add `DEBUG_MOBILISM` variable to add some verboses. You can also add `DEBUG_MOBILISM` variable to add some verboses (only to `.env` file).

38
main.py
View file

@ -33,11 +33,10 @@ class Scraper:
raise ValueError(self.errorFormat(message = "Cookie containing the SID not found.")) raise ValueError(self.errorFormat(message = "Cookie containing the SID not found."))
if self.debug: print("SID retrieval done,", end = " ") if self.debug: print("SID retrieval done,", end = " ")
if self.debug: print("connection attempt...", end = " ") if self.debug: print("Connection attempt...")
reponse = session.post(f"{self.url}/ucp.php", data = self.loginData, params = {"mode": "login"}) # connect to the forum using credentials reponse = session.post(f"{self.url}/ucp.php", data = self.loginData, params = {"mode": "login"}) # connect to the forum using credentials
if reponse.status_code != 200: if reponse.status_code != 200:
raise ConnectionRefusedError(self.errorFormat(code = reponse.status_code, message = "Unable to connect")) raise ConnectionRefusedError(self.errorFormat(code = reponse.status_code, message = "Unable to connect"))
if self.debug: print("Connection done.")
reponse = session.get(f"{self.url}/index.php", cookies = reponse.cookies, params = {"sid": self.loginData["sid"]}) # back to index page reponse = session.get(f"{self.url}/index.php", cookies = reponse.cookies, params = {"sid": self.loginData["sid"]}) # back to index page
if reponse.status_code != 200: if reponse.status_code != 200:
@ -48,6 +47,8 @@ class Scraper:
def search(self, session) -> list: def search(self, session) -> list:
if self.debug: print("Going to search page...", end = " ") if self.debug: print("Going to search page...", end = " ")
reponse = session.get(f"{self.url}/search.php", params = {"keywords": self.requested_app, "sr": "topics", "sf": "titleonly"}) reponse = session.get(f"{self.url}/search.php", params = {"keywords": self.requested_app, "sr": "topics", "sf": "titleonly"})
if "Sorry but you are not permitted to use the search system. If you're not logged in please" in reponse.text:
raise ConnectionError(self.errorFormat(message = "Connection failed, check credentials"))
if reponse.status_code != 200: if reponse.status_code != 200:
raise ConnectionError(self.errorFormat(code = reponse.status_code, message = "Impossible to make the search")) raise ConnectionError(self.errorFormat(code = reponse.status_code, message = "Impossible to make the search"))
@ -99,18 +100,27 @@ def save(elements):
f.write("\n") f.write("\n")
print(f"{taille} éléments ont étés enrengistés dans le fichier {filename}.") print(f"{taille} éléments ont étés enrengistés dans le fichier {filename}.")
if __name__ == "__main__": if __name__ == "__main__":
argv = argv[1:] argv = argv[1:]
if len(argv) >= 3 and len(argv) <= 4: if len(argv) < 1:
save(Scraper(*argv).work()) print("No App to retrieve.")
else: exit(1)
load_dotenv()
try:
try: try:
load_dotenv() debug = environ["DEBUG_MOBILISM"].lower() in ("yes", "true", "1")
try: except:
debug = environ["DEBUG_MOBILISM"].lower() in ("yes", "true", "1") debug = False
except: try:
debug = False pseudoMobilism = environ["PSEUDO_MOBILISM"]
save(Scraper(environ["PSEUDO_MOBILISM"], environ["PASSWORD_MOBILISM"], environ["APP_MOBILISM"], debug).work()) passwordMobilism = environ["PASSWORD_MOBILISM"]
except KeyError: except:
print('Please fill in the username and password (with ") by args or with .env file.') if len(argv) >= 3:
pseudoMobilism = argv[0]
passwordMobilism = argv[1]
argv = argv[-2:]
else:
raise KeyError
save(Scraper(pseudoMobilism, passwordMobilism, " ".join([n for n in argv]), debug).work())
except KeyError:
print('Please fill in the username and password (with quotes) by args or with .env file and give an app to retrieve.')