Anri/mobilismScrap
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Wiki Activity

Convert Search page to a list (parsing)

Browse source
This commit is contained in:
Mylloon 2021-09-01 11:34:25 +02:00
parent ac99305a16
commit 785ab6a396
4 changed files with 151 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

111
app/src/main/java/com/mylloon/mobidl/MainActivity.kt
View file

@ -25,9 +25,8 @@ import androidx.recyclerview.widget.RecyclerView
import com.google.android.material.floatingactionbutton.FloatingActionButton
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
import java.util.*
import android.widget.Toast
import kotlinx.coroutines.*
import java.util.*
class MainActivity : AppCompatActivity() {
@ -35,6 +34,7 @@ class MainActivity : AppCompatActivity() {
private var inSettings: Boolean = false // by default your not in settings page
private var prefs: SharedPreferences? = null // first run detection
private val sharedPref = "com.mylloon.MobiDL" // shared pref name
var timeOfLastToast: Long = System.currentTimeMillis() - 2000
companion object {
private var instance: MainActivity? = null
@ -163,7 +163,10 @@ class MainActivity : AppCompatActivity() {
val checkedItems = booleanArrayOf(
false // get this info from the app files somewhere
)
builder.setMultiChoiceItems(arrayOf(getString(R.string.updateCheck)), checkedItems) { _, _, isChecked ->
builder.setMultiChoiceItems(
arrayOf(getString(R.string.updateCheck)),
checkedItems
) { _, _, isChecked ->
println("Update for ${button?.text.toString()}: " + if (isChecked) "enabled" else "disabled" + "")
}
builder.setNeutralButton(R.string.cancel) { dialog, _ -> dialog.cancel() }
@ -175,7 +178,11 @@ class MainActivity : AppCompatActivity() {
if (password != null) { // call script
callScrapper(user, password.toString(), button?.text.toString())
} else {
Toast.makeText(applicationContext, R.string.notConnected, Toast.LENGTH_LONG)
Toast.makeText(
applicationContext,
R.string.notConnected,
Toast.LENGTH_LONG
)
.show()
loginPage()
}
@ -233,7 +240,13 @@ class MainActivity : AppCompatActivity() {
}
}
private fun callScrapper(user: String, password: String, app: String, captchaID: String? = null, captchaResponse: String? = null) {
private fun callScrapper(
user: String,
password: String,
app: String,
captchaID: String? = null,
captchaResponse: String? = null
) {
if (ContextCompat.checkSelfPermission(this@MainActivity, Manifest.permission.INTERNET)
!= PackageManager.PERMISSION_GRANTED
) {
@ -242,39 +255,95 @@ class MainActivity : AppCompatActivity() {
this.finishAffinity()
} else {
runBlocking { // GlobalScope.launch {
val returns: MutableList<String>? = Scraper(user, password, app, captchaID = captchaID, captchaResponse = captchaResponse, debug = true).search()
val returns: MutableList<Map<String, String?>>? = Scraper(
user,
password,
app,
captchaID = captchaID,
captchaResponse = captchaResponse
).search()
if (returns != null) { // if job is needed
if (returns[0] == "errorNotConnected") loginPage()
if (returns[0] == "loginAttemptsExceeded") {
if (returns[1] == "null") callScrapper(user, password, app)
if (returns[0].containsKey("errorNotConnected")) {
Toast.makeText(
instance,
"${getString(R.string.connectionFailed)}\n${getString(R.string.credentialsDeleted)}.",
Toast.LENGTH_LONG
).show()
loginPage()
}
if (returns[0].containsKey("loginAttemptsExceeded")) {
val returnsCaptchaQuestion = returns[0]["question"]
val returnsCaptchaID = returns[0]["qaConfirmID"]
if (returnsCaptchaQuestion == null) callScrapper(user, password, app)
val registeredAnswered: Map<String, String> = Gson().fromJson(
assets.open("captcha.json").bufferedReader()
.use { it.readText() },
object : TypeToken<Map<String, String>>() {}.type
)
for ((key, value) in registeredAnswered) {
if (returns[1] == key) return@runBlocking callScrapper(user, password, app, returns[2], value)
if (returnsCaptchaQuestion == key) {
println("${applicationContext().getString(R.string.knownCaptcha)}...")
val now = System.currentTimeMillis()
if ((now - timeOfLastToast) >= 2000) {
Toast.makeText(
applicationContext(),
"${applicationContext().getString(R.string.knownCaptcha)}...",
Toast.LENGTH_SHORT
)
.show()
timeOfLastToast = now
}
return@runBlocking callScrapper(
user,
password,
app,
returnsCaptchaID,
value
)
}
}
val builder: AlertDialog.Builder = AlertDialog.Builder(
instance)
instance
)
builder.setTitle("Captcha")
builder.setMessage(returns[1])
builder.setMessage(returnsCaptchaQuestion)
val input = EditText(instance)
input.inputType = InputType.TYPE_CLASS_TEXT
builder.setView(input)
builder.setPositiveButton(R.string.validate) { _, _ -> callScrapper(user, password, app, returns[2], input.text.toString()) }
builder.setPositiveButton(R.string.validate) { _, _ ->
callScrapper(
user,
password,
app,
returnsCaptchaID,
input.text.toString()
)
}
builder.setNeutralButton(R.string.cancel) { dialog, _ -> dialog.cancel() }
builder.show()
Toast.makeText(instance, "${getString(R.string.connectionFailed)}\n${getString(R.string.credentialsDeleted)}.",
Toast.LENGTH_LONG).show()
}
if (returns[0] == "noSID")
if (returns[0].containsKey("noSID"))
Toast.makeText(instance, R.string.noSID, Toast.LENGTH_SHORT).show()
if (returns[0] == "badSID")
Toast.makeText(instance, "${getString(R.string.badSID)}...", Toast.LENGTH_SHORT).show()
if (returns[0] == "noResults")
Toast.makeText(instance, "${getString(R.string.badSID)}...", Toast.LENGTH_SHORT).show()
if (returns[0].containsKey("badSID"))
Toast.makeText(
instance,
"${getString(R.string.badSID)}...",
Toast.LENGTH_SHORT
).show()
if (returns[0].containsKey("noResults"))
Toast.makeText(
instance,
"${getString(R.string.noResults)}...",
Toast.LENGTH_SHORT
).show()
if (returns[0].containsKey("gotResults")) {
returns.removeFirst()
Toast.makeText(
instance,
"${returns.size} ${getString(R.string.gotResults)} !",
Toast.LENGTH_SHORT
).show()
}
}
}
}

120
app/src/main/java/com/mylloon/mobidl/Scraper.kt
View file

@ -1,21 +1,15 @@
package com.mylloon.mobidl
import android.widget.Toast
import com.github.kittinunf.fuel.core.FuelManager
import com.github.kittinunf.fuel.httpGet
import com.github.kittinunf.fuel.httpPost
import com.mylloon.mobidl.MainActivity.Companion.applicationContext
import java.lang.Exception
import java.lang.System.currentTimeMillis
import com.github.kittinunf.result.Result.Failure as FuelFailure
import com.github.kittinunf.result.Result.Success as FuelSuccess
var timeOfLastToast: Long = currentTimeMillis() - 2000
class Scraper(
private var pseudo: String,
private var password: String,
private var app: String,
private var debug: Boolean = false,
private var captchaID: String? = null,
private var captchaResponse: String? = null
) {
@ -26,17 +20,11 @@ class Scraper(
private var errorNotConnected = "Log me on automatically each visit"
private var loginAttemptsExceeded = "You exceeded the maximum allowed number of login attempts."
private var searchNotLogged = "Sorry but you are not permitted to use the search system. If you're not logged in please"
private var searchNotLogged =
"Sorry but you are not permitted to use the search system. If you're not logged in please"
private fun errorFormat( // Pretty error message.
code: Int? = null,
message: String = ""
): String {
return "${if (code != null) "[$code]" else ""}${if ((message.isNotEmpty()) and (code is Int)) " " else ""}$message."
}
private fun connect(): MutableList<String>? { // Login to the forum using credentials.
if (debug) println("Connection attempt...")
private fun connect(): MutableList<Map<String, String?>>? { // Login to the forum using credentials.
println("Connection attempt...")
retry++
FuelManager.instance.basePath = url
FuelManager.instance.baseParams = listOf("mode" to "login")
@ -51,7 +39,7 @@ class Scraper(
when (result) {
is FuelFailure -> {
val ex = result.getException()
println(errorFormat(response.statusCode, "Exception: $ex"))
println("[${response.statusCode}] Exception: $ex")
}
is FuelSuccess -> {
println("") // does nothing but is required otherwise everything bugs :(
@ -65,53 +53,50 @@ class Scraper(
return search()
}
private fun error(htmlPage: String): MutableList<String>? { // Handle connection errors
var message = ""
var array: MutableList<String>? = null
var lengthTime = Toast.LENGTH_LONG
private fun error(htmlPage: String): MutableList<Map<String, String?>>? { // Handle connection errors
var array: MutableList<Map<String, String?>>? = null
if (errorNotConnected in htmlPage) {
if (loginAttemptsExceeded in htmlPage) {
message = "${applicationContext().getString(R.string.knownCaptcha)}..."
lengthTime = Toast.LENGTH_SHORT
val qaConfirmID = "(?<=qa_confirm_id\" value=\")(.{32})".toRegex().find(htmlPage)?.value
val qaConfirmID =
"(?<=qa_confirm_id\" value=\")(.{32})".toRegex().find(htmlPage)?.value
var question = "(?<=answer\">)(.*)</l".toRegex().find(htmlPage)?.value
question = question?.dropLast(3)
array = if (question == null) null else mutableListOf("loginAttemptsExceeded", question.toString(), qaConfirmID.toString())
if (question != null) {
question = question.dropLast(3)
}
array = mutableListOf(
mutableMapOf(
"loginAttemptsExceeded" to null,
"question" to question,
"qaConfirmID" to qaConfirmID
)
)
} else {
message = "${applicationContext().getString(R.string.connectionFailed)}\n${applicationContext().getString(R.string.credentialsDeleted)}."
Credentials().delete()
array = mutableListOf("errorNotConnected")
array = mutableListOf(mutableMapOf("errorNotConnected" to null))
}
}
val now = currentTimeMillis()
fun sendToast() {
Toast.makeText(applicationContext(), message, lengthTime)
.show()
timeOfLastToast = now
}
println(errorFormat(message = message))
if ((now - timeOfLastToast) >= 2000) sendToast()
return array
}
fun search(): MutableList<String>? { // Do the research.
fun search(): MutableList<Map<String, String?>>? { // Do the research.
if (sid == null) {
println(errorFormat(message = "SID not found"))
return if (retry < 3) connect() else mutableListOf("noSID")
println("SID not found")
return if (retry < 3) connect() else mutableListOf(mutableMapOf("noSID" to null))
} else {
if (debug) println("SID recovered")
println("SID recovered")
retry = 0
}
if (debug) println("Going to search page...")
println("Going to search page...")
FuelManager.instance.basePath = url
FuelManager.instance.baseParams = listOf("sid" to sid, "sr" to "topics", "sf" to "titleonly")
FuelManager.instance.baseParams =
listOf("sid" to sid, "sr" to "topics", "sf" to "titleonly")
val session = "/search.php"
.httpGet(listOf("keywords" to app))
.responseString { _, response, result ->
when (result) {
is FuelFailure -> {
val ex = result.getException()
println(errorFormat(response.statusCode, "Exception: $ex"))
println("[${response.statusCode}] Exception: $ex")
}
is FuelSuccess -> {
println("") // does nothing but is required otherwise everything bugs :(
@ -120,50 +105,57 @@ class Scraper(
}
val data = session.join().data.decodeToString()
return if (searchNotLogged in data) {
println(errorFormat(message = "The SID doesn't work, new attempt..."))
if (retry < 3) connect() else mutableListOf("badSID")
println("The SID doesn't work, new attempt...")
if (retry < 3) connect() else mutableListOf(mutableMapOf("badSID" to null))
} else parse(data)
}
private fun parse(htmlPage: String): MutableList<String>? { // Parse HTML response to a clean list.
if (debug) println("Fetching results for $app...")
private fun parse(htmlPage: String): MutableList<Map<String, String?>> { // Parse HTML response to a clean list.
println("Fetching results for $app...")
// println(htmlPage)
if ("No suitable matches were found." in htmlPage) return mutableListOf("noResults")
if ("No suitable matches were found." in htmlPage) return mutableListOf(mutableMapOf("noResults" to null))
val elements: MutableList<String> = htmlPage.split("<tr>\n<td>").toMutableList()
val finalElements = mutableListOf<Map<String, String?>>()
elements.removeFirst()
val lastIndex = elements.toList().lastIndex
elements[lastIndex] = elements[lastIndex].split("</td>\n</tr>")[0]
finalElements.add(0, mapOf("gotResults" to null))
for (i in elements.indices) {
var title: String?
var author: String?
var link: String?
var data: String?
title = try {
Regex(" ?&amp; ?").replace("", Regex("""class="topictitle">(.*)</a>""").find(elements[i])?.value!!)
val title: String? = try {
Regex("""class="topictitle">(.*)</a>""").findAll(elements[i])
.map { it.groupValues[1] }.toList()[0].replace(Regex("( ?&amp; ?)"), " & ")
} catch (e: Exception) {
null
}
author = try {
val regex = """(<br />|</strong>)\n\n?<i class="icon-user"></i> by <a href="\./memberlist\.php\?mode=viewprofile&amp;u=\d+"( style="color: #.*;" class="username-coloured")?>(.*)</a>"""
Regex(regex).find(elements[i])?.value!!
val author: String? = try {
val regex =
"""(<br />|</strong>)\n\n?<i class="icon-user"></i> by <a href="\./memberlist\.php\?mode=viewprofile&amp;u=\d+&amp;sid=.+"( style="color: #.*;" class="username-coloured")?>(.*)</a>"""
Regex(regex).findAll(elements[i]).map { it.groupValues.last() }.toList()[0]
} catch (e: Exception) {
null
}
link = try {
Regex("""\./viewtopic\.php\?f=(\d*)&amp;t=(\d*)&amp""").find(elements[i])?.value!!
val link: String? = try {
val temp =
Regex("""\./viewtopic\.php\?f=(\d*)&amp;t=(\d*)&amp""").findAll(elements[i])
"/viewtopic.php?f=${
temp.map { it.groupValues[1] }.toList()[0]
}&t=${temp.map { it.groupValues[2] }.toList()[0]}"
} catch (e: Exception) {
null
}
data = try {
Regex("""</a> <i class="icon-time"></i> <small>(.*)</small>""").find(elements[i])?.value!!
val date: String? = try {
Regex("""</a> <i class="icon-time"></i> <small>(.*)</small>""").findAll(elements[i])
.map { it.groupValues[1] }.toList()[0]
} catch (e: Exception) {
null
}
finalElements.add(i, mapOf("title" to title, "author" to author, "link" to link, "data" to data))
finalElements.add(
i + 1,
mapOf("title" to title, "author" to author, "link" to link, "date" to date)
)
}
println(finalElements)
return null
println("Search parsed!")
return finalElements
}
}

2
app/src/main/res/values-fr-rFR/strings.xml
View file

@ -26,4 +26,6 @@
<string name="noSID">Aucun SID n\'a été trouvé</string>
<string name="badSID">Le SID ne fonctionne pas, nouvel essaie</string>
<string name="updateCheck">Vérification des mises à jour</string>
<string name="noResults">Aucun résultat n\'a été trouvé</string>
<string name="gotResults">résultats ont été trouvés</string>
</resources>

2
app/src/main/res/values/strings.xml
View file

@ -33,4 +33,6 @@
<string name="knownCaptcha">Resolving captchas</string>
<string name="noSID">No SID was found</string>
<string name="badSID">The SID does not work, new attempt</string>
<string name="noResults">No results were found</string>
<string name="gotResults">results were found</string>
</resources>