Convert Search page to a list (parsing)

app/src/main/java/com/mylloon/mobidl/MainActivity.kt

@@ -25,9 +25,8 @@ import androidx.recyclerview.widget.RecyclerView
 import com.google.android.material.floatingactionbutton.FloatingActionButton
 import com.google.gson.Gson
 import com.google.gson.reflect.TypeToken
-import java.util.*
-import android.widget.Toast
 import kotlinx.coroutines.*
+import java.util.*
 
 
 class MainActivity : AppCompatActivity() {
@@ -35,6 +34,7 @@ class MainActivity : AppCompatActivity() {
     private var inSettings: Boolean = false // by default your not in settings page
     private var prefs: SharedPreferences? = null // first run detection
     private val sharedPref = "com.mylloon.MobiDL" // shared pref name
+    var timeOfLastToast: Long = System.currentTimeMillis() - 2000
 
     companion object {
         private var instance: MainActivity? = null
@@ -163,7 +163,10 @@ class MainActivity : AppCompatActivity() {
                         val checkedItems = booleanArrayOf(
                             false // get this info from the app files somewhere
                         )
-                        builder.setMultiChoiceItems(arrayOf(getString(R.string.updateCheck)), checkedItems) { _, _, isChecked ->
+                        builder.setMultiChoiceItems(
+                            arrayOf(getString(R.string.updateCheck)),
+                            checkedItems
+                        ) { _, _, isChecked ->
                             println("Update for ${button?.text.toString()}: " + if (isChecked) "enabled" else "disabled" + "")
                         }
                         builder.setNeutralButton(R.string.cancel) { dialog, _ -> dialog.cancel() }
@@ -175,7 +178,11 @@ class MainActivity : AppCompatActivity() {
                         if (password != null) { // call script
                             callScrapper(user, password.toString(), button?.text.toString())
                         } else {
-                            Toast.makeText(applicationContext, R.string.notConnected, Toast.LENGTH_LONG)
+                            Toast.makeText(
+                                applicationContext,
+                                R.string.notConnected,
+                                Toast.LENGTH_LONG
+                            )
                                 .show()
                             loginPage()
                         }
@@ -233,7 +240,13 @@ class MainActivity : AppCompatActivity() {
         }
     }
 
-    private fun callScrapper(user: String, password: String, app: String, captchaID: String? = null, captchaResponse: String? = null) {
+    private fun callScrapper(
+        user: String,
+        password: String,
+        app: String,
+        captchaID: String? = null,
+        captchaResponse: String? = null
+    ) {
         if (ContextCompat.checkSelfPermission(this@MainActivity, Manifest.permission.INTERNET)
             != PackageManager.PERMISSION_GRANTED
         ) {
@@ -242,39 +255,95 @@ class MainActivity : AppCompatActivity() {
             this.finishAffinity()
         } else {
             runBlocking { // GlobalScope.launch {
-            val returns: MutableList<String>? = Scraper(user, password, app, captchaID = captchaID, captchaResponse = captchaResponse, debug = true).search()
+                val returns: MutableList<Map<String, String?>>? = Scraper(
+                    user,
+                    password,
+                    app,
+                    captchaID = captchaID,
+                    captchaResponse = captchaResponse
+                ).search()
                 if (returns != null) { // if job is needed
-                    if (returns[0] == "errorNotConnected") loginPage()
+                    if (returns[0].containsKey("errorNotConnected")) {
-                    if (returns[0] == "loginAttemptsExceeded") {
+                        Toast.makeText(
-                        if (returns[1] == "null") callScrapper(user, password, app)
+                            instance,
+                            "${getString(R.string.connectionFailed)}\n${getString(R.string.credentialsDeleted)}.",
+                            Toast.LENGTH_LONG
+                        ).show()
+                        loginPage()
+                    }
+                    if (returns[0].containsKey("loginAttemptsExceeded")) {
+                        val returnsCaptchaQuestion = returns[0]["question"]
+                        val returnsCaptchaID = returns[0]["qaConfirmID"]
+                        if (returnsCaptchaQuestion == null) callScrapper(user, password, app)
                         val registeredAnswered: Map<String, String> = Gson().fromJson(
                             assets.open("captcha.json").bufferedReader()
                                 .use { it.readText() },
                             object : TypeToken<Map<String, String>>() {}.type
                         )
-                        for((key, value) in registeredAnswered){
+                        for ((key, value) in registeredAnswered) {
-                            if (returns[1] == key) return@runBlocking callScrapper(user, password, app, returns[2], value)
+                            if (returnsCaptchaQuestion == key) {
+                                println("${applicationContext().getString(R.string.knownCaptcha)}...")
+                                val now = System.currentTimeMillis()
+                                if ((now - timeOfLastToast) >= 2000) {
+                                    Toast.makeText(
+                                        applicationContext(),
+                                        "${applicationContext().getString(R.string.knownCaptcha)}...",
+                                        Toast.LENGTH_SHORT
+                                    )
+                                        .show()
+                                    timeOfLastToast = now
+                                }
+                                return@runBlocking callScrapper(
+                                    user,
+                                    password,
+                                    app,
+                                    returnsCaptchaID,
+                                    value
+                                )
+                            }
                         }
                         val builder: AlertDialog.Builder = AlertDialog.Builder(
-                            instance)
+                            instance
+                        )
                         builder.setTitle("Captcha")
-                        builder.setMessage(returns[1])
+                        builder.setMessage(returnsCaptchaQuestion)
                         val input = EditText(instance)
                         input.inputType = InputType.TYPE_CLASS_TEXT
                         builder.setView(input)
-                        builder.setPositiveButton(R.string.validate) { _, _ -> callScrapper(user, password, app, returns[2], input.text.toString()) }
+                        builder.setPositiveButton(R.string.validate) { _, _ ->
+                            callScrapper(
+                                user,
+                                password,
+                                app,
+                                returnsCaptchaID,
+                                input.text.toString()
+                            )
+                        }
                         builder.setNeutralButton(R.string.cancel) { dialog, _ -> dialog.cancel() }
                         builder.show()
-
-                        Toast.makeText(instance, "${getString(R.string.connectionFailed)}\n${getString(R.string.credentialsDeleted)}.",
-                            Toast.LENGTH_LONG).show()
                     }
-                    if (returns[0] == "noSID")
+                    if (returns[0].containsKey("noSID"))
                         Toast.makeText(instance, R.string.noSID, Toast.LENGTH_SHORT).show()
-                    if (returns[0] == "badSID")
+                    if (returns[0].containsKey("badSID"))
-                        Toast.makeText(instance, "${getString(R.string.badSID)}...", Toast.LENGTH_SHORT).show()
+                        Toast.makeText(
-                    if (returns[0] == "noResults")
+                            instance,
-                        Toast.makeText(instance, "${getString(R.string.badSID)}...", Toast.LENGTH_SHORT).show()
+                            "${getString(R.string.badSID)}...",
+                            Toast.LENGTH_SHORT
+                        ).show()
+                    if (returns[0].containsKey("noResults"))
+                        Toast.makeText(
+                            instance,
+                            "${getString(R.string.noResults)}...",
+                            Toast.LENGTH_SHORT
+                        ).show()
+                    if (returns[0].containsKey("gotResults")) {
+                        returns.removeFirst()
+                        Toast.makeText(
+                            instance,
+                            "${returns.size} ${getString(R.string.gotResults)} !",
+                            Toast.LENGTH_SHORT
+                        ).show()
+                    }
                 }
             }
         }

app/src/main/java/com/mylloon/mobidl/Scraper.kt

@@ -1,21 +1,15 @@
 package com.mylloon.mobidl
 
-import android.widget.Toast
 import com.github.kittinunf.fuel.core.FuelManager
 import com.github.kittinunf.fuel.httpGet
 import com.github.kittinunf.fuel.httpPost
-import com.mylloon.mobidl.MainActivity.Companion.applicationContext
-import java.lang.Exception
-import java.lang.System.currentTimeMillis
 import com.github.kittinunf.result.Result.Failure as FuelFailure
 import com.github.kittinunf.result.Result.Success as FuelSuccess
 
-var timeOfLastToast: Long = currentTimeMillis() - 2000
 class Scraper(
     private var pseudo: String,
     private var password: String,
     private var app: String,
-    private var debug: Boolean = false,
     private var captchaID: String? = null,
     private var captchaResponse: String? = null
 ) {
@@ -26,17 +20,11 @@ class Scraper(
 
     private var errorNotConnected = "Log me on automatically each visit"
     private var loginAttemptsExceeded = "You exceeded the maximum allowed number of login attempts."
-    private var searchNotLogged = "Sorry but you are not permitted to use the search system. If you're not logged in please"
+    private var searchNotLogged =
+        "Sorry but you are not permitted to use the search system. If you're not logged in please"
 
-    private fun errorFormat( // Pretty error message.
+    private fun connect(): MutableList<Map<String, String?>>? { // Login to the forum using credentials.
-        code: Int? = null,
+        println("Connection attempt...")
-        message: String = ""
-    ): String {
-        return "${if (code != null) "[$code]" else ""}${if ((message.isNotEmpty()) and (code is Int)) " " else ""}$message."
-    }
-
-    private fun connect(): MutableList<String>? { // Login to the forum using credentials.
-        if (debug) println("Connection attempt...")
         retry++
         FuelManager.instance.basePath = url
         FuelManager.instance.baseParams = listOf("mode" to "login")
@@ -51,7 +39,7 @@ class Scraper(
                 when (result) {
                     is FuelFailure -> {
                         val ex = result.getException()
-                        println(errorFormat(response.statusCode, "Exception: $ex"))
+                        println("[${response.statusCode}] Exception: $ex")
                     }
                     is FuelSuccess -> {
                         println("") // does nothing but is required otherwise everything bugs :(
@@ -65,53 +53,50 @@ class Scraper(
         return search()
     }
 
-    private fun error(htmlPage: String): MutableList<String>? { // Handle connection errors
+    private fun error(htmlPage: String): MutableList<Map<String, String?>>? { // Handle connection errors
-        var message = ""
+        var array: MutableList<Map<String, String?>>? = null
-        var array: MutableList<String>? = null
-        var lengthTime = Toast.LENGTH_LONG
         if (errorNotConnected in htmlPage) {
             if (loginAttemptsExceeded in htmlPage) {
-                message = "${applicationContext().getString(R.string.knownCaptcha)}..."
+                val qaConfirmID =
-                lengthTime = Toast.LENGTH_SHORT
+                    "(?<=qa_confirm_id\" value=\")(.{32})".toRegex().find(htmlPage)?.value
-                val qaConfirmID = "(?<=qa_confirm_id\" value=\")(.{32})".toRegex().find(htmlPage)?.value
                 var question = "(?<=answer\">)(.*)</l".toRegex().find(htmlPage)?.value
-                question = question?.dropLast(3)
+                if (question != null) {
-                array = if (question == null) null else mutableListOf("loginAttemptsExceeded", question.toString(), qaConfirmID.toString())
+                    question = question.dropLast(3)
+                }
+                array = mutableListOf(
+                    mutableMapOf(
+                        "loginAttemptsExceeded" to null,
+                        "question" to question,
+                        "qaConfirmID" to qaConfirmID
+                    )
+                )
             } else {
-                message = "${applicationContext().getString(R.string.connectionFailed)}\n${applicationContext().getString(R.string.credentialsDeleted)}."
                 Credentials().delete()
-                array = mutableListOf("errorNotConnected")
+                array = mutableListOf(mutableMapOf("errorNotConnected" to null))
             }
         }
-        val now = currentTimeMillis()
-        fun sendToast() {
-            Toast.makeText(applicationContext(), message, lengthTime)
-                .show()
-            timeOfLastToast = now
-        }
-        println(errorFormat(message = message))
-        if ((now - timeOfLastToast) >= 2000) sendToast()
         return array
     }
 
-    fun search(): MutableList<String>? { // Do the research.
+    fun search(): MutableList<Map<String, String?>>? { // Do the research.
         if (sid == null) {
-            println(errorFormat(message = "SID not found"))
+            println("SID not found")
-            return if (retry < 3) connect() else mutableListOf("noSID")
+            return if (retry < 3) connect() else mutableListOf(mutableMapOf("noSID" to null))
         } else {
-            if (debug) println("SID recovered")
+            println("SID recovered")
             retry = 0
         }
-        if (debug) println("Going to search page...")
+        println("Going to search page...")
         FuelManager.instance.basePath = url
-        FuelManager.instance.baseParams = listOf("sid" to sid, "sr" to "topics", "sf" to "titleonly")
+        FuelManager.instance.baseParams =
+            listOf("sid" to sid, "sr" to "topics", "sf" to "titleonly")
         val session = "/search.php"
             .httpGet(listOf("keywords" to app))
             .responseString { _, response, result ->
                 when (result) {
                     is FuelFailure -> {
                         val ex = result.getException()
-                        println(errorFormat(response.statusCode, "Exception: $ex"))
+                        println("[${response.statusCode}] Exception: $ex")
                     }
                     is FuelSuccess -> {
                         println("") // does nothing but is required otherwise everything bugs :(
@@ -120,50 +105,57 @@ class Scraper(
             }
         val data = session.join().data.decodeToString()
         return if (searchNotLogged in data) {
-            println(errorFormat(message = "The SID doesn't work, new attempt..."))
+            println("The SID doesn't work, new attempt...")
-            if (retry < 3) connect() else mutableListOf("badSID")
+            if (retry < 3) connect() else mutableListOf(mutableMapOf("badSID" to null))
         } else parse(data)
     }
 
-    private fun parse(htmlPage: String): MutableList<String>? { // Parse HTML response to a clean list.
+    private fun parse(htmlPage: String): MutableList<Map<String, String?>> { // Parse HTML response to a clean list.
-        if (debug) println("Fetching results for $app...")
+        println("Fetching results for $app...")
         // println(htmlPage)
-        if ("No suitable matches were found." in htmlPage) return mutableListOf("noResults")
+        if ("No suitable matches were found." in htmlPage) return mutableListOf(mutableMapOf("noResults" to null))
         val elements: MutableList<String> = htmlPage.split("<tr>\n<td>").toMutableList()
         val finalElements = mutableListOf<Map<String, String?>>()
         elements.removeFirst()
         val lastIndex = elements.toList().lastIndex
         elements[lastIndex] = elements[lastIndex].split("</td>\n</tr>")[0]
+        finalElements.add(0, mapOf("gotResults" to null))
         for (i in elements.indices) {
-            var title: String?
+            val title: String? = try {
-            var author: String?
+                Regex("""class="topictitle">(.*)</a>""").findAll(elements[i])
-            var link: String?
+                    .map { it.groupValues[1] }.toList()[0].replace(Regex("( ?&amp; ?)"), " & ")
-            var data: String?
-            title = try {
-                Regex(" ?&amp; ?").replace("", Regex("""class="topictitle">(.*)</a>""").find(elements[i])?.value!!)
             } catch (e: Exception) {
                 null
             }
-            author = try {
+            val author: String? = try {
-                val regex = """(<br />|</strong>)\n\n?<i class="icon-user"></i> by <a href="\./memberlist\.php\?mode=viewprofile&amp;u=\d+"( style="color: #.*;" class="username-coloured")?>(.*)</a>"""
+                val regex =
-                Regex(regex).find(elements[i])?.value!!
+                    """(<br />|</strong>)\n\n?<i class="icon-user"></i> by <a href="\./memberlist\.php\?mode=viewprofile&amp;u=\d+&amp;sid=.+"( style="color: #.*;" class="username-coloured")?>(.*)</a>"""
+                Regex(regex).findAll(elements[i]).map { it.groupValues.last() }.toList()[0]
             } catch (e: Exception) {
                 null
             }
-            link = try {
+            val link: String? = try {
-                Regex("""\./viewtopic\.php\?f=(\d*)&amp;t=(\d*)&amp""").find(elements[i])?.value!!
+                val temp =
+                    Regex("""\./viewtopic\.php\?f=(\d*)&amp;t=(\d*)&amp""").findAll(elements[i])
+                "/viewtopic.php?f=${
+                    temp.map { it.groupValues[1] }.toList()[0]
+                }&t=${temp.map { it.groupValues[2] }.toList()[0]}"
             } catch (e: Exception) {
                 null
             }
-            data = try {
+            val date: String? = try {
-                Regex("""</a> <i class="icon-time"></i> <small>(.*)</small>""").find(elements[i])?.value!!
+                Regex("""</a> <i class="icon-time"></i> <small>(.*)</small>""").findAll(elements[i])
+                    .map { it.groupValues[1] }.toList()[0]
             } catch (e: Exception) {
                 null
             }
-            finalElements.add(i, mapOf("title" to title, "author" to author, "link" to link, "data" to data))
+            finalElements.add(
+                i + 1,
+                mapOf("title" to title, "author" to author, "link" to link, "date" to date)
+            )
         }
-        println(finalElements)
+        println("Search parsed!")
-        return null
+        return finalElements
     }
 
 }

app/src/main/res/values-fr-rFR/strings.xml

@@ -26,4 +26,6 @@
     <string name="noSID">Aucun SID n\'a été trouvé</string>
     <string name="badSID">Le SID ne fonctionne pas, nouvel essaie</string>
     <string name="updateCheck">Vérification des mises à jour</string>
+    <string name="noResults">Aucun résultat n\'a été trouvé</string>
+    <string name="gotResults">résultats ont été trouvés</string>
 </resources>

app/src/main/res/values/strings.xml

@@ -33,4 +33,6 @@
     <string name="knownCaptcha">Resolving captchas</string>
     <string name="noSID">No SID was found</string>
     <string name="badSID">The SID does not work, new attempt</string>
+    <string name="noResults">No results were found</string>
+    <string name="gotResults">results were found</string>
 </resources>