Convert Search page to a list (parsing)
This commit is contained in:
parent
ac99305a16
commit
785ab6a396
4 changed files with 151 additions and 86 deletions
|
|
@ -25,9 +25,8 @@ import androidx.recyclerview.widget.RecyclerView
|
|||
import com.google.android.material.floatingactionbutton.FloatingActionButton
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.reflect.TypeToken
|
||||
import java.util.*
|
||||
import android.widget.Toast
|
||||
import kotlinx.coroutines.*
|
||||
import java.util.*
|
||||
|
||||
|
||||
class MainActivity : AppCompatActivity() {
|
||||
|
|
@ -35,6 +34,7 @@ class MainActivity : AppCompatActivity() {
|
|||
private var inSettings: Boolean = false // by default your not in settings page
|
||||
private var prefs: SharedPreferences? = null // first run detection
|
||||
private val sharedPref = "com.mylloon.MobiDL" // shared pref name
|
||||
var timeOfLastToast: Long = System.currentTimeMillis() - 2000
|
||||
|
||||
companion object {
|
||||
private var instance: MainActivity? = null
|
||||
|
|
@ -163,7 +163,10 @@ class MainActivity : AppCompatActivity() {
|
|||
val checkedItems = booleanArrayOf(
|
||||
false // get this info from the app files somewhere
|
||||
)
|
||||
builder.setMultiChoiceItems(arrayOf(getString(R.string.updateCheck)), checkedItems) { _, _, isChecked ->
|
||||
builder.setMultiChoiceItems(
|
||||
arrayOf(getString(R.string.updateCheck)),
|
||||
checkedItems
|
||||
) { _, _, isChecked ->
|
||||
println("Update for ${button?.text.toString()}: " + if (isChecked) "enabled" else "disabled" + "")
|
||||
}
|
||||
builder.setNeutralButton(R.string.cancel) { dialog, _ -> dialog.cancel() }
|
||||
|
|
@ -175,7 +178,11 @@ class MainActivity : AppCompatActivity() {
|
|||
if (password != null) { // call script
|
||||
callScrapper(user, password.toString(), button?.text.toString())
|
||||
} else {
|
||||
Toast.makeText(applicationContext, R.string.notConnected, Toast.LENGTH_LONG)
|
||||
Toast.makeText(
|
||||
applicationContext,
|
||||
R.string.notConnected,
|
||||
Toast.LENGTH_LONG
|
||||
)
|
||||
.show()
|
||||
loginPage()
|
||||
}
|
||||
|
|
@ -233,7 +240,13 @@ class MainActivity : AppCompatActivity() {
|
|||
}
|
||||
}
|
||||
|
||||
private fun callScrapper(user: String, password: String, app: String, captchaID: String? = null, captchaResponse: String? = null) {
|
||||
private fun callScrapper(
|
||||
user: String,
|
||||
password: String,
|
||||
app: String,
|
||||
captchaID: String? = null,
|
||||
captchaResponse: String? = null
|
||||
) {
|
||||
if (ContextCompat.checkSelfPermission(this@MainActivity, Manifest.permission.INTERNET)
|
||||
!= PackageManager.PERMISSION_GRANTED
|
||||
) {
|
||||
|
|
@ -242,39 +255,95 @@ class MainActivity : AppCompatActivity() {
|
|||
this.finishAffinity()
|
||||
} else {
|
||||
runBlocking { // GlobalScope.launch {
|
||||
val returns: MutableList<String>? = Scraper(user, password, app, captchaID = captchaID, captchaResponse = captchaResponse, debug = true).search()
|
||||
val returns: MutableList<Map<String, String?>>? = Scraper(
|
||||
user,
|
||||
password,
|
||||
app,
|
||||
captchaID = captchaID,
|
||||
captchaResponse = captchaResponse
|
||||
).search()
|
||||
if (returns != null) { // if job is needed
|
||||
if (returns[0] == "errorNotConnected") loginPage()
|
||||
if (returns[0] == "loginAttemptsExceeded") {
|
||||
if (returns[1] == "null") callScrapper(user, password, app)
|
||||
if (returns[0].containsKey("errorNotConnected")) {
|
||||
Toast.makeText(
|
||||
instance,
|
||||
"${getString(R.string.connectionFailed)}\n${getString(R.string.credentialsDeleted)}.",
|
||||
Toast.LENGTH_LONG
|
||||
).show()
|
||||
loginPage()
|
||||
}
|
||||
if (returns[0].containsKey("loginAttemptsExceeded")) {
|
||||
val returnsCaptchaQuestion = returns[0]["question"]
|
||||
val returnsCaptchaID = returns[0]["qaConfirmID"]
|
||||
if (returnsCaptchaQuestion == null) callScrapper(user, password, app)
|
||||
val registeredAnswered: Map<String, String> = Gson().fromJson(
|
||||
assets.open("captcha.json").bufferedReader()
|
||||
.use { it.readText() },
|
||||
object : TypeToken<Map<String, String>>() {}.type
|
||||
)
|
||||
for((key, value) in registeredAnswered){
|
||||
if (returns[1] == key) return@runBlocking callScrapper(user, password, app, returns[2], value)
|
||||
for ((key, value) in registeredAnswered) {
|
||||
if (returnsCaptchaQuestion == key) {
|
||||
println("${applicationContext().getString(R.string.knownCaptcha)}...")
|
||||
val now = System.currentTimeMillis()
|
||||
if ((now - timeOfLastToast) >= 2000) {
|
||||
Toast.makeText(
|
||||
applicationContext(),
|
||||
"${applicationContext().getString(R.string.knownCaptcha)}...",
|
||||
Toast.LENGTH_SHORT
|
||||
)
|
||||
.show()
|
||||
timeOfLastToast = now
|
||||
}
|
||||
return@runBlocking callScrapper(
|
||||
user,
|
||||
password,
|
||||
app,
|
||||
returnsCaptchaID,
|
||||
value
|
||||
)
|
||||
}
|
||||
}
|
||||
val builder: AlertDialog.Builder = AlertDialog.Builder(
|
||||
instance)
|
||||
instance
|
||||
)
|
||||
builder.setTitle("Captcha")
|
||||
builder.setMessage(returns[1])
|
||||
builder.setMessage(returnsCaptchaQuestion)
|
||||
val input = EditText(instance)
|
||||
input.inputType = InputType.TYPE_CLASS_TEXT
|
||||
builder.setView(input)
|
||||
builder.setPositiveButton(R.string.validate) { _, _ -> callScrapper(user, password, app, returns[2], input.text.toString()) }
|
||||
builder.setPositiveButton(R.string.validate) { _, _ ->
|
||||
callScrapper(
|
||||
user,
|
||||
password,
|
||||
app,
|
||||
returnsCaptchaID,
|
||||
input.text.toString()
|
||||
)
|
||||
}
|
||||
builder.setNeutralButton(R.string.cancel) { dialog, _ -> dialog.cancel() }
|
||||
builder.show()
|
||||
|
||||
Toast.makeText(instance, "${getString(R.string.connectionFailed)}\n${getString(R.string.credentialsDeleted)}.",
|
||||
Toast.LENGTH_LONG).show()
|
||||
}
|
||||
if (returns[0] == "noSID")
|
||||
if (returns[0].containsKey("noSID"))
|
||||
Toast.makeText(instance, R.string.noSID, Toast.LENGTH_SHORT).show()
|
||||
if (returns[0] == "badSID")
|
||||
Toast.makeText(instance, "${getString(R.string.badSID)}...", Toast.LENGTH_SHORT).show()
|
||||
if (returns[0] == "noResults")
|
||||
Toast.makeText(instance, "${getString(R.string.badSID)}...", Toast.LENGTH_SHORT).show()
|
||||
if (returns[0].containsKey("badSID"))
|
||||
Toast.makeText(
|
||||
instance,
|
||||
"${getString(R.string.badSID)}...",
|
||||
Toast.LENGTH_SHORT
|
||||
).show()
|
||||
if (returns[0].containsKey("noResults"))
|
||||
Toast.makeText(
|
||||
instance,
|
||||
"${getString(R.string.noResults)}...",
|
||||
Toast.LENGTH_SHORT
|
||||
).show()
|
||||
if (returns[0].containsKey("gotResults")) {
|
||||
returns.removeFirst()
|
||||
Toast.makeText(
|
||||
instance,
|
||||
"${returns.size} ${getString(R.string.gotResults)} !",
|
||||
Toast.LENGTH_SHORT
|
||||
).show()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,15 @@
|
|||
package com.mylloon.mobidl
|
||||
|
||||
import android.widget.Toast
|
||||
import com.github.kittinunf.fuel.core.FuelManager
|
||||
import com.github.kittinunf.fuel.httpGet
|
||||
import com.github.kittinunf.fuel.httpPost
|
||||
import com.mylloon.mobidl.MainActivity.Companion.applicationContext
|
||||
import java.lang.Exception
|
||||
import java.lang.System.currentTimeMillis
|
||||
import com.github.kittinunf.result.Result.Failure as FuelFailure
|
||||
import com.github.kittinunf.result.Result.Success as FuelSuccess
|
||||
|
||||
var timeOfLastToast: Long = currentTimeMillis() - 2000
|
||||
class Scraper(
|
||||
private var pseudo: String,
|
||||
private var password: String,
|
||||
private var app: String,
|
||||
private var debug: Boolean = false,
|
||||
private var captchaID: String? = null,
|
||||
private var captchaResponse: String? = null
|
||||
) {
|
||||
|
|
@ -26,17 +20,11 @@ class Scraper(
|
|||
|
||||
private var errorNotConnected = "Log me on automatically each visit"
|
||||
private var loginAttemptsExceeded = "You exceeded the maximum allowed number of login attempts."
|
||||
private var searchNotLogged = "Sorry but you are not permitted to use the search system. If you're not logged in please"
|
||||
private var searchNotLogged =
|
||||
"Sorry but you are not permitted to use the search system. If you're not logged in please"
|
||||
|
||||
private fun errorFormat( // Pretty error message.
|
||||
code: Int? = null,
|
||||
message: String = ""
|
||||
): String {
|
||||
return "${if (code != null) "[$code]" else ""}${if ((message.isNotEmpty()) and (code is Int)) " " else ""}$message."
|
||||
}
|
||||
|
||||
private fun connect(): MutableList<String>? { // Login to the forum using credentials.
|
||||
if (debug) println("Connection attempt...")
|
||||
private fun connect(): MutableList<Map<String, String?>>? { // Login to the forum using credentials.
|
||||
println("Connection attempt...")
|
||||
retry++
|
||||
FuelManager.instance.basePath = url
|
||||
FuelManager.instance.baseParams = listOf("mode" to "login")
|
||||
|
|
@ -51,7 +39,7 @@ class Scraper(
|
|||
when (result) {
|
||||
is FuelFailure -> {
|
||||
val ex = result.getException()
|
||||
println(errorFormat(response.statusCode, "Exception: $ex"))
|
||||
println("[${response.statusCode}] Exception: $ex")
|
||||
}
|
||||
is FuelSuccess -> {
|
||||
println("") // does nothing but is required otherwise everything bugs :(
|
||||
|
|
@ -65,53 +53,50 @@ class Scraper(
|
|||
return search()
|
||||
}
|
||||
|
||||
private fun error(htmlPage: String): MutableList<String>? { // Handle connection errors
|
||||
var message = ""
|
||||
var array: MutableList<String>? = null
|
||||
var lengthTime = Toast.LENGTH_LONG
|
||||
private fun error(htmlPage: String): MutableList<Map<String, String?>>? { // Handle connection errors
|
||||
var array: MutableList<Map<String, String?>>? = null
|
||||
if (errorNotConnected in htmlPage) {
|
||||
if (loginAttemptsExceeded in htmlPage) {
|
||||
message = "${applicationContext().getString(R.string.knownCaptcha)}..."
|
||||
lengthTime = Toast.LENGTH_SHORT
|
||||
val qaConfirmID = "(?<=qa_confirm_id\" value=\")(.{32})".toRegex().find(htmlPage)?.value
|
||||
val qaConfirmID =
|
||||
"(?<=qa_confirm_id\" value=\")(.{32})".toRegex().find(htmlPage)?.value
|
||||
var question = "(?<=answer\">)(.*)</l".toRegex().find(htmlPage)?.value
|
||||
question = question?.dropLast(3)
|
||||
array = if (question == null) null else mutableListOf("loginAttemptsExceeded", question.toString(), qaConfirmID.toString())
|
||||
if (question != null) {
|
||||
question = question.dropLast(3)
|
||||
}
|
||||
array = mutableListOf(
|
||||
mutableMapOf(
|
||||
"loginAttemptsExceeded" to null,
|
||||
"question" to question,
|
||||
"qaConfirmID" to qaConfirmID
|
||||
)
|
||||
)
|
||||
} else {
|
||||
message = "${applicationContext().getString(R.string.connectionFailed)}\n${applicationContext().getString(R.string.credentialsDeleted)}."
|
||||
Credentials().delete()
|
||||
array = mutableListOf("errorNotConnected")
|
||||
array = mutableListOf(mutableMapOf("errorNotConnected" to null))
|
||||
}
|
||||
}
|
||||
val now = currentTimeMillis()
|
||||
fun sendToast() {
|
||||
Toast.makeText(applicationContext(), message, lengthTime)
|
||||
.show()
|
||||
timeOfLastToast = now
|
||||
}
|
||||
println(errorFormat(message = message))
|
||||
if ((now - timeOfLastToast) >= 2000) sendToast()
|
||||
return array
|
||||
}
|
||||
|
||||
fun search(): MutableList<String>? { // Do the research.
|
||||
fun search(): MutableList<Map<String, String?>>? { // Do the research.
|
||||
if (sid == null) {
|
||||
println(errorFormat(message = "SID not found"))
|
||||
return if (retry < 3) connect() else mutableListOf("noSID")
|
||||
println("SID not found")
|
||||
return if (retry < 3) connect() else mutableListOf(mutableMapOf("noSID" to null))
|
||||
} else {
|
||||
if (debug) println("SID recovered")
|
||||
println("SID recovered")
|
||||
retry = 0
|
||||
}
|
||||
if (debug) println("Going to search page...")
|
||||
println("Going to search page...")
|
||||
FuelManager.instance.basePath = url
|
||||
FuelManager.instance.baseParams = listOf("sid" to sid, "sr" to "topics", "sf" to "titleonly")
|
||||
FuelManager.instance.baseParams =
|
||||
listOf("sid" to sid, "sr" to "topics", "sf" to "titleonly")
|
||||
val session = "/search.php"
|
||||
.httpGet(listOf("keywords" to app))
|
||||
.responseString { _, response, result ->
|
||||
when (result) {
|
||||
is FuelFailure -> {
|
||||
val ex = result.getException()
|
||||
println(errorFormat(response.statusCode, "Exception: $ex"))
|
||||
println("[${response.statusCode}] Exception: $ex")
|
||||
}
|
||||
is FuelSuccess -> {
|
||||
println("") // does nothing but is required otherwise everything bugs :(
|
||||
|
|
@ -120,50 +105,57 @@ class Scraper(
|
|||
}
|
||||
val data = session.join().data.decodeToString()
|
||||
return if (searchNotLogged in data) {
|
||||
println(errorFormat(message = "The SID doesn't work, new attempt..."))
|
||||
if (retry < 3) connect() else mutableListOf("badSID")
|
||||
println("The SID doesn't work, new attempt...")
|
||||
if (retry < 3) connect() else mutableListOf(mutableMapOf("badSID" to null))
|
||||
} else parse(data)
|
||||
}
|
||||
|
||||
private fun parse(htmlPage: String): MutableList<String>? { // Parse HTML response to a clean list.
|
||||
if (debug) println("Fetching results for $app...")
|
||||
private fun parse(htmlPage: String): MutableList<Map<String, String?>> { // Parse HTML response to a clean list.
|
||||
println("Fetching results for $app...")
|
||||
// println(htmlPage)
|
||||
if ("No suitable matches were found." in htmlPage) return mutableListOf("noResults")
|
||||
if ("No suitable matches were found." in htmlPage) return mutableListOf(mutableMapOf("noResults" to null))
|
||||
val elements: MutableList<String> = htmlPage.split("<tr>\n<td>").toMutableList()
|
||||
val finalElements = mutableListOf<Map<String, String?>>()
|
||||
elements.removeFirst()
|
||||
val lastIndex = elements.toList().lastIndex
|
||||
elements[lastIndex] = elements[lastIndex].split("</td>\n</tr>")[0]
|
||||
finalElements.add(0, mapOf("gotResults" to null))
|
||||
for (i in elements.indices) {
|
||||
var title: String?
|
||||
var author: String?
|
||||
var link: String?
|
||||
var data: String?
|
||||
title = try {
|
||||
Regex(" ?& ?").replace("", Regex("""class="topictitle">(.*)</a>""").find(elements[i])?.value!!)
|
||||
val title: String? = try {
|
||||
Regex("""class="topictitle">(.*)</a>""").findAll(elements[i])
|
||||
.map { it.groupValues[1] }.toList()[0].replace(Regex("( ?& ?)"), " & ")
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
author = try {
|
||||
val regex = """(<br />|</strong>)\n\n?<i class="icon-user"></i> by <a href="\./memberlist\.php\?mode=viewprofile&u=\d+"( style="color: #.*;" class="username-coloured")?>(.*)</a>"""
|
||||
Regex(regex).find(elements[i])?.value!!
|
||||
val author: String? = try {
|
||||
val regex =
|
||||
"""(<br />|</strong>)\n\n?<i class="icon-user"></i> by <a href="\./memberlist\.php\?mode=viewprofile&u=\d+&sid=.+"( style="color: #.*;" class="username-coloured")?>(.*)</a>"""
|
||||
Regex(regex).findAll(elements[i]).map { it.groupValues.last() }.toList()[0]
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
link = try {
|
||||
Regex("""\./viewtopic\.php\?f=(\d*)&t=(\d*)&""").find(elements[i])?.value!!
|
||||
val link: String? = try {
|
||||
val temp =
|
||||
Regex("""\./viewtopic\.php\?f=(\d*)&t=(\d*)&""").findAll(elements[i])
|
||||
"/viewtopic.php?f=${
|
||||
temp.map { it.groupValues[1] }.toList()[0]
|
||||
}&t=${temp.map { it.groupValues[2] }.toList()[0]}"
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
data = try {
|
||||
Regex("""</a> <i class="icon-time"></i> <small>(.*)</small>""").find(elements[i])?.value!!
|
||||
val date: String? = try {
|
||||
Regex("""</a> <i class="icon-time"></i> <small>(.*)</small>""").findAll(elements[i])
|
||||
.map { it.groupValues[1] }.toList()[0]
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
finalElements.add(i, mapOf("title" to title, "author" to author, "link" to link, "data" to data))
|
||||
finalElements.add(
|
||||
i + 1,
|
||||
mapOf("title" to title, "author" to author, "link" to link, "date" to date)
|
||||
)
|
||||
}
|
||||
println(finalElements)
|
||||
return null
|
||||
println("Search parsed!")
|
||||
return finalElements
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,4 +26,6 @@
|
|||
<string name="noSID">Aucun SID n\'a été trouvé</string>
|
||||
<string name="badSID">Le SID ne fonctionne pas, nouvel essaie</string>
|
||||
<string name="updateCheck">Vérification des mises à jour</string>
|
||||
<string name="noResults">Aucun résultat n\'a été trouvé</string>
|
||||
<string name="gotResults">résultats ont été trouvés</string>
|
||||
</resources>
|
||||
|
|
@ -33,4 +33,6 @@
|
|||
<string name="knownCaptcha">Resolving captchas</string>
|
||||
<string name="noSID">No SID was found</string>
|
||||
<string name="badSID">The SID does not work, new attempt</string>
|
||||
<string name="noResults">No results were found</string>
|
||||
<string name="gotResults">results were found</string>
|
||||
</resources>
|
||||
Reference in a new issue