Make MAL Tracking Slightly Less Shitty (#2042)

* * fix cookieManager not clearing cookies properly
* manually clear tracking prefs when !isLogged (e.g. cookies were cleared)

* use full url for removing cookies

* add interceptor for all non-login network calls
* attempt auto login if cookies are missing
* move handling of csrf token to interceptor

* * move methods around to improve readability
* fix TrackSearchAdapter not updating other fields if cover_url is missing
* revert accidental removal of feature in https://github.com/inorichi/tachiyomi/issues/65
* avoid login if credentials are missing

* fix eol

* *separate login flow from rxjava for reuse in sync

* *use less expensive method of finding manga

* *move variable declaration

* formatting

* set total chapters in remote track
This commit is contained in:
MCAxiaz 2019-06-09 05:31:19 -07:00 committed by inorichi
parent 8ebda219c4
commit 9ba7312caf
5 changed files with 296 additions and 202 deletions

View File

@ -10,11 +10,11 @@ import eu.kanade.tachiyomi.data.track.model.TrackSearch
import okhttp3.HttpUrl import okhttp3.HttpUrl
import rx.Completable import rx.Completable
import rx.Observable import rx.Observable
import java.lang.Exception
class Myanimelist(private val context: Context, id: Int) : TrackService(id) { class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
companion object { companion object {
const val READING = 1 const val READING = 1
const val COMPLETED = 2 const val COMPLETED = 2
const val ON_HOLD = 3 const val ON_HOLD = 3
@ -29,7 +29,8 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
const val LOGGED_IN_COOKIE = "is_logged_in" const val LOGGED_IN_COOKIE = "is_logged_in"
} }
private val api by lazy { MyanimelistApi(client) } private val interceptor by lazy { MyAnimeListInterceptor(this) }
private val api by lazy { MyanimelistApi(client, interceptor) }
override val name: String override val name: String
get() = "MyAnimeList" get() = "MyAnimeList"
@ -62,7 +63,7 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
} }
override fun add(track: Track): Observable<Track> { override fun add(track: Track): Observable<Track> {
return api.addLibManga(track, getCSRF()) return api.addLibManga(track)
} }
override fun update(track: Track): Observable<Track> { override fun update(track: Track): Observable<Track> {
@ -70,11 +71,11 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
track.status = COMPLETED track.status = COMPLETED
} }
return api.updateLibManga(track, getCSRF()) return api.updateLibManga(track)
} }
override fun bind(track: Track): Observable<Track> { override fun bind(track: Track): Observable<Track> {
return api.findLibManga(track, getCSRF()) return api.findLibManga(track)
.flatMap { remoteTrack -> .flatMap { remoteTrack ->
if (remoteTrack != null) { if (remoteTrack != null) {
track.copyPersonalFrom(remoteTrack) track.copyPersonalFrom(remoteTrack)
@ -93,7 +94,7 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
} }
override fun refresh(track: Track): Observable<Track> { override fun refresh(track: Track): Observable<Track> {
return api.getLibManga(track, getCSRF()) return api.getLibManga(track)
.map { remoteTrack -> .map { remoteTrack ->
track.copyPersonalFrom(remoteTrack) track.copyPersonalFrom(remoteTrack)
track.total_chapters = remoteTrack.total_chapters track.total_chapters = remoteTrack.total_chapters
@ -104,26 +105,44 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
override fun login(username: String, password: String): Completable { override fun login(username: String, password: String): Completable {
logout() logout()
return api.login(username, password) return Observable.fromCallable { api.login(username, password) }
.doOnNext { csrf -> saveCSRF(csrf) } .doOnNext { csrf -> saveCSRF(csrf) }
.doOnNext { saveCredentials(username, password) } .doOnNext { saveCredentials(username, password) }
.doOnError { logout() } .doOnError { logout() }
.toCompletable() .toCompletable()
} }
// Attempt to login again if cookies have been cleared but credentials are still filled
fun ensureLoggedIn() {
if (isAuthorized) return
if (!isLogged) throw Exception("MAL Login Credentials not found")
val username = getUsername()
val password = getPassword()
logout()
try {
val csrf = api.login(username, password)
saveCSRF(csrf)
saveCredentials(username, password)
} catch (e: Exception) {
logout()
throw e
}
}
override fun logout() { override fun logout() {
super.logout() super.logout()
preferences.trackToken(this).delete() preferences.trackToken(this).delete()
networkService.cookieManager.remove(HttpUrl.parse(BASE_URL)!!) networkService.cookieManager.remove(HttpUrl.parse(BASE_URL)!!)
} }
override val isLogged: Boolean val isAuthorized: Boolean
get() = !getUsername().isEmpty() && get() = super.isLogged &&
!getPassword().isEmpty() && getCSRF().isNotEmpty() &&
checkCookies() && checkCookies()
!getCSRF().isEmpty()
private fun getCSRF(): String = preferences.trackToken(this).getOrDefault() fun getCSRF(): String = preferences.trackToken(this).getOrDefault()
private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf) private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf)

View File

@ -0,0 +1,49 @@
package eu.kanade.tachiyomi.data.track.myanimelist
import okhttp3.Interceptor
import okhttp3.RequestBody
import okhttp3.Response
import okio.Buffer
import org.json.JSONObject
class MyAnimeListInterceptor(private val myanimelist: Myanimelist): Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
myanimelist.ensureLoggedIn()
var request = chain.request()
request.body()?.let {
val contentType = it.contentType().toString()
val updatedBody = when {
contentType.contains("x-www-form-urlencoded") -> updateFormBody(it)
contentType.contains("json") -> updateJsonBody(it)
else -> it
}
request = request.newBuilder().post(updatedBody).build()
}
return chain.proceed(request)
}
private fun bodyToString(requestBody: RequestBody): String {
Buffer().use {
requestBody.writeTo(it)
return it.readUtf8()
}
}
private fun updateFormBody(requestBody: RequestBody): RequestBody {
val formString = bodyToString(requestBody)
return RequestBody.create(requestBody.contentType(),
"$formString${if (formString.isNotEmpty()) "&" else ""}${MyanimelistApi.CSRF}=${myanimelist.getCSRF()}")
}
private fun updateJsonBody(requestBody: RequestBody): RequestBody {
val jsonString = bodyToString(requestBody)
val newBody = JSONObject(jsonString)
.put(MyanimelistApi.CSRF, myanimelist.getCSRF())
return RequestBody.create(requestBody.contentType(), newBody.toString())
}
}

View File

@ -22,61 +22,122 @@ import java.io.InputStreamReader
import java.util.zip.GZIPInputStream import java.util.zip.GZIPInputStream
class MyanimelistApi(private val client: OkHttpClient) { class MyanimelistApi(private val client: OkHttpClient, interceptor: MyAnimeListInterceptor) {
fun addLibManga(track: Track, csrf: String): Observable<Track> { private val authClient = client.newBuilder().addInterceptor(interceptor).build()
return Observable.defer {
client.newCall(POST(url = getAddUrl(), body = getMangaPostPayload(track, csrf)))
.asObservableSuccess()
.map { track }
}
}
fun updateLibManga(track: Track, csrf: String): Observable<Track> {
return Observable.defer {
client.newCall(POST(url = getUpdateUrl(), body = getMangaPostPayload(track, csrf)))
.asObservableSuccess()
.map { track }
}
}
fun search(query: String): Observable<List<TrackSearch>> { fun search(query: String): Observable<List<TrackSearch>> {
return client.newCall(GET(getSearchUrl(query))) return if (query.startsWith(PREFIX_MY)) {
.asObservable() val realQuery = query.removePrefix(PREFIX_MY)
.flatMap { response -> getList()
Observable.from(Jsoup.parse(response.consumeBody()) .flatMap { Observable.from(it) }
.select("div.js-categories-seasonal.js-block-list.list") .filter { it.title.contains(realQuery, true) }
.select("table").select("tbody") .toList()
.select("tr").drop(1)) }
} else {
.filter { row -> client.newCall(GET(searchUrl(query)))
row.select(TD)[2].text() != "Novel" .asObservable()
} .flatMap { response ->
.map { row -> Observable.from(Jsoup.parse(response.consumeBody())
TrackSearch.create(TrackManager.MYANIMELIST).apply { .select("div.js-categories-seasonal.js-block-list.list")
title = row.searchTitle() .select("table").select("tbody")
media_id = row.searchMediaId() .select("tr").drop(1))
total_chapters = row.searchTotalChapters()
summary = row.searchSummary()
cover_url = row.searchCoverUrl()
tracking_url = mangaUrl(media_id)
publishing_status = row.searchPublishingStatus()
publishing_type = row.searchPublishingType()
start_date = row.searchStartDate()
} }
} .filter { row ->
.toList() row.select(TD)[2].text() != "Novel"
}
.map { row ->
TrackSearch.create(TrackManager.MYANIMELIST).apply {
title = row.searchTitle()
media_id = row.searchMediaId()
total_chapters = row.searchTotalChapters()
summary = row.searchSummary()
cover_url = row.searchCoverUrl()
tracking_url = mangaUrl(media_id)
publishing_status = row.searchPublishingStatus()
publishing_type = row.searchPublishingType()
start_date = row.searchStartDate()
}
}
.toList()
}
} }
private fun getList(csrf: String): Observable<List<TrackSearch>> { fun addLibManga(track: Track): Observable<Track> {
return getListUrl(csrf) return Observable.defer {
authClient.newCall(POST(url = addUrl(), body = mangaPostPayload(track)))
.asObservableSuccess()
.map { track }
}
}
fun updateLibManga(track: Track): Observable<Track> {
return Observable.defer {
authClient.newCall(POST(url = updateUrl(), body = mangaPostPayload(track)))
.asObservableSuccess()
.map { track }
}
}
fun findLibManga(track: Track): Observable<Track?> {
return authClient.newCall(GET(url = listEntryUrl(track.media_id)))
.asObservable()
.map {response ->
var libTrack: Track? = null
response.use {
if (it.priorResponse()?.isRedirect != true) {
val trackForm = Jsoup.parse(it.consumeBody())
libTrack = Track.create(TrackManager.MYANIMELIST).apply {
last_chapter_read = trackForm.select("#add_manga_num_read_chapters").`val`().toInt()
total_chapters = trackForm.select("#totalChap").text().toInt()
status = trackForm.select("#add_manga_status > option[selected]").`val`().toInt()
score = trackForm.select("#add_manga_score > option[selected]").`val`().toFloatOrNull() ?: 0f
}
}
}
libTrack
}
}
fun getLibManga(track: Track): Observable<Track> {
return findLibManga(track)
.map { it ?: throw Exception("Could not find manga") }
}
fun login(username: String, password: String): String {
val csrf = getSessionInfo()
login(username, password, csrf)
return csrf
}
private fun getSessionInfo(): String {
val response = client.newCall(GET(loginUrl())).execute()
return Jsoup.parse(response.consumeBody())
.select("meta[name=csrf_token]")
.attr("content")
}
private fun login(username: String, password: String, csrf: String) {
val response = client.newCall(POST(url = loginUrl(), body = loginPostBody(username, password, csrf))).execute()
response.use {
if (response.priorResponse()?.code() != 302) throw Exception("Authentication error")
}
}
private fun getList(): Observable<List<TrackSearch>> {
return getListUrl()
.flatMap { url -> .flatMap { url ->
getListXml(url) getListXml(url)
} }
.flatMap { doc -> .flatMap { doc ->
Observable.from(doc.select("manga")) Observable.from(doc.select("manga"))
} }
.map { it -> .map {
TrackSearch.create(TrackManager.MYANIMELIST).apply { TrackSearch.create(TrackManager.MYANIMELIST).apply {
title = it.selectText("manga_title")!! title = it.selectText("manga_title")!!
media_id = it.selectInt("manga_mangadb_id") media_id = it.selectInt("manga_mangadb_id")
@ -90,107 +151,8 @@ class MyanimelistApi(private val client: OkHttpClient) {
.toList() .toList()
} }
private fun getListXml(url: String): Observable<Document> { private fun getListUrl(): Observable<String> {
return client.newCall(GET(url)) return authClient.newCall(POST(url = exportListUrl(), body = exportPostBody()))
.asObservable()
.map { response ->
Jsoup.parse(response.consumeXmlBody(), "", Parser.xmlParser())
}
}
fun findLibManga(track: Track, csrf: String): Observable<Track?> {
return getList(csrf)
.map { list -> list.find { it.media_id == track.media_id } }
}
fun getLibManga(track: Track, csrf: String): Observable<Track> {
return findLibManga(track, csrf)
.map { it ?: throw Exception("Could not find manga") }
}
fun login(username: String, password: String): Observable<String> {
return getSessionInfo()
.flatMap { csrf ->
login(username, password, csrf)
}
}
private fun getSessionInfo(): Observable<String> {
return client.newCall(GET(getLoginUrl()))
.asObservable()
.map { response ->
Jsoup.parse(response.consumeBody())
.select("meta[name=csrf_token]")
.attr("content")
}
}
private fun login(username: String, password: String, csrf: String): Observable<String> {
return client.newCall(POST(url = getLoginUrl(), body = getLoginPostBody(username, password, csrf)))
.asObservable()
.map { response ->
response.use {
if (response.priorResponse()?.code() != 302) throw Exception("Authentication error")
}
csrf
}
}
private fun getLoginPostBody(username: String, password: String, csrf: String): RequestBody {
return FormBody.Builder()
.add("user_name", username)
.add("password", password)
.add("cookie", "1")
.add("sublogin", "Login")
.add("submit", "1")
.add(CSRF, csrf)
.build()
}
private fun getExportPostBody(csrf: String): RequestBody {
return FormBody.Builder()
.add("type", "2")
.add("subexport", "Export My List")
.add(CSRF, csrf)
.build()
}
private fun getMangaPostPayload(track: Track, csrf: String): RequestBody {
val body = JSONObject()
.put("manga_id", track.media_id)
.put("status", track.status)
.put("score", track.score)
.put("num_read_chapters", track.last_chapter_read)
.put(CSRF, csrf)
return RequestBody.create(MediaType.parse("application/json; charset=utf-8"), body.toString())
}
private fun getLoginUrl() = Uri.parse(baseUrl).buildUpon()
.appendPath("login.php")
.toString()
private fun getSearchUrl(query: String): String {
val col = "c[]"
return Uri.parse(baseUrl).buildUpon()
.appendPath("manga.php")
.appendQueryParameter("q", query)
.appendQueryParameter(col, "a")
.appendQueryParameter(col, "b")
.appendQueryParameter(col, "c")
.appendQueryParameter(col, "d")
.appendQueryParameter(col, "e")
.appendQueryParameter(col, "g")
.toString()
}
private fun getExportListUrl() = Uri.parse(baseUrl).buildUpon()
.appendPath("panel.php")
.appendQueryParameter("go", "export")
.toString()
private fun getListUrl(csrf: String): Observable<String> {
return client.newCall(POST(url = getExportListUrl(), body = getExportPostBody(csrf)))
.asObservable() .asObservable()
.map {response -> .map {response ->
baseUrl + Jsoup.parse(response.consumeBody()) baseUrl + Jsoup.parse(response.consumeBody())
@ -200,17 +162,17 @@ class MyanimelistApi(private val client: OkHttpClient) {
} }
} }
private fun getUpdateUrl() = Uri.parse(baseModifyListUrl).buildUpon() private fun getListXml(url: String): Observable<Document> {
.appendPath("edit.json") return authClient.newCall(GET(url))
.toString() .asObservable()
.map { response ->
private fun getAddUrl() = Uri.parse(baseModifyListUrl).buildUpon() Jsoup.parse(response.consumeXmlBody(), "", Parser.xmlParser())
.appendPath( "add.json") }
.toString() }
private fun Response.consumeBody(): String? { private fun Response.consumeBody(): String? {
use { use {
if (it.code() != 200) throw Exception("Login error") if (it.code() != 200) throw Exception("HTTP error ${it.code()}")
return it.body()?.string() return it.body()?.string()
} }
} }
@ -229,37 +191,105 @@ class MyanimelistApi(private val client: OkHttpClient) {
} }
companion object { companion object {
const val baseUrl = "https://myanimelist.net" const val CSRF = "csrf_token"
private const val baseUrl = "https://myanimelist.net"
private const val baseMangaUrl = "$baseUrl/manga/" private const val baseMangaUrl = "$baseUrl/manga/"
private const val baseModifyListUrl = "$baseUrl/ownlist/manga" private const val baseModifyListUrl = "$baseUrl/ownlist/manga"
private const val PREFIX_MY = "my:"
private const val TD = "td"
fun mangaUrl(remoteId: Int) = baseMangaUrl + remoteId private fun mangaUrl(remoteId: Int) = baseMangaUrl + remoteId
fun Element.searchTitle() = select("strong").text()!! private fun loginUrl() = Uri.parse(baseUrl).buildUpon()
.appendPath("login.php")
.toString()
fun Element.searchTotalChapters() = if (select(TD)[4].text() == "-") 0 else select(TD)[4].text().toInt() private fun searchUrl(query: String): String {
val col = "c[]"
return Uri.parse(baseUrl).buildUpon()
.appendPath("manga.php")
.appendQueryParameter("q", query)
.appendQueryParameter(col, "a")
.appendQueryParameter(col, "b")
.appendQueryParameter(col, "c")
.appendQueryParameter(col, "d")
.appendQueryParameter(col, "e")
.appendQueryParameter(col, "g")
.toString()
}
fun Element.searchCoverUrl() = select("img") private fun exportListUrl() = Uri.parse(baseUrl).buildUpon()
.appendPath("panel.php")
.appendQueryParameter("go", "export")
.toString()
private fun updateUrl() = Uri.parse(baseModifyListUrl).buildUpon()
.appendPath("edit.json")
.toString()
private fun addUrl() = Uri.parse(baseModifyListUrl).buildUpon()
.appendPath( "add.json")
.toString()
private fun listEntryUrl(mediaId: Int) = Uri.parse(baseModifyListUrl).buildUpon()
.appendPath(mediaId.toString())
.appendPath("edit")
.toString()
private fun loginPostBody(username: String, password: String, csrf: String): RequestBody {
return FormBody.Builder()
.add("user_name", username)
.add("password", password)
.add("cookie", "1")
.add("sublogin", "Login")
.add("submit", "1")
.add(CSRF, csrf)
.build()
}
private fun exportPostBody(): RequestBody {
return FormBody.Builder()
.add("type", "2")
.add("subexport", "Export My List")
.build()
}
private fun mangaPostPayload(track: Track): RequestBody {
val body = JSONObject()
.put("manga_id", track.media_id)
.put("status", track.status)
.put("score", track.score)
.put("num_read_chapters", track.last_chapter_read)
return RequestBody.create(MediaType.parse("application/json; charset=utf-8"), body.toString())
}
private fun Element.searchTitle() = select("strong").text()!!
private fun Element.searchTotalChapters() = if (select(TD)[4].text() == "-") 0 else select(TD)[4].text().toInt()
private fun Element.searchCoverUrl() = select("img")
.attr("data-src") .attr("data-src")
.split("\\?")[0] .split("\\?")[0]
.replace("/r/50x70/", "/") .replace("/r/50x70/", "/")
fun Element.searchMediaId() = select("div.picSurround") private fun Element.searchMediaId() = select("div.picSurround")
.select("a").attr("id") .select("a").attr("id")
.replace("sarea", "") .replace("sarea", "")
.toInt() .toInt()
fun Element.searchSummary() = select("div.pt4") private fun Element.searchSummary() = select("div.pt4")
.first() .first()
.ownText()!! .ownText()!!
fun Element.searchPublishingStatus() = if (select(TD).last().text() == "-") PUBLISHING else FINISHED private fun Element.searchPublishingStatus() = if (select(TD).last().text() == "-") "Publishing" else "Finished"
fun Element.searchPublishingType() = select(TD)[2].text()!! private fun Element.searchPublishingType() = select(TD)[2].text()!!
fun Element.searchStartDate() = select(TD)[6].text()!! private fun Element.searchStartDate() = select(TD)[6].text()!!
fun getStatus(status: String) = when (status) { private fun getStatus(status: String) = when (status) {
"Reading" -> 1 "Reading" -> 1
"Completed" -> 2 "Completed" -> 2
"On-Hold" -> 3 "On-Hold" -> 3
@ -267,10 +297,5 @@ class MyanimelistApi(private val client: OkHttpClient) {
"Plan to Read" -> 6 "Plan to Read" -> 6
else -> 1 else -> 1
} }
const val CSRF = "csrf_token"
const val TD = "td"
private const val FINISHED = "Finished"
private const val PUBLISHING = "Publishing"
} }
} }

View File

@ -47,11 +47,12 @@ class AndroidCookieJar(context: Context) : CookieJar {
} }
fun remove(url: HttpUrl) { fun remove(url: HttpUrl) {
val cookies = manager.getCookie(url.toString()) ?: return val urlString = url.toString()
val domain = ".${url.host()}" val cookies = manager.getCookie(urlString) ?: return
cookies.split(";") cookies.split(";")
.map { it.substringBefore("=") } .map { it.substringBefore("=") }
.onEach { manager.setCookie(domain, "$it=;Max-Age=-1") } .onEach { manager.setCookie(urlString, "$it=;Max-Age=-1") }
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
syncManager.sync() syncManager.sync()

View File

@ -52,27 +52,27 @@ class TrackSearchAdapter(context: Context)
.diskCacheStrategy(DiskCacheStrategy.RESOURCE) .diskCacheStrategy(DiskCacheStrategy.RESOURCE)
.centerCrop() .centerCrop()
.into(view.track_search_cover) .into(view.track_search_cover)
}
if (track.publishing_status.isNullOrBlank()) { if (track.publishing_status.isNullOrBlank()) {
view.track_search_status.gone() view.track_search_status.gone()
view.track_search_status_result.gone() view.track_search_status_result.gone()
} else { } else {
view.track_search_status_result.text = track.publishing_status.capitalize() view.track_search_status_result.text = track.publishing_status.capitalize()
} }
if (track.publishing_type.isNullOrBlank()) { if (track.publishing_type.isNullOrBlank()) {
view.track_search_type.gone() view.track_search_type.gone()
view.track_search_type_result.gone() view.track_search_type_result.gone()
} else { } else {
view.track_search_type_result.text = track.publishing_type.capitalize() view.track_search_type_result.text = track.publishing_type.capitalize()
} }
if (track.start_date.isNullOrBlank()) { if (track.start_date.isNullOrBlank()) {
view.track_search_start.gone() view.track_search_start.gone()
view.track_search_start_result.gone() view.track_search_start_result.gone()
} else { } else {
view.track_search_start_result.text = track.start_date view.track_search_start_result.text = track.start_date
}
} }
} }
} }