Smarter Chapters and cleanup (#87)

* Smarter Chapters and cleanup

* Fix check
This commit is contained in:
Syer10 2021-05-18 13:52:15 -04:00 committed by GitHub
parent be930bb68b
commit e8df84416c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 131 additions and 115 deletions

View File

@ -27,96 +27,105 @@ import org.jetbrains.exposed.sql.update
object Chapter { object Chapter {
/** get chapter list when showing a manga */ /** get chapter list when showing a manga */
suspend fun getChapterList(mangaId: Int, onlineFetch: Boolean): List<ChapterDataClass> { suspend fun getChapterList(mangaId: Int, onlineFetch: Boolean?): List<ChapterDataClass> {
return if (!onlineFetch) { return if (onlineFetch == true) {
getSourceChapters(mangaId)
} else {
transaction { transaction {
ChapterTable.select { ChapterTable.manga eq mangaId }.orderBy(ChapterTable.chapterIndex to DESC) ChapterTable.select { ChapterTable.manga eq mangaId }.orderBy(ChapterTable.chapterIndex to DESC)
.map { .map {
ChapterTable.toDataClass(it) ChapterTable.toDataClass(it)
} }
}.ifEmpty {
// If it was explicitly set to offline dont grab chapters
if (onlineFetch == null) {
getSourceChapters(mangaId)
} else emptyList()
} }
} else { }
}
val mangaDetails = getManga(mangaId) private suspend fun getSourceChapters(mangaId: Int): List<ChapterDataClass> {
val source = getHttpSource(mangaDetails.sourceId.toLong()) val mangaDetails = getManga(mangaId)
val chapterList = source.fetchChapterList( val source = getHttpSource(mangaDetails.sourceId.toLong())
SManga.create().apply { val chapterList = source.fetchChapterList(
title = mangaDetails.title SManga.create().apply {
url = mangaDetails.url title = mangaDetails.title
} url = mangaDetails.url
).awaitSingle() }
).awaitSingle()
val chapterCount = chapterList.count() val chapterCount = chapterList.count()
transaction { transaction {
chapterList.reversed().forEachIndexed { index, fetchedChapter -> chapterList.reversed().forEachIndexed { index, fetchedChapter ->
val chapterEntry = ChapterTable.select { ChapterTable.url eq fetchedChapter.url }.firstOrNull() val chapterEntry = ChapterTable.select { ChapterTable.url eq fetchedChapter.url }.firstOrNull()
if (chapterEntry == null) { if (chapterEntry == null) {
ChapterTable.insert { ChapterTable.insert {
it[url] = fetchedChapter.url it[url] = fetchedChapter.url
it[name] = fetchedChapter.name it[name] = fetchedChapter.name
it[date_upload] = fetchedChapter.date_upload it[date_upload] = fetchedChapter.date_upload
it[chapter_number] = fetchedChapter.chapter_number it[chapter_number] = fetchedChapter.chapter_number
it[scanlator] = fetchedChapter.scanlator it[scanlator] = fetchedChapter.scanlator
it[chapterIndex] = index + 1 it[chapterIndex] = index + 1
it[manga] = mangaId it[manga] = mangaId
} }
} else { } else {
ChapterTable.update({ ChapterTable.url eq fetchedChapter.url }) { ChapterTable.update({ ChapterTable.url eq fetchedChapter.url }) {
it[name] = fetchedChapter.name it[name] = fetchedChapter.name
it[date_upload] = fetchedChapter.date_upload it[date_upload] = fetchedChapter.date_upload
it[chapter_number] = fetchedChapter.chapter_number it[chapter_number] = fetchedChapter.chapter_number
it[scanlator] = fetchedChapter.scanlator it[scanlator] = fetchedChapter.scanlator
it[chapterIndex] = index + 1 it[chapterIndex] = index + 1
it[manga] = mangaId it[manga] = mangaId
}
} }
} }
} }
}
// clear any orphaned chapters that are in the db but not in `chapterList` // clear any orphaned chapters that are in the db but not in `chapterList`
val dbChapterCount = transaction { ChapterTable.select { ChapterTable.manga eq mangaId }.count() } val dbChapterCount = transaction { ChapterTable.select { ChapterTable.manga eq mangaId }.count() }
if (dbChapterCount > chapterCount) { // we got some clean up due if (dbChapterCount > chapterCount) { // we got some clean up due
val dbChapterList = transaction { ChapterTable.select { ChapterTable.manga eq mangaId } } val dbChapterList = transaction { ChapterTable.select { ChapterTable.manga eq mangaId } }
dbChapterList.forEach { dbChapterList.forEach {
if (it[ChapterTable.chapterIndex] >= chapterList.size || if (it[ChapterTable.chapterIndex] >= chapterList.size ||
chapterList[it[ChapterTable.chapterIndex] - 1].url != it[ChapterTable.url] chapterList[it[ChapterTable.chapterIndex] - 1].url != it[ChapterTable.url]
) { ) {
transaction { transaction {
PageTable.deleteWhere { PageTable.chapter eq it[ChapterTable.id] } PageTable.deleteWhere { PageTable.chapter eq it[ChapterTable.id] }
ChapterTable.deleteWhere { ChapterTable.id eq it[ChapterTable.id] } ChapterTable.deleteWhere { ChapterTable.id eq it[ChapterTable.id] }
}
} }
} }
} }
}
val dbChapterMap = transaction { val dbChapterMap = transaction {
ChapterTable.select { ChapterTable.manga eq mangaId } ChapterTable.select { ChapterTable.manga eq mangaId }
.associateBy({ it[ChapterTable.url] }, { it }) .associateBy({ it[ChapterTable.url] }, { it })
} }
return chapterList.mapIndexed { index, it -> return chapterList.mapIndexed { index, it ->
val dbChapter = dbChapterMap.getValue(it.url) val dbChapter = dbChapterMap.getValue(it.url)
ChapterDataClass( ChapterDataClass(
it.url, it.url,
it.name, it.name,
it.date_upload, it.date_upload,
it.chapter_number, it.chapter_number,
it.scanlator, it.scanlator,
mangaId, mangaId,
dbChapter[ChapterTable.isRead], dbChapter[ChapterTable.isRead],
dbChapter[ChapterTable.isBookmarked], dbChapter[ChapterTable.isBookmarked],
dbChapter[ChapterTable.lastPageRead], dbChapter[ChapterTable.lastPageRead],
chapterCount - index, chapterCount - index,
) chapterList.size
} )
} }
} }
@ -125,9 +134,9 @@ object Chapter {
val chapterEntry = transaction { val chapterEntry = transaction {
ChapterTable.select { ChapterTable.select {
(ChapterTable.chapterIndex eq chapterIndex) and (ChapterTable.manga eq mangaId) (ChapterTable.chapterIndex eq chapterIndex) and (ChapterTable.manga eq mangaId)
}.firstOrNull()!! }.first()
} }
val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
val source = getHttpSource(mangaEntry[MangaTable.sourceReference]) val source = getHttpSource(mangaEntry[MangaTable.sourceReference])
val pageList = source.fetchPageList( val pageList = source.fetchPageList(

View File

@ -159,7 +159,7 @@ object Extension {
it[this.classFQName] = className it[this.classFQName] = className
} }
val extensionId = ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.firstOrNull()!![ExtensionTable.id].value val extensionId = ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.first()[ExtensionTable.id].value
sources.forEach { httpSource -> sources.forEach { httpSource ->
SourceTable.insert { SourceTable.insert {
@ -195,7 +195,7 @@ object Extension {
fun uninstallExtension(pkgName: String) { fun uninstallExtension(pkgName: String) {
logger.debug("Uninstalling $pkgName") logger.debug("Uninstalling $pkgName")
val extensionRecord = transaction { ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.firstOrNull()!! } val extensionRecord = transaction { ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.first() }
val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk") val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk")
val jarPath = "${applicationDirs.extensionsRoot}/$fileNameWithoutType.jar" val jarPath = "${applicationDirs.extensionsRoot}/$fileNameWithoutType.jar"
transaction { transaction {
@ -234,7 +234,7 @@ object Extension {
} }
suspend fun getExtensionIcon(apkName: String): Pair<InputStream, String> { suspend fun getExtensionIcon(apkName: String): Pair<InputStream, String> {
val iconUrl = transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.firstOrNull()!! }[ExtensionTable.iconUrl] val iconUrl = transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl]
val saveDir = "${applicationDirs.extensionsRoot}/icon" val saveDir = "${applicationDirs.extensionsRoot}/icon"

View File

@ -37,7 +37,7 @@ object Manga {
} }
suspend fun getManga(mangaId: Int, onlineFetch: Boolean = false): MangaDataClass { suspend fun getManga(mangaId: Int, onlineFetch: Boolean = false): MangaDataClass {
var mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } var mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
return if (mangaEntry[MangaTable.initialized] && !onlineFetch) { return if (mangaEntry[MangaTable.initialized] && !onlineFetch) {
MangaDataClass( MangaDataClass(
@ -78,14 +78,14 @@ object Manga {
it[MangaTable.description] = truncate(fetchedManga.description, 4096) it[MangaTable.description] = truncate(fetchedManga.description, 4096)
it[MangaTable.genre] = fetchedManga.genre it[MangaTable.genre] = fetchedManga.genre
it[MangaTable.status] = fetchedManga.status it[MangaTable.status] = fetchedManga.status
if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url.orEmpty().isNotEmpty())
it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
} }
} }
clearMangaThumbnail(mangaId) clearMangaThumbnail(mangaId)
mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
MangaDataClass( MangaDataClass(
mangaId, mangaId,
@ -117,7 +117,7 @@ object Manga {
return getCachedImageResponse(saveDir, fileName) { return getCachedImageResponse(saveDir, fileName) {
getManga(mangaId) // make sure is initialized getManga(mangaId) // make sure is initialized
val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
val sourceId = mangaEntry[MangaTable.sourceReference] val sourceId = mangaEntry[MangaTable.sourceReference]
val source = getHttpSource(sourceId) val source = getHttpSource(sourceId)
@ -130,7 +130,7 @@ object Manga {
} }
} }
suspend fun clearMangaThumbnail(mangaId: Int) { private fun clearMangaThumbnail(mangaId: Int) {
val saveDir = applicationDirs.thumbnailsRoot val saveDir = applicationDirs.thumbnailsRoot
val fileName = mangaId.toString() val fileName = mangaId.toString()

View File

@ -40,7 +40,7 @@ object MangaList {
val mangasPage = this val mangasPage = this
val mangaList = transaction { val mangaList = transaction {
return@transaction mangasPage.mangas.map { manga -> return@transaction mangasPage.mangas.map { manga ->
var mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull() val mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull()
if (mangaEntry == null) { // create manga entry if (mangaEntry == null) { // create manga entry
val mangaId = MangaTable.insertAndGetId { val mangaId = MangaTable.insertAndGetId {
it[url] = manga.url it[url] = manga.url

View File

@ -40,16 +40,16 @@ object Page {
} }
suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int): Pair<InputStream, String> { suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int): Pair<InputStream, String> {
val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
val source = getHttpSource(mangaEntry[MangaTable.sourceReference]) val source = getHttpSource(mangaEntry[MangaTable.sourceReference])
val chapterEntry = transaction { val chapterEntry = transaction {
ChapterTable.select { ChapterTable.select {
(ChapterTable.chapterIndex eq chapterIndex) and (ChapterTable.manga eq mangaId) (ChapterTable.chapterIndex eq chapterIndex) and (ChapterTable.manga eq mangaId)
}.firstOrNull()!! }.first()
} }
val chapterId = chapterEntry[ChapterTable.id].value val chapterId = chapterEntry[ChapterTable.id].value
val pageEntry = transaction { PageTable.select { (PageTable.chapter eq chapterId) and (PageTable.index eq index) }.firstOrNull()!! } val pageEntry = transaction { PageTable.select { (PageTable.chapter eq chapterId) and (PageTable.index eq index) }.first() }
val tachiPage = Page( val tachiPage = Page(
pageEntry[PageTable.index], pageEntry[PageTable.index],
@ -78,11 +78,11 @@ object Page {
// TODO: rewrite this to match tachiyomi // TODO: rewrite this to match tachiyomi
private val applicationDirs by DI.global.instance<ApplicationDirs>() private val applicationDirs by DI.global.instance<ApplicationDirs>()
fun getChapterDir(mangaId: Int, chapterId: Int): String { fun getChapterDir(mangaId: Int, chapterId: Int): String {
val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
val sourceId = mangaEntry[MangaTable.sourceReference] val sourceId = mangaEntry[MangaTable.sourceReference]
val source = getHttpSource(sourceId) val source = getHttpSource(sourceId)
val sourceEntry = transaction { SourceTable.select { SourceTable.id eq sourceId }.firstOrNull()!! } val sourceEntry = transaction { SourceTable.select { SourceTable.id eq sourceId }.first() }
val chapterEntry = transaction { ChapterTable.select { ChapterTable.id eq chapterId }.firstOrNull()!! } val chapterEntry = transaction { ChapterTable.select { ChapterTable.id eq chapterId }.first() }
val chapterDir = when { val chapterDir = when {
chapterEntry[ChapterTable.scanlator] != null -> "${chapterEntry[ChapterTable.scanlator]}_${chapterEntry[ChapterTable.name]}" chapterEntry[ChapterTable.scanlator] != null -> "${chapterEntry[ChapterTable.scanlator]}_${chapterEntry[ChapterTable.name]}"

View File

@ -198,7 +198,7 @@ object LegacyBackupImport : LegacyBackupBase() {
it[description] = fetchedManga.description it[description] = fetchedManga.description
it[genre] = fetchedManga.genre it[genre] = fetchedManga.genre
it[status] = fetchedManga.status it[status] = fetchedManga.status
if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url.orEmpty().isNotEmpty())
it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
} }
} }

View File

@ -10,7 +10,6 @@ package ir.armor.tachidesk.impl.util
import okhttp3.Response import okhttp3.Response
import okio.buffer import okio.buffer
import okio.sink import okio.sink
import java.io.BufferedInputStream
import java.io.File import java.io.File
import java.io.FileInputStream import java.io.FileInputStream
import java.io.InputStream import java.io.InputStream
@ -19,12 +18,12 @@ import java.nio.file.Paths
object CachedImageResponse { object CachedImageResponse {
private fun pathToInputStream(path: String): InputStream { private fun pathToInputStream(path: String): InputStream {
return BufferedInputStream(FileInputStream(path)) return FileInputStream(path).buffered()
} }
private fun findFileNameStartingWith(directoryPath: String, fileName: String): String? { private fun findFileNameStartingWith(directoryPath: String, fileName: String): String? {
val target = "$fileName." val target = "$fileName."
File(directoryPath).listFiles().forEach { file -> File(directoryPath).listFiles().orEmpty().forEach { file ->
if (file.name.startsWith(target)) if (file.name.startsWith(target))
return "$directoryPath/${file.name}" return "$directoryPath/${file.name}"
} }
@ -57,16 +56,13 @@ object CachedImageResponse {
} }
} }
} }
return Pair( return pathToInputStream(fullPath) to contentType
pathToInputStream(fullPath),
contentType
)
} else { } else {
throw Exception("request error! ${response.code}") throw Exception("request error! ${response.code}")
} }
} }
suspend fun clearCachedImage(saveDir: String, fileName: String) { fun clearCachedImage(saveDir: String, fileName: String) {
val cachedFile = findFileNameStartingWith(saveDir, fileName) val cachedFile = findFileNameStartingWith(saveDir, fileName)
cachedFile?.also { cachedFile?.also {
File(it).delete() File(it).delete()

View File

@ -32,12 +32,12 @@ object GetHttpSource {
} }
val sourceRecord = transaction { val sourceRecord = transaction {
SourceTable.select { SourceTable.id eq sourceId }.firstOrNull()!! SourceTable.select { SourceTable.id eq sourceId }.first()
} }
val extensionId = sourceRecord[SourceTable.extension] val extensionId = sourceRecord[SourceTable.extension]
val extensionRecord = transaction { val extensionRecord = transaction {
ExtensionTable.select { ExtensionTable.id eq extensionId }.firstOrNull()!! ExtensionTable.select { ExtensionTable.id eq extensionId }.first()
} }
val apkName = extensionRecord[ExtensionTable.apkName] val apkName = extensionRecord[ExtensionTable.apkName]

View File

@ -11,6 +11,7 @@ import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.Call import okhttp3.Call
import okhttp3.Callback import okhttp3.Callback
import okhttp3.Response import okhttp3.Response
import okhttp3.internal.closeQuietly
import java.io.IOException import java.io.IOException
import kotlin.coroutines.resume import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException import kotlin.coroutines.resumeWithException
@ -26,7 +27,9 @@ suspend fun Call.await(): Response {
return return
} }
continuation.resume(response) continuation.resume(response) {
response.body?.closeQuietly()
}
} }
override fun onFailure(call: Call, e: IOException) { override fun onFailure(call: Call, e: IOException) {

View File

@ -71,12 +71,14 @@ object PackageTools {
if (handler.hasException()) { if (handler.hasException()) {
val errorFile: Path = File(applicationDirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt") val errorFile: Path = File(applicationDirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt")
logger.error( logger.error(
"Detail Error Information in File $errorFile\n" + """
"Please report this file to one of following link if possible (any one).\n" + Detail Error Information in File $errorFile
" https://sourceforge.net/p/dex2jar/tickets/\n" + Please report this file to one of following link if possible (any one).
" https://bitbucket.org/pxb1988/dex2jar/issues\n" + https://sourceforge.net/p/dex2jar/tickets/
" https://github.com/pxb1988/dex2jar/issues\n" + https://bitbucket.org/pxb1988/dex2jar/issues
" dex2jar@googlegroups.com" https://github.com/pxb1988/dex2jar/issues
dex2jar@googlegroups.com
""".trimIndent()
) )
handler.dump(errorFile, emptyArray<String>()) handler.dump(errorFile, emptyArray<String>())
} }
@ -98,18 +100,21 @@ object PackageTools {
applicationInfo.metaData = Bundle().apply { applicationInfo.metaData = Bundle().apply {
val appTag = doc.getElementsByTagName("application").item(0) val appTag = doc.getElementsByTagName("application").item(0)
appTag?.childNodes?.toList()?.filter { appTag?.childNodes?.toList()
it.nodeType == Node.ELEMENT_NODE .orEmpty()
}?.map { .asSequence()
it as Element .filter {
}?.filter { it.nodeType == Node.ELEMENT_NODE
it.tagName == "meta-data" }.map {
}?.map { it as Element
putString( }.filter {
it.attributes.getNamedItem("android:name").nodeValue, it.tagName == "meta-data"
it.attributes.getNamedItem("android:value").nodeValue }.forEach {
) putString(
} it.attributes.getNamedItem("android:name").nodeValue,
it.attributes.getNamedItem("android:value").nodeValue
)
}
} }
signatures = ( signatures = (

View File

@ -25,7 +25,7 @@ data class ChapterDataClass(
val lastPageRead: Int, val lastPageRead: Int,
/** this chapter's index, starts with 1 */ /** this chapter's index, starts with 1 */
val index: Int? = null, val index: Int,
/** total chapter count, used to calculate if there's a next and prev chapter */ /** total chapter count, used to calculate if there's a next and prev chapter */
val chapterCount: Int? = null, val chapterCount: Int? = null,

View File

@ -96,7 +96,10 @@ object JavalinSetup {
logger.error("NullPointerException while handling the request", e) logger.error("NullPointerException while handling the request", e)
ctx.status(404) ctx.status(404)
} }
app.exception(NoSuchElementException::class.java) { e, ctx ->
logger.error("NoSuchElementException while handling the request", e)
ctx.status(404)
}
app.exception(IOException::class.java) { e, ctx -> app.exception(IOException::class.java) { e, ctx ->
logger.error("IOException while handling the request", e) logger.error("IOException while handling the request", e)
ctx.status(500) ctx.status(500)
@ -257,7 +260,7 @@ object JavalinSetup {
app.get("/api/v1/manga/:mangaId/chapters") { ctx -> app.get("/api/v1/manga/:mangaId/chapters") { ctx ->
val mangaId = ctx.pathParam("mangaId").toInt() val mangaId = ctx.pathParam("mangaId").toInt()
val onlineFetch = ctx.queryParam("onlineFetch", "false").toBoolean() val onlineFetch = ctx.queryParam("onlineFetch")?.toBoolean()
ctx.json(future { getChapterList(mangaId, onlineFetch) }) ctx.json(future { getChapterList(mangaId, onlineFetch) })
} }