Mirrors/tachiyomi-extensions-inspector
2
0
Fork 0
mirror of https://github.com/tachiyomiorg/tachiyomi-extensions-inspector.git synced 2024-10-31 22:45:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add initial testing suit

Browse Source
This commit is contained in:
Syer10 2021-04-03 16:42:13 -04:00
parent c8a8ce07e2
commit c0df7d314b
15 changed files with 327 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -7,3 +7,5 @@ gradle.properties
build
server/src/main/resources/react
server/tmp/
server/tachiserver-data/

1
build.gradle.kts
View File

@ -53,6 +53,7 @@ configure(projects) {
val coroutinesVersion = "1.4.2"
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:$coroutinesVersion")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion")
// Dependency Injection

6
server/build.gradle.kts
View File

@ -80,6 +80,9 @@ dependencies {
// uncomment to test extensions directly
// implementation(fileTree("lib/"))
// Testing
testImplementation(kotlin("test-junit5"))
}
val name = "ir.armor.tachidesk.Main"
@ -137,6 +140,9 @@ tasks {
)
}
}
test {
useJUnit()
}
}
launch4j { //used for windows

13
server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt
View File

@ -39,12 +39,17 @@ import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import uy.kohesive.injekt.injectLazy
import java.io.File
import java.io.InputStream
object Extension {
private val logger = KotlinLogging.logger {}
private val dirs by DI.global.instance<ApplicationDirs>()
data class InstallableAPK(
val apkFilePath: String,
@ -58,7 +63,7 @@ object Extension {
return installAPK {
val apkURL = ExtensionGithubApi.getApkUrl(extensionRecord)
val apkName = Uri.parse(apkURL).lastPathSegment!!
val apkSavePath = "${ApplicationDirs.extensionsRoot}/$apkName"
val apkSavePath = "${dirs.extensionsRoot}/$apkName"
// download apk file
downloadAPKFile(apkURL, apkSavePath)
@ -79,7 +84,7 @@ object Extension {
if (!isInstalled) {
val fileNameWithoutType = apkName.substringBefore(".apk")
val dirPathWithoutType = "${ApplicationDirs.extensionsRoot}/$fileNameWithoutType"
val dirPathWithoutType = "${dirs.extensionsRoot}/$fileNameWithoutType"
val jarFilePath = "$dirPathWithoutType.jar"
val dexFilePath = "$dirPathWithoutType.dex"
@ -193,7 +198,7 @@ object Extension {
val extensionRecord = transaction { ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.firstOrNull()!! }
val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk")
val jarPath = "${ApplicationDirs.extensionsRoot}/$fileNameWithoutType.jar"
val jarPath = "${dirs.extensionsRoot}/$fileNameWithoutType.jar"
transaction {
val extensionId = extensionRecord[ExtensionTable.id].value
@ -232,7 +237,7 @@ object Extension {
suspend fun getExtensionIcon(apkName: String): Pair<InputStream, String> {
val iconUrl = transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.firstOrNull()!! }[ExtensionTable.iconUrl]
val saveDir = "${ApplicationDirs.extensionsRoot}/icon"
val saveDir = "${dirs.extensionsRoot}/icon"
return getCachedImageResponse(saveDir, apkName) {
network.client.newCall(

6
server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt
View File

@ -22,6 +22,9 @@ import ir.armor.tachidesk.server.ApplicationDirs
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import java.io.InputStream
object Manga {
@ -95,9 +98,10 @@ object Manga {
}
}
private val dirs by DI.global.instance<ApplicationDirs>()
suspend fun getMangaThumbnail(mangaId: Int): Pair<InputStream, String> {
val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! }
val saveDir = ApplicationDirs.thumbnailsRoot
val saveDir = dirs.thumbnailsRoot
val fileName = mangaId.toString()
return getCachedImageResponse(saveDir, fileName) {

6
server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt
View File

@ -21,6 +21,9 @@ import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import java.io.File
import java.io.InputStream
@ -73,6 +76,7 @@ object Page {
}
// TODO: rewrite this to match tachiyomi
private val dirs by DI.global.instance<ApplicationDirs>()
fun getChapterDir(mangaId: Int, chapterId: Int): String {
val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! }
val sourceId = mangaEntry[MangaTable.sourceReference]
@ -88,7 +92,7 @@ object Page {
val mangaTitle = mangaEntry[MangaTable.title]
val sourceName = source.toString()
val mangaDir = "${ApplicationDirs.mangaRoot}/$sourceName/$mangaTitle/$chapterDir"
val mangaDir = "${dirs.mangaRoot}/$sourceName/$mangaTitle/$chapterDir"
// make sure dirs exist
File(mangaDir).mkdirs()
return mangaDir

4
server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt
View File

@ -24,7 +24,7 @@ object Source {
return transaction {
SourceTable.selectAll().map {
SourceDataClass(
it[SourceTable.id].value.toString(),
it[SourceTable.id].value,
it[SourceTable.name],
it[SourceTable.lang],
getExtensionIconUrl(ExtensionTable.select { ExtensionTable.id eq it[SourceTable.extension] }.first()[ExtensionTable.apkName]),
@ -39,7 +39,7 @@ object Source {
val source = SourceTable.select { SourceTable.id eq sourceId }.firstOrNull()
SourceDataClass(
sourceId.toString(),
sourceId,
source?.get(SourceTable.name),
source?.get(SourceTable.lang),
source?.let { ExtensionTable.select { ExtensionTable.id eq source[SourceTable.extension] }.first()[ExtensionTable.iconUrl] },

6
server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt
View File

@ -16,10 +16,14 @@ import ir.armor.tachidesk.model.database.SourceTable
import ir.armor.tachidesk.server.ApplicationDirs
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import java.util.concurrent.ConcurrentHashMap
object GetHttpSource {
private val sourceCache = ConcurrentHashMap<Long, HttpSource>()
private val dirs by DI.global.instance<ApplicationDirs>()
fun getHttpSource(sourceId: Long): HttpSource {
val cachedResult: HttpSource? = sourceCache[sourceId]
@ -39,7 +43,7 @@ object GetHttpSource {
val apkName = extensionRecord[ExtensionTable.apkName]
val className = extensionRecord[ExtensionTable.classFQName]
val jarName = apkName.substringBefore(".apk") + ".jar"
val jarPath = "${ApplicationDirs.extensionsRoot}/$jarName"
val jarPath = "${dirs.extensionsRoot}/$jarName"
when (val instance = loadExtensionSources(jarPath, className)) {
is Source -> listOf(instance)

6
server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt
View File

@ -11,6 +11,9 @@ import ir.armor.tachidesk.server.ApplicationDirs
import mu.KotlinLogging
import net.dongliu.apk.parser.ApkFile
import net.dongliu.apk.parser.ApkParsers
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import org.w3c.dom.Element
import org.w3c.dom.Node
import xyz.nulldev.androidcompat.pm.InstalledPackage.Companion.toList
@ -31,6 +34,7 @@ import javax.xml.parsers.DocumentBuilderFactory
object PackageTools {
private val logger = KotlinLogging.logger {}
private val dirs by DI.global.instance<ApplicationDirs>()
const val EXTENSION_FEATURE = "tachiyomi.extension"
const val METADATA_SOURCE_CLASS = "tachiyomi.extension.class"
@ -65,7 +69,7 @@ object PackageTools {
.skipExceptions(false)
.to(jarFilePath)
if (handler.hasException()) {
val errorFile: Path = File(ApplicationDirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt")
val errorFile: Path = File(dirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt")
logger.error(
"Detail Error Information in File $errorFile\n" +
"Please report this file to one of following link if possible (any one).\n" +

6
server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt
View File

@ -18,10 +18,14 @@ import ir.armor.tachidesk.server.ApplicationDirs
import org.jetbrains.exposed.sql.Database
import org.jetbrains.exposed.sql.SchemaUtils
import org.jetbrains.exposed.sql.transactions.transaction
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
object DBMangaer {
val db by lazy {
Database.connect("jdbc:h2:${ApplicationDirs.dataRoot}/database", "org.h2.Driver")
val dirs by DI.global.instance<ApplicationDirs>()
Database.connect("jdbc:h2:${dirs.dataRoot}/database", "org.h2.Driver")
}
}

2
server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt
View File

@ -8,7 +8,7 @@ package ir.armor.tachidesk.model.dataclass
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
data class SourceDataClass(
val id: String,
val id: Long,
val name: String?,
val lang: String?,
val iconUrl: String?,

63
server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt
View File

@ -8,6 +8,7 @@ package ir.armor.tachidesk.server
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import ch.qos.logback.classic.Level
import com.typesafe.config.Config
import eu.kanade.tachiyomi.App
import ir.armor.tachidesk.Main
import ir.armor.tachidesk.model.dataclass.makeDataBaseTables
@ -15,7 +16,11 @@ import ir.armor.tachidesk.server.util.systemTray
import mu.KotlinLogging
import net.harawata.appdirs.AppDirsFactory
import org.kodein.di.DI
import org.kodein.di.bind
import org.kodein.di.conf.global
import org.kodein.di.instance
import org.kodein.di.singleton
import org.slf4j.Logger
import xyz.nulldev.androidcompat.AndroidCompat
import xyz.nulldev.androidcompat.AndroidCompatInitializer
import xyz.nulldev.ts.config.ConfigKodeinModule
@ -24,43 +29,57 @@ import java.io.File
private val logger = KotlinLogging.logger {}
object ApplicationDirs {
val dataRoot = AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)!!
class ApplicationDirs(
val dataRoot: String = AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)
) {
val extensionsRoot = "$dataRoot/extensions"
val thumbnailsRoot = "$dataRoot/thumbnails"
val mangaRoot = "$dataRoot/manga"
}
val serverConfig: ServerConfig by lazy { GlobalConfigManager.module() }
val serverConfig: ServerConfig by DI.global.instance()
val systemTray by lazy { systemTray() }
val androidCompat by lazy { AndroidCompat() }
fun applicationSetup() {
// register server config
GlobalConfigManager.registerModule(
ServerConfig.register(GlobalConfigManager.config)
)
// set application wide logging level
if (serverConfig.debugLogsEnabled) {
(mu.KotlinLogging.logger(org.slf4j.Logger.ROOT_LOGGER_NAME).underlyingLogger as ch.qos.logback.classic.Logger).level = Level.DEBUG
fun applicationSetup(rootDir: String? = null, config: Config = GlobalConfigManager.config) {
val dirs = if (rootDir != null) {
ApplicationDirs(rootDir)
} else {
ApplicationDirs()
}
// make dirs we need
listOf(
ApplicationDirs.dataRoot,
ApplicationDirs.extensionsRoot,
"${ApplicationDirs.extensionsRoot}/icon",
ApplicationDirs.thumbnailsRoot
dirs.dataRoot,
dirs.extensionsRoot,
dirs.extensionsRoot + "/icon",
dirs.thumbnailsRoot
).forEach {
File(it).mkdirs()
}
// Application dirs
DI.global.addImport(DI.Module("Server") {
bind<ApplicationDirs>() with singleton { dirs }
bind<ServerConfig>() with singleton { ServerConfig.register(config) }
})
// Load config API
DI.global.addImport(ConfigKodeinModule().create())
// Load Android compatibility dependencies
AndroidCompatInitializer().init()
// start app
androidCompat.startApp(App())
// set application wide logging level
if (serverConfig.debugLogsEnabled) {
(KotlinLogging.logger(Logger.ROOT_LOGGER_NAME).underlyingLogger as ch.qos.logback.classic.Logger).level = Level.DEBUG
}
// create conf file if doesn't exist
try {
val dataConfFile = File("${ApplicationDirs.dataRoot}/server.conf")
val dataConfFile = File("${dirs.dataRoot}/server.conf")
if (!dataConfFile.exists()) {
Main::class.java.getResourceAsStream("/server-reference.conf").use { input ->
dataConfFile.outputStream().use { output ->
@ -75,19 +94,13 @@ fun applicationSetup() {
makeDataBaseTables()
// create system tray
if (serverConfig.systemTrayEnabled)
if (serverConfig.systemTrayEnabled) {
try {
systemTray
} catch (e: Exception) {
e.printStackTrace()
}
// Load config API
DI.global.addImport(ConfigKodeinModule().create())
// Load Android compatibility dependencies
AndroidCompatInitializer().init()
// start app
androidCompat.startApp(App())
}
// Disable jetty's logging
System.setProperty("org.eclipse.jetty.util.log.announce", "false")

186
server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt Normal file
View File

@ -0,0 +1,186 @@
package ir.armor.tachidesk
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import ir.armor.tachidesk.impl.Extension.installExtension
import ir.armor.tachidesk.impl.Extension.uninstallExtension
import ir.armor.tachidesk.impl.Extension.updateExtension
import ir.armor.tachidesk.impl.ExtensionsList.getExtensionList
import ir.armor.tachidesk.impl.Source.getSourceList
import ir.armor.tachidesk.impl.util.GetHttpSource.getHttpSource
import ir.armor.tachidesk.impl.util.awaitSingle
import ir.armor.tachidesk.model.dataclass.ExtensionDataClass
import ir.armor.tachidesk.server.applicationSetup
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import mu.KotlinLogging
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import rx.Observable
import java.io.File
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestExtensions {
private val logger = KotlinLogging.logger {}
private lateinit var extensions: List<ExtensionDataClass>
private lateinit var sources: List<HttpSource>
private val mangaToFetch = mutableListOf<Pair<HttpSource, SManga>>()
private val failedToFetch = mutableListOf<Pair<HttpSource, Exception>>()
private val mangaFailedToFetch = mutableListOf<Triple<HttpSource, SManga, Exception>>()
private val chaptersToFetch = mutableListOf<Triple<HttpSource, SManga, SChapter>>()
private val chaptersFailedToFetch = mutableListOf<Triple<HttpSource, SManga, Throwable>>()
private val chaptersPageListFailedToFetch = mutableListOf<Triple<HttpSource, Pair<SManga, SChapter>, Exception>>()
@BeforeAll
fun setup() {
val dataRoot = File("tmp/TestDesk").absolutePath
applicationSetup(dataRoot, loadConfigs(dataRoot))
setLoggingEnabled(false)
runBlocking {
extensions = getExtensionList()
extensions.forEach {
when {
it.obsolete -> {
uninstallExtension(it.pkgName)
}
it.hasUpdate -> {
updateExtension(it.pkgName)
}
else -> {
installExtension(it.pkgName)
}
}
}
sources = getSourceList().map { getHttpSource(it.id) }
}
setLoggingEnabled(true)
File("tmp/TestDesk/sources.txt").writeText(sources.joinToString("\n") { "${it.name} - ${it.lang.toUpperCase()} - ${it.id}" })
}
@Test
fun runTest() {
runBlocking(Dispatchers.Default) {
val semaphore = Semaphore(10)
sources.mapIndexed { index, source ->
async {
semaphore.withPermit {
logger.info { "$index - Now fetching popular manga from $source" }
try {
mangaToFetch += source to (source.fetchPopularManga(1)
.awaitSingleRepeat().mangas.firstOrNull()
?: throw Exception("Source returned no manga"))
} catch (e: Exception) {
logger.warn { "Failed to fetch popular manga from $source: ${e.message}" }
failedToFetch += source to e
}
}
}
}.awaitAll()
File("tmp/TestDesk/failedToFetch.txt").writeText(
failedToFetch.joinToString("\n") { (source, exception) ->
"${source.name} (${source.lang.toUpperCase()}, ${source.id}):" +
" ${exception.message}"
}
)
logger.info { "Now fetching manga info from ${mangaToFetch.size} sources" }
mangaToFetch.mapIndexed { index, (source, manga) ->
async {
semaphore.withPermit {
logger.info { "$index - Now fetching manga from $source" }
try {
manga.copyFrom(source.fetchMangaDetails(manga).awaitSingleRepeat())
manga.initialized = true
} catch (e: Exception) {
logger.warn {
"Failed to fetch manga info from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}"
}
mangaFailedToFetch += Triple(source, manga, e)
}
}
}
}.awaitAll()
File("tmp/TestDesk/MangaFailedToFetch.txt").writeText(
mangaFailedToFetch.joinToString("\n") { (source, manga, exception) ->
"${source.name} (${source.lang}, ${source.id}):" +
" ${manga.title} (${source.mangaDetailsRequest(manga).url}):" +
" ${exception.message}"
}
)
logger.info { "Now fetching manga chapters from ${mangaToFetch.size} sources" }
mangaToFetch.filter { it.second.initialized }.mapIndexed { index, (source, manga) ->
async {
semaphore.withPermit {
logger.info { "$index - Now fetching manga chapters from $source" }
try {
chaptersToFetch += Triple(
source,
manga,
source.fetchChapterList(manga).awaitSingleRepeat().firstOrNull() ?: throw Exception("Source returned no chapters")
)
} catch (e: Exception) {
logger.warn {
"Failed to fetch manga chapters from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}"
}
chaptersFailedToFetch += Triple(source, manga, e)
} catch (e: NoClassDefFoundError) {
logger.warn {
"Failed to fetch manga chapters from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}"
}
chaptersFailedToFetch += Triple(source, manga, e)
}
}
}
}.awaitAll()
File("tmp/TestDesk/ChaptersFailedToFetch.txt").writeText(
chaptersFailedToFetch.joinToString("\n") { (source, manga, exception) ->
"${source.name} (${source.lang}, ${source.id}):" +
" ${manga.title} (${source.mangaDetailsRequest(manga).url}):" +
" ${exception.message}"
}
)
chaptersToFetch.mapIndexed { index, (source, manga, chapter) ->
async {
semaphore.withPermit {
logger.info { "$index - Now fetching page list from $source" }
try {
source.fetchPageList(chapter).awaitSingleRepeat()
} catch (e: Exception) {
logger.warn {
"Failed to fetch manga info from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}"
}
chaptersPageListFailedToFetch += Triple(source, manga to chapter, e)
}
}
}
}.awaitAll()
File("tmp/TestDesk/ChapterPageListFailedToFetch.txt").writeText(
chaptersPageListFailedToFetch.joinToString("\n") { (source, manga, exception) ->
"${source.name} (${source.lang}, ${source.id}):" +
" ${manga.first.title} (${source.mangaDetailsRequest(manga.first).url}):" +
" ${manga.second.name} (${manga.second.url}): ${exception.message}"
}
)
}
}
private suspend fun <T> Observable<T>.awaitSingleRepeat(): T {
for (i in 1..2) {
try {
return awaitSingle()
} catch (e: Exception) {}
}
return awaitSingle()
}
}

44
server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt Normal file
View File

@ -0,0 +1,44 @@
package ir.armor.tachidesk
import ch.qos.logback.classic.Level
import com.typesafe.config.Config
import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigRenderOptions
import mu.KotlinLogging
import org.slf4j.Logger
import java.io.File
/**
* Load configs
*/
fun loadConfigs(dataRoot: String): Config {
val logger = KotlinLogging.logger {}
//Load reference configs
val compatConfig = ConfigFactory.parseResources("compat-reference.conf")
val serverConfig = ConfigFactory.parseResources("server-reference.conf")
//Load user config
val userConfig =
File(dataRoot, "server.conf").let {
ConfigFactory.parseFile(it)
}
val config = ConfigFactory.empty()
.withFallback(userConfig)
.withFallback(compatConfig)
.withFallback(serverConfig)
.resolve()
logger.debug {
"Loaded config:\n" + config.root().render(ConfigRenderOptions.concise().setFormatted(true))
}
return config
}
fun setLoggingEnabled(enabled: Boolean = true) {
val logger = (KotlinLogging.logger(Logger.ROOT_LOGGER_NAME).underlyingLogger as ch.qos.logback.classic.Logger)
logger.level = if (enabled) {
Level.DEBUG
} else Level.ERROR
}

13
server/src/test/resources/server-reference.conf Normal file
View File

@ -0,0 +1,13 @@
# Server ip and port bindings
server.ip = "0.0.0.0"
server.port = 4567
# Socks5 proxy
server.socksProxy = false
server.socksProxyHost = ""
server.socksProxyPort = ""
# misc
server.debugLogsEnabled = true
server.systemTrayEnabled = false
server.initialOpenInBrowserEnabled = true