From 9caae5f1e556eab627800d5deb557917784ac7f3 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Fri, 29 Jan 2021 14:19:24 +0330 Subject: [PATCH] thumbnail caching --- .../kanade/tachiyomi/network/NetworkHelper.kt | 2 + .../main/kotlin/ir/armor/tachidesk/Config.kt | 1 + .../main/kotlin/ir/armor/tachidesk/Main.kt | 13 ++ .../kotlin/ir/armor/tachidesk/util/File.kt | 30 +++ .../kotlin/ir/armor/tachidesk/util/Manga.kt | 207 +++++++++++++----- .../ir/armor/tachidesk/util/MangaList.kt | 62 ++++-- .../kotlin/ir/armor/tachidesk/util/Util.kt | 1 + webUI/react/src/components/MangaCard.tsx | 18 +- webUI/react/src/components/MangaGrid.tsx | 27 ++- webUI/react/src/screens/MangaList.tsx | 1 + webUI/react/src/screens/SearchSingle.tsx | 1 + 11 files changed, 279 insertions(+), 84 deletions(-) create mode 100644 server/src/main/kotlin/ir/armor/tachidesk/util/File.kt diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt index d567210..22a03b3 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt @@ -27,6 +27,8 @@ class NetworkHelper(context: Context) { // .cache(Cache(cacheDir, cacheSize)) .connectTimeout(30, TimeUnit.SECONDS) .readTimeout(30, TimeUnit.SECONDS) +// .dispatcher(Dispatcher(Executors.newFixedThreadPool(1))) + // .addInterceptor(UserAgentInterceptor()) // if (BuildConfig.DEBUG) { diff --git a/server/src/main/kotlin/ir/armor/tachidesk/Config.kt b/server/src/main/kotlin/ir/armor/tachidesk/Config.kt index 6597c78..b807036 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/Config.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/Config.kt @@ -9,4 +9,5 @@ import net.harawata.appdirs.AppDirsFactory object Config { val dataRoot = AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null) val extensionsRoot = "$dataRoot/extensions" + val thumbnailsRoot = "$dataRoot/thumbnails" } diff --git a/server/src/main/kotlin/ir/armor/tachidesk/Main.kt b/server/src/main/kotlin/ir/armor/tachidesk/Main.kt index 105acb1..d22a6f9 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/Main.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/Main.kt @@ -11,9 +11,11 @@ import ir.armor.tachidesk.util.getChapterList import ir.armor.tachidesk.util.getExtensionList import ir.armor.tachidesk.util.getManga import ir.armor.tachidesk.util.getMangaList +import ir.armor.tachidesk.util.getMangaUpdateQueueThread import ir.armor.tachidesk.util.getPages import ir.armor.tachidesk.util.getSource import ir.armor.tachidesk.util.getSourceList +import ir.armor.tachidesk.util.getThumbnail import ir.armor.tachidesk.util.installAPK import ir.armor.tachidesk.util.sourceFilters import ir.armor.tachidesk.util.sourceGlobalSearch @@ -54,6 +56,8 @@ class Main { // start app androidCompat.startApp(App()) + Thread(getMangaUpdateQueueThread).start() + val app = Javalin.create { config -> try { this::class.java.classLoader.getResource("/react/index.html") @@ -116,6 +120,15 @@ class Main { ctx.json(getPages(chapterId, mangaId)) } + app.get("api/v1/manga/:mangaId/thumbnail") { ctx -> + val mangaId = ctx.pathParam("mangaId").toInt() + println("got request for: $mangaId") + val result = getThumbnail(mangaId) + + ctx.result(result.first) + ctx.header("content-type", result.second) + } + // global search app.get("/api/v1/search/:searchTerm") { ctx -> val searchTerm = ctx.pathParam("searchTerm") diff --git a/server/src/main/kotlin/ir/armor/tachidesk/util/File.kt b/server/src/main/kotlin/ir/armor/tachidesk/util/File.kt new file mode 100644 index 0000000..0be7d68 --- /dev/null +++ b/server/src/main/kotlin/ir/armor/tachidesk/util/File.kt @@ -0,0 +1,30 @@ +package ir.armor.tachidesk.util + +import java.io.BufferedInputStream +import java.io.File +import java.io.FileInputStream +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Paths + +fun writeStream(fileStream: InputStream, path: String) { + Files.newOutputStream(Paths.get(path)).use { os -> + val buffer = ByteArray(1024) + var len: Int + while (fileStream.read(buffer).also { len = it } > 0) { + os.write(buffer, 0, len) + } + } +} + +fun pathToInputStream(path: String): InputStream { + return BufferedInputStream(FileInputStream(path)) +} + +fun findFileNameStartingWith(directoryPath: String, fileName: String): String? { + File(directoryPath).listFiles().forEach { file -> + if (file.name.startsWith(fileName)) + return "$directoryPath/${file.name}" + } + return null +} diff --git a/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt b/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt index ed95ff3..01bc975 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt @@ -4,77 +4,168 @@ package ir.armor.tachidesk.util * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.source.model.SManga +import ir.armor.tachidesk.Config import ir.armor.tachidesk.database.dataclass.MangaDataClass import ir.armor.tachidesk.database.table.MangaStatus import ir.armor.tachidesk.database.table.MangaTable import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update +import java.io.InputStream +import java.util.concurrent.ArrayBlockingQueue -fun getManga(mangaId: Int): MangaDataClass { - return transaction { - var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! +val getMangaUpdateQueue = ArrayBlockingQueue>(1000) +@Volatile +var getMangaCount = 0 - return@transaction if (mangaEntry[MangaTable.initialized]) { - MangaDataClass( - mangaId, - mangaEntry[MangaTable.sourceReference].value, +val getMangaUpdateQueueThread = Runnable { + while (true) { + val p = getMangaUpdateQueue.take() + println("took ${p.first}") + while (getMangaCount > 0) { + println("count is $getMangaCount") + Thread.sleep(1000) + } + val mangaId = p.first + println("working on $mangaId") + val fetchedManga = p.second!! + try { + transaction { + println("transaction start $mangaId") + MangaTable.update({ MangaTable.id eq mangaId }) { - mangaEntry[MangaTable.url], - mangaEntry[MangaTable.title], - mangaEntry[MangaTable.thumbnail_url], + it[MangaTable.initialized] = true - true, - - mangaEntry[MangaTable.artist], - mangaEntry[MangaTable.author], - mangaEntry[MangaTable.description], - mangaEntry[MangaTable.genre], - MangaStatus.valueOf(mangaEntry[MangaTable.status]).name, - ) - } else { // initialize manga - val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value) - val fetchedManga = source.fetchMangaDetails( - SManga.create().apply { - url = mangaEntry[MangaTable.url] - title = mangaEntry[MangaTable.title] + it[MangaTable.artist] = fetchedManga.artist + it[MangaTable.author] = fetchedManga.author + it[MangaTable.description] = fetchedManga.description + it[MangaTable.genre] = fetchedManga.genre + it[MangaTable.status] = fetchedManga.status + if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) + it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url } - ).toBlocking().first() - - // update database - MangaTable.update({ MangaTable.id eq mangaId }) { -// it[url] = fetchedManga.url -// it[title] = fetchedManga.title - it[initialized] = true - - it[artist] = fetchedManga.artist - it[author] = fetchedManga.author - it[description] = fetchedManga.description - it[genre] = fetchedManga.genre - it[status] = fetchedManga.status - if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) - it[thumbnail_url] = fetchedManga.thumbnail_url + println("transaction end $mangaId") } - - mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! - - MangaDataClass( - mangaId, - mangaEntry[MangaTable.sourceReference].value, - - mangaEntry[MangaTable.url], - mangaEntry[MangaTable.title], - mangaEntry[MangaTable.thumbnail_url], - - true, - - mangaEntry[MangaTable.artist], - mangaEntry[MangaTable.author], - mangaEntry[MangaTable.description], - mangaEntry[MangaTable.genre], - MangaStatus.valueOf(mangaEntry[MangaTable.status]).name, - ) + } catch (e: Exception) { + println(e) + } + } +} + +fun getManga(mangaId: Int, proxyThumbnail: Boolean = true): MangaDataClass { + synchronized(getMangaCount) { + getMangaCount++ + } + return try { + transaction { + var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! + + return@transaction if (mangaEntry[MangaTable.initialized]) { + println("${mangaEntry[MangaTable.title]} is initialized") + println("${mangaEntry[MangaTable.thumbnail_url]}") + MangaDataClass( + mangaId, + mangaEntry[MangaTable.sourceReference].value, + + mangaEntry[MangaTable.url], + mangaEntry[MangaTable.title], + if (proxyThumbnail) proxyThumbnailUrl(mangaId) else mangaEntry[MangaTable.thumbnail_url], + + true, + + mangaEntry[MangaTable.artist], + mangaEntry[MangaTable.author], + mangaEntry[MangaTable.description], + mangaEntry[MangaTable.genre], + MangaStatus.valueOf(mangaEntry[MangaTable.status]).name, + ) + } else { // initialize manga + val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value) + val fetchedManga = source.fetchMangaDetails( + SManga.create().apply { + url = mangaEntry[MangaTable.url] + title = mangaEntry[MangaTable.title] + } + ).toBlocking().first() + + // update database + // TODO: sqlite gets fucked here + println("putting $mangaId") + getMangaUpdateQueue.put(Pair(mangaId, fetchedManga)) + +// mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! + val newThumbnail = + if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) { + fetchedManga.thumbnail_url + } else mangaEntry[MangaTable.thumbnail_url] + + MangaDataClass( + mangaId, + mangaEntry[MangaTable.sourceReference].value, + + mangaEntry[MangaTable.url], + mangaEntry[MangaTable.title], + if (proxyThumbnail) proxyThumbnailUrl(mangaId) else newThumbnail, + + true, + + fetchedManga.artist, + fetchedManga.author, + fetchedManga.description, + fetchedManga.genre, + MangaStatus.valueOf(fetchedManga.status).name, + ) + } + } + } finally { + synchronized(getMangaCount) { + getMangaCount-- + } + } +} + +fun getThumbnail(mangaId: Int): Pair { + return transaction { + var filePath = Config.thumbnailsRoot + "/$mangaId" + var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! + + val potentialCache = findFileNameStartingWith(Config.thumbnailsRoot, mangaId.toString()) + if (potentialCache != null) { + println("using cached thumbnail file") + return@transaction Pair( + pathToInputStream(potentialCache), + "image/${potentialCache.substringAfter("$mangaId.")}" + ) + } + + val sourceId = mangaEntry[MangaTable.sourceReference].value + println("getting source for $mangaId") + val source = getHttpSource(sourceId) + var thumbnailUrl = mangaEntry[MangaTable.thumbnail_url] + if (thumbnailUrl == null || thumbnailUrl.isEmpty()) { + thumbnailUrl = getManga(mangaId, proxyThumbnail = false).thumbnailUrl!! + } + println(thumbnailUrl) + val response = source.client.newCall( + GET(thumbnailUrl, source.headers) + ).execute() + + println(response.code) + + if (response.code == 200) { + val contentType = response.headers["content-type"]!! + filePath += "." + contentType.substringAfter("image/") + + writeStream(response.body!!.byteStream(), filePath) + + return@transaction Pair( + pathToInputStream(filePath), + contentType + ) + } else { + throw Exception("request error! ${response.code}") } } } diff --git a/server/src/main/kotlin/ir/armor/tachidesk/util/MangaList.kt b/server/src/main/kotlin/ir/armor/tachidesk/util/MangaList.kt index de53709..06d616d 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/util/MangaList.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/util/MangaList.kt @@ -13,6 +13,10 @@ import org.jetbrains.exposed.sql.insertAndGetId import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction +fun proxyThumbnailUrl(mangaId: Int): String { + return "http://127.0.0.1:4567/api/v1/manga/$mangaId/thumbnail" +} + fun getMangaList(sourceId: Long, pageNum: Int = 1, popular: Boolean): PagedMangaListDataClass { val source = getHttpSource(sourceId.toLong()) val mangasPage = if (popular) { @@ -31,8 +35,8 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass { val mangaList = transaction { return@transaction mangasPage.mangas.map { manga -> var mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull() - var mangaEntityId = if (mangaEntry == null) { // create manga entry - MangaTable.insertAndGetId { + if (mangaEntry == null) { // create manga entry + val mangaId = MangaTable.insertAndGetId { it[url] = manga.url it[title] = manga.title @@ -41,30 +45,46 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass { it[description] = manga.description it[genre] = manga.genre it[status] = manga.status - it[thumbnail_url] = manga.genre + it[thumbnail_url] = manga.thumbnail_url it[sourceReference] = sourceId }.value + + MangaDataClass( + mangaId, + sourceId, + + manga.url, + manga.title, + proxyThumbnailUrl(mangaId), + + manga.initialized, + + manga.artist, + manga.author, + manga.description, + manga.genre, + MangaStatus.valueOf(manga.status).name, + ) } else { - mangaEntry[MangaTable.id].value + val mangaId = mangaEntry[MangaTable.id].value + MangaDataClass( + mangaId, + sourceId, + + manga.url, + manga.title, + proxyThumbnailUrl(mangaId), + + true, + + mangaEntry[MangaTable.artist], + mangaEntry[MangaTable.author], + mangaEntry[MangaTable.description], + mangaEntry[MangaTable.genre], + MangaStatus.valueOf(mangaEntry[MangaTable.status]).name, + ) } - - MangaDataClass( - mangaEntityId, - sourceId, - - manga.url, - manga.title, - manga.thumbnail_url, - - manga.initialized, - - manga.artist, - manga.author, - manga.description, - manga.genre, - MangaStatus.valueOf(manga.status).name, - ) } } return PagedMangaListDataClass( diff --git a/server/src/main/kotlin/ir/armor/tachidesk/util/Util.kt b/server/src/main/kotlin/ir/armor/tachidesk/util/Util.kt index 5f9aa9f..54077fd 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/util/Util.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/util/Util.kt @@ -12,6 +12,7 @@ fun applicationSetup() { // make dirs we need File(Config.dataRoot).mkdirs() File(Config.extensionsRoot).mkdirs() + File(Config.thumbnailsRoot).mkdirs() makeDataBaseTables() } diff --git a/webUI/react/src/components/MangaCard.tsx b/webUI/react/src/components/MangaCard.tsx index 12ae32a..2182af2 100644 --- a/webUI/react/src/components/MangaCard.tsx +++ b/webUI/react/src/components/MangaCard.tsx @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { makeStyles } from '@material-ui/core/styles'; import Card from '@material-ui/core/Card'; import CardActionArea from '@material-ui/core/CardActionArea'; @@ -43,15 +43,29 @@ const useStyles = makeStyles({ interface IProps { manga: IManga + setMangaThumbnailUrl: (thumbnailUrl:string) => void } const MangaCard = React.forwardRef((props: IProps, ref) => { const { manga: { id, title, thumbnailUrl, - }, + }, setMangaThumbnailUrl, } = props; const classes = useStyles(); + console.log(`${title} has ${thumbnailUrl}`); + + if (thumbnailUrl === null || thumbnailUrl.length === 0) { + useEffect(() => { + fetch(`http://127.0.0.1:4567/api/v1/manga/${id}/`) + .then((response) => response.json()) + .then((data: IManga) => { + setMangaThumbnailUrl(data.thumbnailUrl); + }); + }, + []); + } + return ( diff --git a/webUI/react/src/components/MangaGrid.tsx b/webUI/react/src/components/MangaGrid.tsx index ac2d752..19bb9b4 100644 --- a/webUI/react/src/components/MangaGrid.tsx +++ b/webUI/react/src/components/MangaGrid.tsx @@ -12,15 +12,21 @@ interface IProps{ hasNextPage: boolean lastPageNum: number setLastPageNum: (lastPageNum: number) => void + setMangas: (mangas: IManga[]) => void } export default function MangaGrid(props: IProps) { const { - mangas, message, hasNextPage, lastPageNum, setLastPageNum, + mangas, message, hasNextPage, lastPageNum, setLastPageNum, setMangas, } = props; let mapped; const lastManga = useRef(); + function setMangaThumbnailUrl(index: number, thumbnailUrl: string) { + mangas[index].thumbnailUrl = thumbnailUrl; + setMangas(mangas); + } + const scrollHandler = () => { if (lastManga.current) { const rect = lastManga.current.getBoundingClientRect(); @@ -41,9 +47,24 @@ export default function MangaGrid(props: IProps) { } else { mapped = mangas.map((it, idx) => { if (idx === mangas.length - 1) { - return ; + return ( + setMangaThumbnailUrl(idx, thumbnailUrl) + } + /> + ); } - return ; + return ( + setMangaThumbnailUrl(idx, thumbnailUrl) + } + /> + ); }); } diff --git a/webUI/react/src/screens/MangaList.tsx b/webUI/react/src/screens/MangaList.tsx index 78dd7dd..6a1e25f 100644 --- a/webUI/react/src/screens/MangaList.tsx +++ b/webUI/react/src/screens/MangaList.tsx @@ -37,6 +37,7 @@ export default function MangaList(props: { popular: boolean }) { return (