thumbnail caching

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -27,6 +27,8 @@ class NetworkHelper(context: Context) {
 //            .cache(Cache(cacheDir, cacheSize))
             .connectTimeout(30, TimeUnit.SECONDS)
             .readTimeout(30, TimeUnit.SECONDS)
+//            .dispatcher(Dispatcher(Executors.newFixedThreadPool(1)))
+
 //            .addInterceptor(UserAgentInterceptor())
 
 //        if (BuildConfig.DEBUG) {

server/src/main/kotlin/ir/armor/tachidesk/Config.kt

@@ -9,4 +9,5 @@ import net.harawata.appdirs.AppDirsFactory
 object Config {
     val dataRoot = AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)
     val extensionsRoot = "$dataRoot/extensions"
+    val thumbnailsRoot = "$dataRoot/thumbnails"
 }

server/src/main/kotlin/ir/armor/tachidesk/Main.kt

@@ -11,9 +11,11 @@ import ir.armor.tachidesk.util.getChapterList
 import ir.armor.tachidesk.util.getExtensionList
 import ir.armor.tachidesk.util.getManga
 import ir.armor.tachidesk.util.getMangaList
+import ir.armor.tachidesk.util.getMangaUpdateQueueThread
 import ir.armor.tachidesk.util.getPages
 import ir.armor.tachidesk.util.getSource
 import ir.armor.tachidesk.util.getSourceList
+import ir.armor.tachidesk.util.getThumbnail
 import ir.armor.tachidesk.util.installAPK
 import ir.armor.tachidesk.util.sourceFilters
 import ir.armor.tachidesk.util.sourceGlobalSearch
@@ -54,6 +56,8 @@ class Main {
             // start app
             androidCompat.startApp(App())
 
+            Thread(getMangaUpdateQueueThread).start()
+
             val app = Javalin.create { config ->
                 try {
                     this::class.java.classLoader.getResource("/react/index.html")
@@ -116,6 +120,15 @@ class Main {
                 ctx.json(getPages(chapterId, mangaId))
             }
 
+            app.get("api/v1/manga/:mangaId/thumbnail") { ctx ->
+                val mangaId = ctx.pathParam("mangaId").toInt()
+                println("got request for: $mangaId")
+                val result = getThumbnail(mangaId)
+
+                ctx.result(result.first)
+                ctx.header("content-type", result.second)
+            }
+
             // global search
             app.get("/api/v1/search/:searchTerm") { ctx ->
                 val searchTerm = ctx.pathParam("searchTerm")

server/src/main/kotlin/ir/armor/tachidesk/util/File.kt

@@ -0,0 +1,30 @@
+package ir.armor.tachidesk.util
+
+import java.io.BufferedInputStream
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
+import java.nio.file.Files
+import java.nio.file.Paths
+
+fun writeStream(fileStream: InputStream, path: String) {
+    Files.newOutputStream(Paths.get(path)).use { os ->
+        val buffer = ByteArray(1024)
+        var len: Int
+        while (fileStream.read(buffer).also { len = it } > 0) {
+            os.write(buffer, 0, len)
+        }
+    }
+}
+
+fun pathToInputStream(path: String): InputStream {
+    return BufferedInputStream(FileInputStream(path))
+}
+
+fun findFileNameStartingWith(directoryPath: String, fileName: String): String? {
+    File(directoryPath).listFiles().forEach { file ->
+        if (file.name.startsWith(fileName))
+            return "$directoryPath/${file.name}"
+    }
+    return null
+}

server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt

@@ -4,77 +4,168 @@ package ir.armor.tachidesk.util
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.SManga
+import ir.armor.tachidesk.Config
 import ir.armor.tachidesk.database.dataclass.MangaDataClass
 import ir.armor.tachidesk.database.table.MangaStatus
 import ir.armor.tachidesk.database.table.MangaTable
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
+import java.io.InputStream
+import java.util.concurrent.ArrayBlockingQueue
 
-fun getManga(mangaId: Int): MangaDataClass {
+val getMangaUpdateQueue = ArrayBlockingQueue<Pair<Int, SManga?>>(1000)
-    return transaction {
+@Volatile
-        var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+var getMangaCount = 0
 
-        return@transaction if (mangaEntry[MangaTable.initialized]) {
+val getMangaUpdateQueueThread = Runnable {
-            MangaDataClass(
+    while (true) {
-                mangaId,
+        val p = getMangaUpdateQueue.take()
-                mangaEntry[MangaTable.sourceReference].value,
+        println("took ${p.first}")
+        while (getMangaCount > 0) {
+            println("count is $getMangaCount")
+            Thread.sleep(1000)
+        }
+        val mangaId = p.first
+        println("working on $mangaId")
+        val fetchedManga = p.second!!
+        try {
+            transaction {
+                println("transaction start $mangaId")
+                MangaTable.update({ MangaTable.id eq mangaId }) {
 
-                mangaEntry[MangaTable.url],
+                    it[MangaTable.initialized] = true
-                mangaEntry[MangaTable.title],
-                mangaEntry[MangaTable.thumbnail_url],
 
-                true,
+                    it[MangaTable.artist] = fetchedManga.artist
-
+                    it[MangaTable.author] = fetchedManga.author
-                mangaEntry[MangaTable.artist],
+                    it[MangaTable.description] = fetchedManga.description
-                mangaEntry[MangaTable.author],
+                    it[MangaTable.genre] = fetchedManga.genre
-                mangaEntry[MangaTable.description],
+                    it[MangaTable.status] = fetchedManga.status
-                mangaEntry[MangaTable.genre],
+                    if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty())
-                MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+                        it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
-            )
-        } else { // initialize manga
-            val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value)
-            val fetchedManga = source.fetchMangaDetails(
-                SManga.create().apply {
-                    url = mangaEntry[MangaTable.url]
-                    title = mangaEntry[MangaTable.title]
                 }
-            ).toBlocking().first()
+                println("transaction end $mangaId")
-
-            // update database
-            MangaTable.update({ MangaTable.id eq mangaId }) {
-//            it[url] = fetchedManga.url
-//            it[title] = fetchedManga.title
-                it[initialized] = true
-
-                it[artist] = fetchedManga.artist
-                it[author] = fetchedManga.author
-                it[description] = fetchedManga.description
-                it[genre] = fetchedManga.genre
-                it[status] = fetchedManga.status
-                if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty())
-                    it[thumbnail_url] = fetchedManga.thumbnail_url
             }
-
+        } catch (e: Exception) {
-            mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+            println(e)
-
+        }
-            MangaDataClass(
+    }
-                mangaId,
+}
-                mangaEntry[MangaTable.sourceReference].value,
+
-
+fun getManga(mangaId: Int, proxyThumbnail: Boolean = true): MangaDataClass {
-                mangaEntry[MangaTable.url],
+    synchronized(getMangaCount) {
-                mangaEntry[MangaTable.title],
+        getMangaCount++
-                mangaEntry[MangaTable.thumbnail_url],
+    }
-
+    return try {
-                true,
+        transaction {
-
+            var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
-                mangaEntry[MangaTable.artist],
+
-                mangaEntry[MangaTable.author],
+            return@transaction if (mangaEntry[MangaTable.initialized]) {
-                mangaEntry[MangaTable.description],
+                println("${mangaEntry[MangaTable.title]} is initialized")
-                mangaEntry[MangaTable.genre],
+                println("${mangaEntry[MangaTable.thumbnail_url]}")
-                MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+                MangaDataClass(
-            )
+                        mangaId,
+                        mangaEntry[MangaTable.sourceReference].value,
+
+                        mangaEntry[MangaTable.url],
+                        mangaEntry[MangaTable.title],
+                        if (proxyThumbnail) proxyThumbnailUrl(mangaId) else mangaEntry[MangaTable.thumbnail_url],
+
+                        true,
+
+                        mangaEntry[MangaTable.artist],
+                        mangaEntry[MangaTable.author],
+                        mangaEntry[MangaTable.description],
+                        mangaEntry[MangaTable.genre],
+                        MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+                )
+            } else { // initialize manga
+                val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value)
+                val fetchedManga = source.fetchMangaDetails(
+                        SManga.create().apply {
+                            url = mangaEntry[MangaTable.url]
+                            title = mangaEntry[MangaTable.title]
+                        }
+                ).toBlocking().first()
+
+                // update database
+                // TODO: sqlite gets fucked here
+                println("putting $mangaId")
+                getMangaUpdateQueue.put(Pair(mangaId, fetchedManga))
+
+//            mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+                val newThumbnail =
+                        if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) {
+                            fetchedManga.thumbnail_url
+                        } else mangaEntry[MangaTable.thumbnail_url]
+
+                MangaDataClass(
+                        mangaId,
+                        mangaEntry[MangaTable.sourceReference].value,
+
+                        mangaEntry[MangaTable.url],
+                        mangaEntry[MangaTable.title],
+                        if (proxyThumbnail) proxyThumbnailUrl(mangaId) else newThumbnail,
+
+                        true,
+
+                        fetchedManga.artist,
+                        fetchedManga.author,
+                        fetchedManga.description,
+                        fetchedManga.genre,
+                        MangaStatus.valueOf(fetchedManga.status).name,
+                )
+            }
+        }
+    } finally {
+        synchronized(getMangaCount) {
+            getMangaCount--
+        }
+    }
+}
+
+fun getThumbnail(mangaId: Int): Pair<InputStream, String> {
+    return transaction {
+        var filePath = Config.thumbnailsRoot + "/$mangaId"
+        var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+
+        val potentialCache = findFileNameStartingWith(Config.thumbnailsRoot, mangaId.toString())
+        if (potentialCache != null) {
+            println("using cached thumbnail file")
+            return@transaction Pair(
+                    pathToInputStream(potentialCache),
+                    "image/${potentialCache.substringAfter("$mangaId.")}"
+            )
+        }
+
+        val sourceId = mangaEntry[MangaTable.sourceReference].value
+        println("getting source for $mangaId")
+        val source = getHttpSource(sourceId)
+        var thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
+        if (thumbnailUrl == null || thumbnailUrl.isEmpty()) {
+            thumbnailUrl = getManga(mangaId, proxyThumbnail = false).thumbnailUrl!!
+        }
+        println(thumbnailUrl)
+        val response = source.client.newCall(
+                GET(thumbnailUrl, source.headers)
+        ).execute()
+
+        println(response.code)
+
+        if (response.code == 200) {
+            val contentType = response.headers["content-type"]!!
+            filePath += "." + contentType.substringAfter("image/")
+
+            writeStream(response.body!!.byteStream(), filePath)
+
+            return@transaction Pair(
+                    pathToInputStream(filePath),
+                    contentType
+            )
+        } else {
+            throw Exception("request error! ${response.code}")
         }
     }
 }

server/src/main/kotlin/ir/armor/tachidesk/util/MangaList.kt

@@ -13,6 +13,10 @@ import org.jetbrains.exposed.sql.insertAndGetId
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 
+fun proxyThumbnailUrl(mangaId: Int): String {
+    return "http://127.0.0.1:4567/api/v1/manga/$mangaId/thumbnail"
+}
+
 fun getMangaList(sourceId: Long, pageNum: Int = 1, popular: Boolean): PagedMangaListDataClass {
     val source = getHttpSource(sourceId.toLong())
     val mangasPage = if (popular) {
@@ -31,8 +35,8 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass {
     val mangaList = transaction {
         return@transaction mangasPage.mangas.map { manga ->
             var mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull()
-            var mangaEntityId = if (mangaEntry == null) { // create manga entry
+            if (mangaEntry == null) { // create manga entry
-                MangaTable.insertAndGetId {
+                val mangaId = MangaTable.insertAndGetId {
                     it[url] = manga.url
                     it[title] = manga.title
 
@@ -41,30 +45,46 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass {
                     it[description] = manga.description
                     it[genre] = manga.genre
                     it[status] = manga.status
-                    it[thumbnail_url] = manga.genre
+                    it[thumbnail_url] = manga.thumbnail_url
 
                     it[sourceReference] = sourceId
                 }.value
+
+                MangaDataClass(
+                    mangaId,
+                    sourceId,
+
+                    manga.url,
+                    manga.title,
+                    proxyThumbnailUrl(mangaId),
+
+                    manga.initialized,
+
+                    manga.artist,
+                    manga.author,
+                    manga.description,
+                    manga.genre,
+                    MangaStatus.valueOf(manga.status).name,
+                )
             } else {
-                mangaEntry[MangaTable.id].value
+                val mangaId = mangaEntry[MangaTable.id].value
+                MangaDataClass(
+                    mangaId,
+                    sourceId,
+
+                    manga.url,
+                    manga.title,
+                    proxyThumbnailUrl(mangaId),
+
+                    true,
+
+                    mangaEntry[MangaTable.artist],
+                    mangaEntry[MangaTable.author],
+                    mangaEntry[MangaTable.description],
+                    mangaEntry[MangaTable.genre],
+                    MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+                )
             }
-
-            MangaDataClass(
-                mangaEntityId,
-                sourceId,
-
-                manga.url,
-                manga.title,
-                manga.thumbnail_url,
-
-                manga.initialized,
-
-                manga.artist,
-                manga.author,
-                manga.description,
-                manga.genre,
-                MangaStatus.valueOf(manga.status).name,
-            )
         }
     }
     return PagedMangaListDataClass(

server/src/main/kotlin/ir/armor/tachidesk/util/Util.kt

@@ -12,6 +12,7 @@ fun applicationSetup() {
     // make dirs we need
     File(Config.dataRoot).mkdirs()
     File(Config.extensionsRoot).mkdirs()
+    File(Config.thumbnailsRoot).mkdirs()
 
     makeDataBaseTables()
 }

webUI/react/src/components/MangaCard.tsx

@@ -2,7 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-import React from 'react';
+import React, { useEffect } from 'react';
 import { makeStyles } from '@material-ui/core/styles';
 import Card from '@material-ui/core/Card';
 import CardActionArea from '@material-ui/core/CardActionArea';
@@ -43,15 +43,29 @@ const useStyles = makeStyles({
 
 interface IProps {
     manga: IManga
+    setMangaThumbnailUrl: (thumbnailUrl:string) => void
 }
 const MangaCard = React.forwardRef((props: IProps, ref) => {
     const {
         manga: {
             id, title, thumbnailUrl,
-        },
+        }, setMangaThumbnailUrl,
     } = props;
     const classes = useStyles();
 
+    console.log(`${title} has ${thumbnailUrl}`);
+
+    if (thumbnailUrl === null || thumbnailUrl.length === 0) {
+        useEffect(() => {
+            fetch(`http://127.0.0.1:4567/api/v1/manga/${id}/`)
+                .then((response) => response.json())
+                .then((data: IManga) => {
+                    setMangaThumbnailUrl(data.thumbnailUrl);
+                });
+        },
+        []);
+    }
+
     return (
         <Grid item xs={6} sm={4} md={3} lg={2}>
             <Link to={`/manga/${id}/`}>

webUI/react/src/components/MangaGrid.tsx

@@ -12,15 +12,21 @@ interface IProps{
     hasNextPage: boolean
     lastPageNum: number
     setLastPageNum: (lastPageNum: number) => void
+    setMangas: (mangas: IManga[]) => void
 }
 
 export default function MangaGrid(props: IProps) {
     const {
-        mangas, message, hasNextPage, lastPageNum, setLastPageNum,
+        mangas, message, hasNextPage, lastPageNum, setLastPageNum, setMangas,
     } = props;
     let mapped;
     const lastManga = useRef<HTMLInputElement>();
 
+    function setMangaThumbnailUrl(index: number, thumbnailUrl: string) {
+        mangas[index].thumbnailUrl = thumbnailUrl;
+        setMangas(mangas);
+    }
+
     const scrollHandler = () => {
         if (lastManga.current) {
             const rect = lastManga.current.getBoundingClientRect();
@@ -41,9 +47,24 @@ export default function MangaGrid(props: IProps) {
     } else {
         mapped = mangas.map((it, idx) => {
             if (idx === mangas.length - 1) {
-                return <MangaCard manga={it} ref={lastManga} />;
+                return (
+                    <MangaCard
+                        manga={it}
+                        ref={lastManga}
+                        setMangaThumbnailUrl={
+                            (thumbnailUrl:string) => setMangaThumbnailUrl(idx, thumbnailUrl)
+                        }
+                    />
+                );
             }
-            return <MangaCard manga={it} />;
+            return (
+                <MangaCard
+                    manga={it}
+                    setMangaThumbnailUrl={
+                        (thumbnailUrl:string) => setMangaThumbnailUrl(idx, thumbnailUrl)
+                    }
+                />
+            );
         });
     }
 

webUI/react/src/screens/MangaList.tsx

@@ -37,6 +37,7 @@ export default function MangaList(props: { popular: boolean }) {
     return (
         <MangaGrid
             mangas={mangas}
+            setMangas={setMangas}
             hasNextPage={hasNextPage}
             lastPageNum={lastPageNum}
             setLastPageNum={setLastPageNum}

webUI/react/src/screens/SearchSingle.tsx

@@ -73,6 +73,7 @@ export default function SearchSingle() {
 
     const mangaGrid = (
         <MangaGrid
+            setMangas={setMangas}
             mangas={mangas}
             message={message}
             hasNextPage={hasNextPage}