Mirrors/tachiyomi-extensions-inspector
2
0
Fork 0
mirror of https://github.com/tachiyomiorg/tachiyomi-extensions-inspector.git synced 2025-01-24 06:21:16 +01:00
Code Issues Packages Projects Releases Wiki Activity

thumbnail caching

Browse Source
This commit is contained in:
Aria Moradi 2021-01-29 14:19:24 +03:30
parent 345be95ce9
commit 9caae5f1e5
11 changed files with 279 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt
View File

@ -27,6 +27,8 @@ class NetworkHelper(context: Context) {
// .cache(Cache(cacheDir, cacheSize))
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
// .dispatcher(Dispatcher(Executors.newFixedThreadPool(1)))
// .addInterceptor(UserAgentInterceptor())
// if (BuildConfig.DEBUG) {

1
server/src/main/kotlin/ir/armor/tachidesk/Config.kt
View File

@ -9,4 +9,5 @@ import net.harawata.appdirs.AppDirsFactory
object Config {
val dataRoot = AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)
val extensionsRoot = "$dataRoot/extensions"
val thumbnailsRoot = "$dataRoot/thumbnails"
}

13
server/src/main/kotlin/ir/armor/tachidesk/Main.kt
View File

@ -11,9 +11,11 @@ import ir.armor.tachidesk.util.getChapterList
import ir.armor.tachidesk.util.getExtensionList
import ir.armor.tachidesk.util.getManga
import ir.armor.tachidesk.util.getMangaList
import ir.armor.tachidesk.util.getMangaUpdateQueueThread
import ir.armor.tachidesk.util.getPages
import ir.armor.tachidesk.util.getSource
import ir.armor.tachidesk.util.getSourceList
import ir.armor.tachidesk.util.getThumbnail
import ir.armor.tachidesk.util.installAPK
import ir.armor.tachidesk.util.sourceFilters
import ir.armor.tachidesk.util.sourceGlobalSearch
@ -54,6 +56,8 @@ class Main {
// start app
androidCompat.startApp(App())
Thread(getMangaUpdateQueueThread).start()
val app = Javalin.create { config ->
try {
this::class.java.classLoader.getResource("/react/index.html")
@ -116,6 +120,15 @@ class Main {
ctx.json(getPages(chapterId, mangaId))
}
app.get("api/v1/manga/:mangaId/thumbnail") { ctx ->
val mangaId = ctx.pathParam("mangaId").toInt()
println("got request for: $mangaId")
val result = getThumbnail(mangaId)
ctx.result(result.first)
ctx.header("content-type", result.second)
}
// global search
app.get("/api/v1/search/:searchTerm") { ctx ->
val searchTerm = ctx.pathParam("searchTerm")

30
server/src/main/kotlin/ir/armor/tachidesk/util/File.kt Normal file
View File

@ -0,0 +1,30 @@
package ir.armor.tachidesk.util
import java.io.BufferedInputStream
import java.io.File
import java.io.FileInputStream
import java.io.InputStream
import java.nio.file.Files
import java.nio.file.Paths
fun writeStream(fileStream: InputStream, path: String) {
Files.newOutputStream(Paths.get(path)).use { os ->
val buffer = ByteArray(1024)
var len: Int
while (fileStream.read(buffer).also { len = it } > 0) {
os.write(buffer, 0, len)
}
}
}
fun pathToInputStream(path: String): InputStream {
return BufferedInputStream(FileInputStream(path))
}
fun findFileNameStartingWith(directoryPath: String, fileName: String): String? {
File(directoryPath).listFiles().forEach { file ->
if (file.name.startsWith(fileName))
return "$directoryPath/${file.name}"
}
return null
}

207
server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt
View File

@ -4,77 +4,168 @@ package ir.armor.tachidesk.util
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.SManga
import ir.armor.tachidesk.Config
import ir.armor.tachidesk.database.dataclass.MangaDataClass
import ir.armor.tachidesk.database.table.MangaStatus
import ir.armor.tachidesk.database.table.MangaTable
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import java.io.InputStream
import java.util.concurrent.ArrayBlockingQueue
fun getManga(mangaId: Int): MangaDataClass {
return transaction {
var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
val getMangaUpdateQueue = ArrayBlockingQueue<Pair<Int, SManga?>>(1000)
@Volatile
var getMangaCount = 0
return@transaction if (mangaEntry[MangaTable.initialized]) {
MangaDataClass(
mangaId,
mangaEntry[MangaTable.sourceReference].value,
val getMangaUpdateQueueThread = Runnable {
while (true) {
val p = getMangaUpdateQueue.take()
println("took ${p.first}")
while (getMangaCount > 0) {
println("count is $getMangaCount")
Thread.sleep(1000)
}
val mangaId = p.first
println("working on $mangaId")
val fetchedManga = p.second!!
try {
transaction {
println("transaction start $mangaId")
MangaTable.update({ MangaTable.id eq mangaId }) {
mangaEntry[MangaTable.url],
mangaEntry[MangaTable.title],
mangaEntry[MangaTable.thumbnail_url],
it[MangaTable.initialized] = true
true,
mangaEntry[MangaTable.artist],
mangaEntry[MangaTable.author],
mangaEntry[MangaTable.description],
mangaEntry[MangaTable.genre],
MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
)
} else { // initialize manga
val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value)
val fetchedManga = source.fetchMangaDetails(
SManga.create().apply {
url = mangaEntry[MangaTable.url]
title = mangaEntry[MangaTable.title]
it[MangaTable.artist] = fetchedManga.artist
it[MangaTable.author] = fetchedManga.author
it[MangaTable.description] = fetchedManga.description
it[MangaTable.genre] = fetchedManga.genre
it[MangaTable.status] = fetchedManga.status
if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty())
it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
}
).toBlocking().first()
// update database
MangaTable.update({ MangaTable.id eq mangaId }) {
// it[url] = fetchedManga.url
// it[title] = fetchedManga.title
it[initialized] = true
it[artist] = fetchedManga.artist
it[author] = fetchedManga.author
it[description] = fetchedManga.description
it[genre] = fetchedManga.genre
it[status] = fetchedManga.status
if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty())
it[thumbnail_url] = fetchedManga.thumbnail_url
println("transaction end $mangaId")
}
mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
MangaDataClass(
mangaId,
mangaEntry[MangaTable.sourceReference].value,
mangaEntry[MangaTable.url],
mangaEntry[MangaTable.title],
mangaEntry[MangaTable.thumbnail_url],
true,
mangaEntry[MangaTable.artist],
mangaEntry[MangaTable.author],
mangaEntry[MangaTable.description],
mangaEntry[MangaTable.genre],
MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
)
} catch (e: Exception) {
println(e)
}
}
}
fun getManga(mangaId: Int, proxyThumbnail: Boolean = true): MangaDataClass {
synchronized(getMangaCount) {
getMangaCount++
}
return try {
transaction {
var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
return@transaction if (mangaEntry[MangaTable.initialized]) {
println("${mangaEntry[MangaTable.title]} is initialized")
println("${mangaEntry[MangaTable.thumbnail_url]}")
MangaDataClass(
mangaId,
mangaEntry[MangaTable.sourceReference].value,
mangaEntry[MangaTable.url],
mangaEntry[MangaTable.title],
if (proxyThumbnail) proxyThumbnailUrl(mangaId) else mangaEntry[MangaTable.thumbnail_url],
true,
mangaEntry[MangaTable.artist],
mangaEntry[MangaTable.author],
mangaEntry[MangaTable.description],
mangaEntry[MangaTable.genre],
MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
)
} else { // initialize manga
val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value)
val fetchedManga = source.fetchMangaDetails(
SManga.create().apply {
url = mangaEntry[MangaTable.url]
title = mangaEntry[MangaTable.title]
}
).toBlocking().first()
// update database
// TODO: sqlite gets fucked here
println("putting $mangaId")
getMangaUpdateQueue.put(Pair(mangaId, fetchedManga))
// mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
val newThumbnail =
if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) {
fetchedManga.thumbnail_url
} else mangaEntry[MangaTable.thumbnail_url]
MangaDataClass(
mangaId,
mangaEntry[MangaTable.sourceReference].value,
mangaEntry[MangaTable.url],
mangaEntry[MangaTable.title],
if (proxyThumbnail) proxyThumbnailUrl(mangaId) else newThumbnail,
true,
fetchedManga.artist,
fetchedManga.author,
fetchedManga.description,
fetchedManga.genre,
MangaStatus.valueOf(fetchedManga.status).name,
)
}
}
} finally {
synchronized(getMangaCount) {
getMangaCount--
}
}
}
fun getThumbnail(mangaId: Int): Pair<InputStream, String> {
return transaction {
var filePath = Config.thumbnailsRoot + "/$mangaId"
var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
val potentialCache = findFileNameStartingWith(Config.thumbnailsRoot, mangaId.toString())
if (potentialCache != null) {
println("using cached thumbnail file")
return@transaction Pair(
pathToInputStream(potentialCache),
"image/${potentialCache.substringAfter("$mangaId.")}"
)
}
val sourceId = mangaEntry[MangaTable.sourceReference].value
println("getting source for $mangaId")
val source = getHttpSource(sourceId)
var thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
if (thumbnailUrl == null || thumbnailUrl.isEmpty()) {
thumbnailUrl = getManga(mangaId, proxyThumbnail = false).thumbnailUrl!!
}
println(thumbnailUrl)
val response = source.client.newCall(
GET(thumbnailUrl, source.headers)
).execute()
println(response.code)
if (response.code == 200) {
val contentType = response.headers["content-type"]!!
filePath += "." + contentType.substringAfter("image/")
writeStream(response.body!!.byteStream(), filePath)
return@transaction Pair(
pathToInputStream(filePath),
contentType
)
} else {
throw Exception("request error! ${response.code}")
}
}
}

62
server/src/main/kotlin/ir/armor/tachidesk/util/MangaList.kt
View File

@ -13,6 +13,10 @@ import org.jetbrains.exposed.sql.insertAndGetId
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
fun proxyThumbnailUrl(mangaId: Int): String {
return "http://127.0.0.1:4567/api/v1/manga/$mangaId/thumbnail"
}
fun getMangaList(sourceId: Long, pageNum: Int = 1, popular: Boolean): PagedMangaListDataClass {
val source = getHttpSource(sourceId.toLong())
val mangasPage = if (popular) {
@ -31,8 +35,8 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass {
val mangaList = transaction {
return@transaction mangasPage.mangas.map { manga ->
var mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull()
var mangaEntityId = if (mangaEntry == null) { // create manga entry
MangaTable.insertAndGetId {
if (mangaEntry == null) { // create manga entry
val mangaId = MangaTable.insertAndGetId {
it[url] = manga.url
it[title] = manga.title
@ -41,30 +45,46 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass {
it[description] = manga.description
it[genre] = manga.genre
it[status] = manga.status
it[thumbnail_url] = manga.genre
it[thumbnail_url] = manga.thumbnail_url
it[sourceReference] = sourceId
}.value
MangaDataClass(
mangaId,
sourceId,
manga.url,
manga.title,
proxyThumbnailUrl(mangaId),
manga.initialized,
manga.artist,
manga.author,
manga.description,
manga.genre,
MangaStatus.valueOf(manga.status).name,
)
} else {
mangaEntry[MangaTable.id].value
val mangaId = mangaEntry[MangaTable.id].value
MangaDataClass(
mangaId,
sourceId,
manga.url,
manga.title,
proxyThumbnailUrl(mangaId),
true,
mangaEntry[MangaTable.artist],
mangaEntry[MangaTable.author],
mangaEntry[MangaTable.description],
mangaEntry[MangaTable.genre],
MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
)
}
MangaDataClass(
mangaEntityId,
sourceId,
manga.url,
manga.title,
manga.thumbnail_url,
manga.initialized,
manga.artist,
manga.author,
manga.description,
manga.genre,
MangaStatus.valueOf(manga.status).name,
)
}
}
return PagedMangaListDataClass(

1
server/src/main/kotlin/ir/armor/tachidesk/util/Util.kt
View File

@ -12,6 +12,7 @@ fun applicationSetup() {
// make dirs we need
File(Config.dataRoot).mkdirs()
File(Config.extensionsRoot).mkdirs()
File(Config.thumbnailsRoot).mkdirs()
makeDataBaseTables()
}

18
webUI/react/src/components/MangaCard.tsx
View File

@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import React from 'react';
import React, { useEffect } from 'react';
import { makeStyles } from '@material-ui/core/styles';
import Card from '@material-ui/core/Card';
import CardActionArea from '@material-ui/core/CardActionArea';
@ -43,15 +43,29 @@ const useStyles = makeStyles({
interface IProps {
manga: IManga
setMangaThumbnailUrl: (thumbnailUrl:string) => void
}
const MangaCard = React.forwardRef((props: IProps, ref) => {
const {
manga: {
id, title, thumbnailUrl,
},
}, setMangaThumbnailUrl,
} = props;
const classes = useStyles();
console.log(`${title} has ${thumbnailUrl}`);
if (thumbnailUrl === null || thumbnailUrl.length === 0) {
useEffect(() => {
fetch(`http://127.0.0.1:4567/api/v1/manga/${id}/`)
.then((response) => response.json())
.then((data: IManga) => {
setMangaThumbnailUrl(data.thumbnailUrl);
});
},
[]);
}
return (
<Grid item xs={6} sm={4} md={3} lg={2}>
<Link to={`/manga/${id}/`}>

27
webUI/react/src/components/MangaGrid.tsx
View File

@ -12,15 +12,21 @@ interface IProps{
hasNextPage: boolean
lastPageNum: number
setLastPageNum: (lastPageNum: number) => void
setMangas: (mangas: IManga[]) => void
}
export default function MangaGrid(props: IProps) {
const {
mangas, message, hasNextPage, lastPageNum, setLastPageNum,
mangas, message, hasNextPage, lastPageNum, setLastPageNum, setMangas,
} = props;
let mapped;
const lastManga = useRef<HTMLInputElement>();
function setMangaThumbnailUrl(index: number, thumbnailUrl: string) {
mangas[index].thumbnailUrl = thumbnailUrl;
setMangas(mangas);
}
const scrollHandler = () => {
if (lastManga.current) {
const rect = lastManga.current.getBoundingClientRect();
@ -41,9 +47,24 @@ export default function MangaGrid(props: IProps) {
} else {
mapped = mangas.map((it, idx) => {
if (idx === mangas.length - 1) {
return <MangaCard manga={it} ref={lastManga} />;
return (
<MangaCard
manga={it}
ref={lastManga}
setMangaThumbnailUrl={
(thumbnailUrl:string) => setMangaThumbnailUrl(idx, thumbnailUrl)
}
/>
);
}
return <MangaCard manga={it} />;
return (
<MangaCard
manga={it}
setMangaThumbnailUrl={
(thumbnailUrl:string) => setMangaThumbnailUrl(idx, thumbnailUrl)
}
/>
);
});
}

1
webUI/react/src/screens/MangaList.tsx
View File

@ -37,6 +37,7 @@ export default function MangaList(props: { popular: boolean }) {
return (
<MangaGrid
mangas={mangas}
setMangas={setMangas}
hasNextPage={hasNextPage}
lastPageNum={lastPageNum}
setLastPageNum={setLastPageNum}

1
webUI/react/src/screens/SearchSingle.tsx
View File

@ -73,6 +73,7 @@ export default function SearchSingle() {
const mangaGrid = (
<MangaGrid
setMangas={setMangas}
mangas={mangas}
message={message}
hasNextPage={hasNextPage}