From b56045e98403da35b99024f9cf0dae9b3d45b6be Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Sat, 29 May 2021 23:05:51 +0430 Subject: [PATCH] downloader backend done --- .../kotlin/suwayomi/tachidesk/TachideskAPI.kt | 52 +++++++- .../impl/download/DownloadManager.kt | 124 ++++++++++++++++++ .../tachidesk/impl/download/Downloader.kt | 63 +++++++-- .../impl/download/model/DownloadChapter.kt | 18 +++ .../impl/download/model/DownloadState.kt | 15 +++ .../impl/download/model/DownloadStatus.kt | 13 ++ 6 files changed, 267 insertions(+), 18 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/impl/download/DownloadManager.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadChapter.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadState.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadStatus.kt diff --git a/server/src/main/kotlin/suwayomi/tachidesk/TachideskAPI.kt b/server/src/main/kotlin/suwayomi/tachidesk/TachideskAPI.kt index 97b9dd9..5c7df9c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/TachideskAPI.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/TachideskAPI.kt @@ -33,6 +33,7 @@ import suwayomi.tachidesk.impl.Source.getSourceList import suwayomi.tachidesk.impl.backup.BackupFlags import suwayomi.tachidesk.impl.backup.legacy.LegacyBackupExport.createLegacyBackup import suwayomi.tachidesk.impl.backup.legacy.LegacyBackupImport.restoreLegacyBackup +import suwayomi.tachidesk.impl.download.DownloadManager import suwayomi.tachidesk.impl.extension.Extension.getExtensionIcon import suwayomi.tachidesk.impl.extension.Extension.installExtension import suwayomi.tachidesk.impl.extension.Extension.uninstallExtension @@ -383,15 +384,56 @@ object TachideskAPI { // Download queue stats app.ws("/api/v1/downloads") { ws -> ws.onConnect { ctx -> - // TODO: send current stat - // TODO: add to downlad subscribers + DownloadManager.addClient(ctx) + DownloadManager.notifyClient(ctx) } - ws.onMessage { - // TODO: send current stat + ws.onMessage { ctx -> + DownloadManager.handleRequest(ctx) } ws.onClose { ctx -> - // TODO: remove from subscribers + DownloadManager.removeClient(ctx) } } + + // Start the downloader + app.get("/api/v1/downloads/start") { ctx -> + DownloadManager.start() + + ctx.status(200) + } + + // Stop the downloader + app.get("/api/v1/downloads/stop") { ctx -> + DownloadManager.stop() + + ctx.status(200) + } + + // clear download queue + app.get("/api/v1/downloads/clear") { ctx -> + DownloadManager.clear() + + ctx.status(200) + } + + // Queue chapter for download + app.get("/api/v1/download/:mangaId/chapter/:chapterIndex") { ctx -> + val chapterIndex = ctx.pathParam("chapterIndex").toInt() + val mangaId = ctx.pathParam("mangaId").toInt() + + DownloadManager.enqueue(chapterIndex, mangaId) + + ctx.status(200) + } + + // delete chapter from download queue + app.delete("/api/v1/download/:mangaId/chapter/:chapterIndex") { ctx -> + val chapterIndex = ctx.pathParam("chapterIndex").toInt() + val mangaId = ctx.pathParam("mangaId").toInt() + + DownloadManager.unqueue(chapterIndex, mangaId) + + ctx.status(200) + } } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/impl/download/DownloadManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/DownloadManager.kt new file mode 100644 index 0000000..f7eda6a --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/DownloadManager.kt @@ -0,0 +1,124 @@ +package suwayomi.tachidesk.impl.download + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import io.javalin.websocket.WsContext +import io.javalin.websocket.WsMessageContext +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.transactions.transaction +import suwayomi.tachidesk.impl.download.model.DownloadChapter +import suwayomi.tachidesk.impl.download.model.DownloadState.Downloading +import suwayomi.tachidesk.impl.download.model.DownloadStatus +import suwayomi.tachidesk.model.table.ChapterTable +import suwayomi.tachidesk.model.table.toDataClass +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + +object DownloadManager { + private val clients = ConcurrentHashMap() + private val downloadQueue = CopyOnWriteArrayList() + private var downloader: Downloader? = null + + fun addClient(ctx: WsContext) { + clients[ctx.sessionId] = ctx + } + + fun removeClient(ctx: WsContext) { + clients.remove(ctx.sessionId) + } + + fun notifyClient(ctx: WsContext) { + ctx.send( + getStatus() + ) + } + + fun handleRequest(ctx: WsMessageContext) { + when (ctx.message()) { + "STATUS" -> notifyClient(ctx) + else -> ctx.send( + """ + |Invalid command. + |Supported commands are: + | - STATUS + | sends the current download status + |""".trimMargin() + ) + } + } + + private fun notifyAllClients() { + val status = getStatus() + clients.forEach { + it.value.send(status) + } + } + + private fun getStatus(): DownloadStatus { + return DownloadStatus( + if (downloader == null || + downloadQueue.none { it.state == Downloading } + ) "Stopped" else "Started", + downloadQueue + ) + } + + fun enqueue(chapterIndex: Int, mangaId: Int) { + if (downloadQueue.none { it.mangaId == mangaId && it.chapterIndex == chapterIndex }) { + downloadQueue.add( + DownloadChapter( + chapterIndex, + mangaId, + chapter = ChapterTable.toDataClass( + transaction { + ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.chapterIndex eq chapterIndex) } + .first() + } + ) + ) + ) + } + notifyAllClients() + } + + fun unqueue(chapterIndex: Int, mangaId: Int) { + downloadQueue.removeIf { it.mangaId == mangaId && it.chapterIndex == chapterIndex } + notifyAllClients() + } + + fun start() { + if (downloader == null) { + downloader = Downloader(downloadQueue) { notifyAllClients() } + downloader!!.start() + } + notifyAllClients() + } + + fun stop() { + downloader?.let { + synchronized(it.shouldStop) { + it.shouldStop = true + } + } + downloader = null + notifyAllClients() + } + + fun clear() { + stop() + downloadQueue.clear() + notifyAllClients() + } +} + +enum class DownloaderState(val state: Int) { + Stopped(0), + Running(1), + Paused(2), +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/impl/download/Downloader.kt b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/Downloader.kt index d5e5be6..aed8cc7 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/impl/download/Downloader.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/Downloader.kt @@ -1,28 +1,65 @@ package suwayomi.tachidesk.impl.download -import org.jetbrains.exposed.sql.ResultRow -import java.util.concurrent.LinkedBlockingQueue - /* * Copyright (C) Contributors to the Suwayomi project - * + * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -data class Download( - val chapter: ResultRow, -) +import kotlinx.coroutines.runBlocking +import suwayomi.tachidesk.impl.Chapter.getChapter +import suwayomi.tachidesk.impl.Page.getPageImage +import suwayomi.tachidesk.impl.download.model.DownloadChapter +import suwayomi.tachidesk.impl.download.model.DownloadState.Downloading +import suwayomi.tachidesk.impl.download.model.DownloadState.Error +import suwayomi.tachidesk.impl.download.model.DownloadState.Finished +import suwayomi.tachidesk.impl.download.model.DownloadState.Queued +import java.util.concurrent.CopyOnWriteArrayList -private val downloadQueue = LinkedBlockingQueue() +class Downloader(private val downloadQueue: CopyOnWriteArrayList, val notifier: () -> Unit) : Thread() { + var shouldStop: Boolean = false -class Downloader { + class DownloadShouldStopException : Exception() - fun start() { - TODO() + fun step() { + notifier() + synchronized(shouldStop) { + if (shouldStop) throw DownloadShouldStopException() + } } - fun stop() { - TODO() + override fun run() { + do { + val download = downloadQueue.firstOrNull { it.state == Queued } ?: break + + try { + download.state = Downloading + step() + + download.chapter = runBlocking { getChapter(download.chapterIndex, download.mangaId) } + step() + + val pageCount = download.chapter!!.pageCount!! + for (pageNum in 0 until pageCount) { + runBlocking { getPageImage(download.mangaId, download.chapterIndex, pageNum) } + // TODO: retry on error with 2,4,8 seconds of wait + // TODO: download multiple pages at once, possible solution: rx observer's strategy is used in Tachiyomi + download.progress = (pageNum + 1).toFloat() / pageCount + step() + } + download.state = Finished + step() + } catch (e: DownloadShouldStopException) { + println("Downloader was stopped") + downloadQueue.filter { it.state == Downloading }.forEach { it.state = Queued } + } catch (e: Exception) { + println("Downloader faced an exception") + downloadQueue.filter { it.state == Downloading }.forEach { it.state = Error } + e.printStackTrace() + } finally { + notifier() + } + } while (!shouldStop) } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadChapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadChapter.kt new file mode 100644 index 0000000..62b2526 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadChapter.kt @@ -0,0 +1,18 @@ +package suwayomi.tachidesk.impl.download.model + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import suwayomi.tachidesk.model.dataclass.ChapterDataClass + +class DownloadChapter( + val chapterIndex: Int, + val mangaId: Int, + var state: DownloadState = DownloadState.Queued, + var progress: Float = 0f, + var chapter: ChapterDataClass? = null, +) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadState.kt b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadState.kt new file mode 100644 index 0000000..89d453d --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadState.kt @@ -0,0 +1,15 @@ +package suwayomi.tachidesk.impl.download.model + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +enum class DownloadState(val state: Int) { + Queued(0), + Downloading(1), + Finished(2), + Error(3), +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadStatus.kt b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadStatus.kt new file mode 100644 index 0000000..1bf9e22 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/impl/download/model/DownloadStatus.kt @@ -0,0 +1,13 @@ +package suwayomi.tachidesk.impl.download.model + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +data class DownloadStatus( + val status: String, + val queue: List, +)