fixed concatenate process and rewrite of part fetch

src/api/routes/service/service.service.ts

@@ -1080,6 +1080,8 @@ export async function downloadCrunchyrollPlaylist(
 }
 
 async function downloadParts(parts: { filename: string; url: string }[], downloadID: number, dir: string, drmkeys?: { kid: string; key: string }[] | undefined) {
+    const downloadedParts: { filename: string; url: string }[] = []
+
     const path = await createFolder()
     const dn = downloading.find((i) => i.id === downloadID)
 
@@ -1087,59 +1089,84 @@ async function downloadParts(parts: { filename: string; url: string }[], downloa
     let totalSizeBytes = 0
     let startTime = Date.now()
 
-    for (const [index, part] of parts.entries()) {
-        let success = false
-        while (!success) {
-            try {
-                const { body } = await fetch(part.url)
+    async function downloadPart(part: { filename: string; url: string }, ind: number) {
+        try {
+            var stream
 
-                var stream = fs.createWriteStream(`${path}/${part.filename}`)
+            console.log(`[Video DOWNLOAD] Fetching Part ${ind + 1}`)
 
-                const readableStream = Readable.from(body as any)
-                let partDownloadedBytes = 0
-                let partSizeBytes = 0
+            const response = await fetch(part.url)
 
-                readableStream.on('data', (chunk) => {
-                    partDownloadedBytes += chunk.length
-                    totalDownloadedBytes += chunk.length
-                    totalSizeBytes += chunk.length
-                })
-
-                await finished(readableStream.pipe(stream))
-
-                console.log(`Fragment ${index + 1} downloaded`)
-
-                if (dn) {
-                    const tot = totalSizeBytes
-                    dn.downloadedParts++
-                    const endTime = Date.now()
-                    const durationInSeconds = (endTime - startTime) / 1000
-                    dn.downloadSpeed = totalDownloadedBytes / 1024 / 1024 / durationInSeconds
-                    dn.totalDownloaded = tot
-                }
-
-                success = true
-            } catch (error) {
-                console.error(`Error occurred during download of fragment ${index + 1}:`, error)
-                server.logger.log({
-                    level: 'error',
-                    message: `Error occurred during download of fragment ${index + 1}`,
-                    error: error,
-                    timestamp: new Date().toISOString(),
-                    section: 'crunchyrollDownloadProcessVideoDownload'
-                })
-                console.log(`Retrying download of fragment ${index + 1}...`)
-                server.logger.log({
-                    level: 'warn',
-                    message: `Retrying download of fragment ${index + 1} because failed`,
-                    timestamp: new Date().toISOString(),
-                    section: 'crunchyrollDownloadProcessVideoDownload'
-                })
-                await new Promise((resolve) => setTimeout(resolve, 5000))
+            if (!response.ok) {
+                throw Error(await response.text())
             }
+
+            console.log(`[Video DOWNLOAD] Writing Part ${ind + 1}`)
+
+            stream = fs.createWriteStream(`${path}/${part.filename}`)
+
+            const readableStream = Readable.from(response.body as any)
+            let partDownloadedBytes = 0
+
+            readableStream.on('data', (chunk) => {
+                partDownloadedBytes += chunk.length
+                totalDownloadedBytes += chunk.length
+                totalSizeBytes += chunk.length
+            })
+
+            await finished(readableStream.pipe(stream))
+
+            console.log(`[Video DOWNLOAD] Part ${ind + 1} downloaded`)
+
+            downloadedParts.push(part)
+
+            if (dn) {
+                const tot = totalSizeBytes
+                dn.downloadedParts++
+                const endTime = Date.now()
+                const durationInSeconds = (endTime - startTime) / 1000
+                dn.downloadSpeed = totalDownloadedBytes / 1024 / 1024 / durationInSeconds
+                dn.totalDownloaded = tot
+            }
+        } catch (error) {
+            console.error(`Error occurred during download of fragment ${ind + 1}:`, error)
+            server.logger.log({
+                level: 'error',
+                message: `Error occurred during download of fragment ${ind + 1}`,
+                error: error,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessVideoDownload'
+            })
+            console.log(`Retrying download of fragment ${ind + 1}...`)
+            server.logger.log({
+                level: 'warn',
+                message: `Retrying download of fragment ${ind + 1} because failed`,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessVideoDownload'
+            })
+            await downloadPart(part, ind)
         }
     }
 
+    for (const [index, part] of parts.entries()) {
+        await downloadPart(part, index)
+    }
+
+    if (parts[6] !== downloadedParts[6] && dn) {
+        messageBox('error', ['Cancel'], 2, 'Video Download failed', 'Video Download failed', 'Validation returned downloaded parts are invalid')
+        server.logger.log({
+            level: 'error',
+            message: 'Video Download failed',
+            error: 'Validation returned downloaded parts are invalid',
+            parts: parts,
+            partsdownloaded: downloadedParts,
+            timestamp: new Date().toISOString(),
+            section: 'VideoCrunchyrollValidation'
+        })
+        await updatePlaylistByID(downloadID, 'failed')
+        return
+    }
+
     return await mergeParts(parts, downloadID, path, dir, drmkeys)
 }
 
@@ -1167,6 +1194,17 @@ async function mergeParts(parts: { filename: string; url: string }[], downloadID
         await concatenateTSFiles(list, concatenatedFile)
 
         if (drmkeys) {
+            const found = await checkFileExistence(`${tmp}/temp-main.m4s`)
+
+            if (!found) {
+                server.logger.log({
+                    level: 'error',
+                    message: `Temp Videofile not found ${downloadID}`,
+                    timestamp: new Date().toISOString(),
+                    section: 'crunchyrollDownloadProcessVideoMerging'
+                })
+            }
+
             await updatePlaylistByID(downloadID, 'decrypting video')
             console.log('Video Decryption started')
             const inputFilePath = `${tmp}/temp-main.m4s`
@@ -1180,6 +1218,17 @@ async function mergeParts(parts: { filename: string; url: string }[], downloadID
             concatenatedFile = `${tmp}/main.m4s`
         }
 
+        const found = await checkFileExistence(`${tmp}/main.m4s`)
+
+        if (!found) {
+            server.logger.log({
+                level: 'error',
+                message: `Temp Videofile not found ${downloadID}`,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessVideoMerging'
+            })
+        }
+
         return new Promise((resolve, reject) => {
             if (!ffmpegP.ffmpeg || !ffmpegP.ffprobe) return
             Ffmpeg()

src/api/services/audio.ts

@@ -1,6 +1,6 @@
 import fs from 'fs'
 import { Readable } from 'stream'
-import { createFolder, deleteFolder } from './folder'
+import { checkFileExistence, createFolder, deleteFolder } from './folder'
 import { concatenateTSFiles } from './concatenate'
 import Ffmpeg from 'fluent-ffmpeg'
 import { getFFMPEGPath } from './ffmpeg'
@@ -11,6 +11,7 @@ import util from 'util'
 import { server } from '../api'
 const exec = util.promisify(require('child_process').exec)
 import { finished } from 'stream/promises'
+import { messageBox } from '../../electron/background'
 
 // Define Downloading Array
 var downloading: Array<{
@@ -34,6 +35,8 @@ export async function downloadMPDAudio(
     downloadID: number,
     drmkeys?: { kid: string; key: string }[] | undefined
 ) {
+    const downloadedParts: { filename: string; url: string }[] = []
+
     const path = await createFolder()
 
     downloading.push({
@@ -44,51 +47,70 @@ export async function downloadMPDAudio(
 
     const dn = downloading.find((i) => i.id === downloadID && i.audio === name)
 
-    for (const [index, part] of parts.entries()) {
-        let success = false
-        while (!success) {
-            try {
-                var stream
+    async function downloadPart(part: { filename: string; url: string }, ind: number) {
+        try {
+            var stream
 
-                const response = await fetch(part.url)
+            console.log(`[${name} DOWNLOAD] Fetching Part ${ind + 1}`)
 
-                if (!response.ok) {
-                    throw Error(await response.text())
-                }
+            const response = await fetch(part.url)
 
-                stream = fs.createWriteStream(`${path}/${part.filename}`)
-
-                const readableStream = Readable.from(response.body as any)
-
-                await finished(readableStream.pipe(stream))
-
-                console.log(`Fragment ${index + 1} downloaded`)
-
-                success = true
-            } catch (error) {
-                if (dn) {
-                    dn.status = 'failed'
-                }
-                console.error(`Error occurred during download of fragment ${index + 1}:`, error)
-                server.logger.log({
-                    level: 'error',
-                    message: `Error occurred during download of fragment ${index + 1}`,
-                    error: error,
-                    timestamp: new Date().toISOString(),
-                    section: 'crunchyrollDownloadProcessAudioDownload'
-                })
-                console.log(`Retrying download of fragment ${index + 1}...`)
-                server.logger.log({
-                    level: 'warn',
-                    message: `Retrying download of fragment ${index + 1} because failed`,
-                    timestamp: new Date().toISOString(),
-                    section: 'crunchyrollDownloadProcessAudioDownload'
-                })
-                await new Promise((resolve) => setTimeout(resolve, 5000))
+            if (!response.ok) {
+                throw Error(await response.text())
             }
+
+            console.log(`[${name} DOWNLOAD] Writing Part ${ind + 1}`)
+
+            stream = fs.createWriteStream(`${path}/${part.filename}`)
+
+            const readableStream = Readable.from(response.body as any)
+
+            await finished(readableStream.pipe(stream))
+
+            console.log(`[${name} DOWNLOAD] Part ${ind + 1} downloaded`)
+
+            downloadedParts.push(part)
+        } catch (error) {
+            console.error(`Error occurred during download of fragment ${ind + 1}:`, error)
+            server.logger.log({
+                level: 'error',
+                message: `Error occurred during download of fragment ${ind + 1}`,
+                error: error,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessAudioDownload'
+            })
+            console.log(`Retrying download of fragment ${ind + 1}...`)
+            server.logger.log({
+                level: 'warn',
+                message: `Retrying download of fragment ${ind + 1} because failed`,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessAudioDownload'
+            })
+            await downloadPart(part, ind)
         }
     }
 
+    for (const [index, part] of parts.entries()) {
+        await downloadPart(part, index)
+    }
+
+    if (parts[6] !== downloadedParts[6] && dn) {
+        dn.status = 'failed'
+        messageBox('error', ['Cancel'], 2, 'Audio Download failed', 'Audio Download failed', 'Validation returned downloaded parts are invalid')
+        server.logger.log({
+            level: 'error',
+            message: 'Audio Download failed',
+            error: 'Validation returned downloaded parts are invalid',
+            parts: parts,
+            partsdownloaded: downloadedParts,
+            timestamp: new Date().toISOString(),
+            section: 'AudioCrunchyrollValidation'
+        })
+        return
+    }
+
+    console.log(`[${name} DOWNLOAD] Parts validated`)
+
     return await mergePartsAudio(parts, path, dir, name, downloadID, drmkeys)
 }
 
@@ -124,6 +146,17 @@ async function mergePartsAudio(
         await concatenateTSFiles(list, concatenatedFile)
 
         if (drmkeys) {
+            const found = await checkFileExistence(`${tmp}/temp-main.m4s`)
+
+            if (!found) {
+                server.logger.log({
+                    level: 'error',
+                    message: `Temp Audiofile not found ${name}`,
+                    timestamp: new Date().toISOString(),
+                    section: 'crunchyrollDownloadProcessAudioMerging'
+                })
+            }
+
             if (dn) {
                 dn.status = 'decrypting'
             }
@@ -139,6 +172,17 @@ async function mergePartsAudio(
             console.log(`Audio Decryption finished`)
         }
 
+        const found = await checkFileExistence(`${tmp}/main.m4s`)
+
+        if (!found) {
+            server.logger.log({
+                level: 'error',
+                message: `Audiofile not found ${name}`,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessAudioMerging'
+            })
+        }
+
         return new Promise((resolve, reject) => {
             if (!ffmpegP.ffmpeg || !ffmpegP.ffprobe) return
             Ffmpeg()

src/api/services/concatenate.ts

@@ -1,4 +1,5 @@
 import fs from 'fs'
+import { finished } from 'stream'
 import { server } from '../api'
 
 export async function concatenateTSFiles(inputFiles: Array<string>, outputFile: string) {
@@ -16,14 +17,23 @@ export async function concatenateTSFiles(inputFiles: Array<string>, outputFile:
             reject(error)
         })
 
-        writeStream.on('finish', () => {
-            console.log('TS files concatenated successfully!')
-            resolve()
-        })
-
         const processNextFile = (index: number) => {
             if (index >= inputFiles.length) {
                 writeStream.end()
+                finished(writeStream, (err) => {
+                    if (err) {
+                        server.logger.log({
+                            level: 'error',
+                            message: `Error while finishing write stream`,
+                            error: err,
+                            timestamp: new Date().toISOString(),
+                            section: 'crunchyrollDownloadProcessConcatenate'
+                        })
+                        return reject(err)
+                    }
+                    console.log('TS files concatenated successfully!')
+                    resolve()
+                })
                 return
             }