fixed concatenate process and rewrite of part fetch

src/api/routes/service/service.service.ts

@@ -1080,6 +1080,8 @@ export async function downloadCrunchyrollPlaylist(
 }
 
 async function downloadParts(parts: { filename: string; url: string }[], downloadID: number, dir: string, drmkeys?: { kid: string; key: string }[] | undefined) {
+    const downloadedParts: { filename: string; url: string }[] = []
+
     const path = await createFolder()
     const dn = downloading.find((i) => i.id === downloadID)
 
@@ -1087,17 +1089,24 @@ async function downloadParts(parts: { filename: string; url: string }[], downloa
     let totalSizeBytes = 0
     let startTime = Date.now()
 
-    for (const [index, part] of parts.entries()) {
+    async function downloadPart(part: { filename: string; url: string }, ind: number) {
-        let success = false
-        while (!success) {
         try {
-                const { body } = await fetch(part.url)
+            var stream
 
-                var stream = fs.createWriteStream(`${path}/${part.filename}`)
+            console.log(`[Video DOWNLOAD] Fetching Part ${ind + 1}`)
 
-                const readableStream = Readable.from(body as any)
+            const response = await fetch(part.url)
+
+            if (!response.ok) {
+                throw Error(await response.text())
+            }
+
+            console.log(`[Video DOWNLOAD] Writing Part ${ind + 1}`)
+
+            stream = fs.createWriteStream(`${path}/${part.filename}`)
+
+            const readableStream = Readable.from(response.body as any)
             let partDownloadedBytes = 0
-                let partSizeBytes = 0
 
             readableStream.on('data', (chunk) => {
                 partDownloadedBytes += chunk.length
@@ -1107,7 +1116,9 @@ async function downloadParts(parts: { filename: string; url: string }[], downloa
 
             await finished(readableStream.pipe(stream))
 
-                console.log(`Fragment ${index + 1} downloaded`)
+            console.log(`[Video DOWNLOAD] Part ${ind + 1} downloaded`)
+
+            downloadedParts.push(part)
 
             if (dn) {
                 const tot = totalSizeBytes
@@ -1117,27 +1128,43 @@ async function downloadParts(parts: { filename: string; url: string }[], downloa
                 dn.downloadSpeed = totalDownloadedBytes / 1024 / 1024 / durationInSeconds
                 dn.totalDownloaded = tot
             }
-
-                success = true
         } catch (error) {
-                console.error(`Error occurred during download of fragment ${index + 1}:`, error)
+            console.error(`Error occurred during download of fragment ${ind + 1}:`, error)
             server.logger.log({
                 level: 'error',
-                    message: `Error occurred during download of fragment ${index + 1}`,
+                message: `Error occurred during download of fragment ${ind + 1}`,
                 error: error,
                 timestamp: new Date().toISOString(),
                 section: 'crunchyrollDownloadProcessVideoDownload'
             })
-                console.log(`Retrying download of fragment ${index + 1}...`)
+            console.log(`Retrying download of fragment ${ind + 1}...`)
             server.logger.log({
                 level: 'warn',
-                    message: `Retrying download of fragment ${index + 1} because failed`,
+                message: `Retrying download of fragment ${ind + 1} because failed`,
                 timestamp: new Date().toISOString(),
                 section: 'crunchyrollDownloadProcessVideoDownload'
             })
-                await new Promise((resolve) => setTimeout(resolve, 5000))
+            await downloadPart(part, ind)
         }
     }
+
+    for (const [index, part] of parts.entries()) {
+        await downloadPart(part, index)
+    }
+
+    if (parts[6] !== downloadedParts[6] && dn) {
+        messageBox('error', ['Cancel'], 2, 'Video Download failed', 'Video Download failed', 'Validation returned downloaded parts are invalid')
+        server.logger.log({
+            level: 'error',
+            message: 'Video Download failed',
+            error: 'Validation returned downloaded parts are invalid',
+            parts: parts,
+            partsdownloaded: downloadedParts,
+            timestamp: new Date().toISOString(),
+            section: 'VideoCrunchyrollValidation'
+        })
+        await updatePlaylistByID(downloadID, 'failed')
+        return
     }
 
     return await mergeParts(parts, downloadID, path, dir, drmkeys)
@@ -1167,6 +1194,17 @@ async function mergeParts(parts: { filename: string; url: string }[], downloadID
         await concatenateTSFiles(list, concatenatedFile)
 
         if (drmkeys) {
+            const found = await checkFileExistence(`${tmp}/temp-main.m4s`)
+
+            if (!found) {
+                server.logger.log({
+                    level: 'error',
+                    message: `Temp Videofile not found ${downloadID}`,
+                    timestamp: new Date().toISOString(),
+                    section: 'crunchyrollDownloadProcessVideoMerging'
+                })
+            }
+
             await updatePlaylistByID(downloadID, 'decrypting video')
             console.log('Video Decryption started')
             const inputFilePath = `${tmp}/temp-main.m4s`
@@ -1180,6 +1218,17 @@ async function mergeParts(parts: { filename: string; url: string }[], downloadID
             concatenatedFile = `${tmp}/main.m4s`
         }
 
+        const found = await checkFileExistence(`${tmp}/main.m4s`)
+
+        if (!found) {
+            server.logger.log({
+                level: 'error',
+                message: `Temp Videofile not found ${downloadID}`,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessVideoMerging'
+            })
+        }
+
         return new Promise((resolve, reject) => {
             if (!ffmpegP.ffmpeg || !ffmpegP.ffprobe) return
             Ffmpeg()

src/api/services/audio.ts

@@ -1,6 +1,6 @@
 import fs from 'fs'
 import { Readable } from 'stream'
-import { createFolder, deleteFolder } from './folder'
+import { checkFileExistence, createFolder, deleteFolder } from './folder'
 import { concatenateTSFiles } from './concatenate'
 import Ffmpeg from 'fluent-ffmpeg'
 import { getFFMPEGPath } from './ffmpeg'
@@ -11,6 +11,7 @@ import util from 'util'
 import { server } from '../api'
 const exec = util.promisify(require('child_process').exec)
 import { finished } from 'stream/promises'
+import { messageBox } from '../../electron/background'
 
 // Define Downloading Array
 var downloading: Array<{
@@ -34,6 +35,8 @@ export async function downloadMPDAudio(
     downloadID: number,
     drmkeys?: { kid: string; key: string }[] | undefined
 ) {
+    const downloadedParts: { filename: string; url: string }[] = []
+
     const path = await createFolder()
 
     downloading.push({
@@ -44,51 +47,70 @@ export async function downloadMPDAudio(
 
     const dn = downloading.find((i) => i.id === downloadID && i.audio === name)
 
-    for (const [index, part] of parts.entries()) {
+    async function downloadPart(part: { filename: string; url: string }, ind: number) {
-        let success = false
-        while (!success) {
         try {
             var stream
 
+            console.log(`[${name} DOWNLOAD] Fetching Part ${ind + 1}`)
+
             const response = await fetch(part.url)
 
             if (!response.ok) {
                 throw Error(await response.text())
             }
 
+            console.log(`[${name} DOWNLOAD] Writing Part ${ind + 1}`)
+
             stream = fs.createWriteStream(`${path}/${part.filename}`)
 
             const readableStream = Readable.from(response.body as any)
 
             await finished(readableStream.pipe(stream))
 
-                console.log(`Fragment ${index + 1} downloaded`)
+            console.log(`[${name} DOWNLOAD] Part ${ind + 1} downloaded`)
 
-                success = true
+            downloadedParts.push(part)
         } catch (error) {
-                if (dn) {
+            console.error(`Error occurred during download of fragment ${ind + 1}:`, error)
-                    dn.status = 'failed'
-                }
-                console.error(`Error occurred during download of fragment ${index + 1}:`, error)
             server.logger.log({
                 level: 'error',
-                    message: `Error occurred during download of fragment ${index + 1}`,
+                message: `Error occurred during download of fragment ${ind + 1}`,
                 error: error,
                 timestamp: new Date().toISOString(),
                 section: 'crunchyrollDownloadProcessAudioDownload'
             })
-                console.log(`Retrying download of fragment ${index + 1}...`)
+            console.log(`Retrying download of fragment ${ind + 1}...`)
             server.logger.log({
                 level: 'warn',
-                    message: `Retrying download of fragment ${index + 1} because failed`,
+                message: `Retrying download of fragment ${ind + 1} because failed`,
                 timestamp: new Date().toISOString(),
                 section: 'crunchyrollDownloadProcessAudioDownload'
             })
-                await new Promise((resolve) => setTimeout(resolve, 5000))
+            await downloadPart(part, ind)
         }
     }
+
+    for (const [index, part] of parts.entries()) {
+        await downloadPart(part, index)
     }
 
+    if (parts[6] !== downloadedParts[6] && dn) {
+        dn.status = 'failed'
+        messageBox('error', ['Cancel'], 2, 'Audio Download failed', 'Audio Download failed', 'Validation returned downloaded parts are invalid')
+        server.logger.log({
+            level: 'error',
+            message: 'Audio Download failed',
+            error: 'Validation returned downloaded parts are invalid',
+            parts: parts,
+            partsdownloaded: downloadedParts,
+            timestamp: new Date().toISOString(),
+            section: 'AudioCrunchyrollValidation'
+        })
+        return
+    }
+
+    console.log(`[${name} DOWNLOAD] Parts validated`)
+
     return await mergePartsAudio(parts, path, dir, name, downloadID, drmkeys)
 }
 
@@ -124,6 +146,17 @@ async function mergePartsAudio(
         await concatenateTSFiles(list, concatenatedFile)
 
         if (drmkeys) {
+            const found = await checkFileExistence(`${tmp}/temp-main.m4s`)
+
+            if (!found) {
+                server.logger.log({
+                    level: 'error',
+                    message: `Temp Audiofile not found ${name}`,
+                    timestamp: new Date().toISOString(),
+                    section: 'crunchyrollDownloadProcessAudioMerging'
+                })
+            }
+
             if (dn) {
                 dn.status = 'decrypting'
             }
@@ -139,6 +172,17 @@ async function mergePartsAudio(
             console.log(`Audio Decryption finished`)
         }
 
+        const found = await checkFileExistence(`${tmp}/main.m4s`)
+
+        if (!found) {
+            server.logger.log({
+                level: 'error',
+                message: `Audiofile not found ${name}`,
+                timestamp: new Date().toISOString(),
+                section: 'crunchyrollDownloadProcessAudioMerging'
+            })
+        }
+
         return new Promise((resolve, reject) => {
             if (!ffmpegP.ffmpeg || !ffmpegP.ffprobe) return
             Ffmpeg()

src/api/services/concatenate.ts

@@ -1,4 +1,5 @@
 import fs from 'fs'
+import { finished } from 'stream'
 import { server } from '../api'
 
 export async function concatenateTSFiles(inputFiles: Array<string>, outputFile: string) {
@@ -16,14 +17,23 @@ export async function concatenateTSFiles(inputFiles: Array<string>, outputFile:
             reject(error)
         })
 
-        writeStream.on('finish', () => {
-            console.log('TS files concatenated successfully!')
-            resolve()
-        })
-
         const processNextFile = (index: number) => {
             if (index >= inputFiles.length) {
                 writeStream.end()
+                finished(writeStream, (err) => {
+                    if (err) {
+                        server.logger.log({
+                            level: 'error',
+                            message: `Error while finishing write stream`,
+                            error: err,
+                            timestamp: new Date().toISOString(),
+                            section: 'crunchyrollDownloadProcessConcatenate'
+                        })
+                        return reject(err)
+                    }
+                    console.log('TS files concatenated successfully!')
+                    resolve()
+                })
                 return
             }