fixed concatenate process and rewrite of part fetch

This commit is contained in:
stratuma 2024-06-02 02:31:34 +02:00
parent 94e82e0f66
commit 73ecea4a64
3 changed files with 193 additions and 90 deletions

View File

@ -1080,6 +1080,8 @@ export async function downloadCrunchyrollPlaylist(
}
async function downloadParts(parts: { filename: string; url: string }[], downloadID: number, dir: string, drmkeys?: { kid: string; key: string }[] | undefined) {
const downloadedParts: { filename: string; url: string }[] = []
const path = await createFolder()
const dn = downloading.find((i) => i.id === downloadID)
@ -1087,59 +1089,84 @@ async function downloadParts(parts: { filename: string; url: string }[], downloa
let totalSizeBytes = 0
let startTime = Date.now()
for (const [index, part] of parts.entries()) {
let success = false
while (!success) {
try {
const { body } = await fetch(part.url)
async function downloadPart(part: { filename: string; url: string }, ind: number) {
try {
var stream
var stream = fs.createWriteStream(`${path}/${part.filename}`)
console.log(`[Video DOWNLOAD] Fetching Part ${ind + 1}`)
const readableStream = Readable.from(body as any)
let partDownloadedBytes = 0
let partSizeBytes = 0
const response = await fetch(part.url)
readableStream.on('data', (chunk) => {
partDownloadedBytes += chunk.length
totalDownloadedBytes += chunk.length
totalSizeBytes += chunk.length
})
await finished(readableStream.pipe(stream))
console.log(`Fragment ${index + 1} downloaded`)
if (dn) {
const tot = totalSizeBytes
dn.downloadedParts++
const endTime = Date.now()
const durationInSeconds = (endTime - startTime) / 1000
dn.downloadSpeed = totalDownloadedBytes / 1024 / 1024 / durationInSeconds
dn.totalDownloaded = tot
}
success = true
} catch (error) {
console.error(`Error occurred during download of fragment ${index + 1}:`, error)
server.logger.log({
level: 'error',
message: `Error occurred during download of fragment ${index + 1}`,
error: error,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessVideoDownload'
})
console.log(`Retrying download of fragment ${index + 1}...`)
server.logger.log({
level: 'warn',
message: `Retrying download of fragment ${index + 1} because failed`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessVideoDownload'
})
await new Promise((resolve) => setTimeout(resolve, 5000))
if (!response.ok) {
throw Error(await response.text())
}
console.log(`[Video DOWNLOAD] Writing Part ${ind + 1}`)
stream = fs.createWriteStream(`${path}/${part.filename}`)
const readableStream = Readable.from(response.body as any)
let partDownloadedBytes = 0
readableStream.on('data', (chunk) => {
partDownloadedBytes += chunk.length
totalDownloadedBytes += chunk.length
totalSizeBytes += chunk.length
})
await finished(readableStream.pipe(stream))
console.log(`[Video DOWNLOAD] Part ${ind + 1} downloaded`)
downloadedParts.push(part)
if (dn) {
const tot = totalSizeBytes
dn.downloadedParts++
const endTime = Date.now()
const durationInSeconds = (endTime - startTime) / 1000
dn.downloadSpeed = totalDownloadedBytes / 1024 / 1024 / durationInSeconds
dn.totalDownloaded = tot
}
} catch (error) {
console.error(`Error occurred during download of fragment ${ind + 1}:`, error)
server.logger.log({
level: 'error',
message: `Error occurred during download of fragment ${ind + 1}`,
error: error,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessVideoDownload'
})
console.log(`Retrying download of fragment ${ind + 1}...`)
server.logger.log({
level: 'warn',
message: `Retrying download of fragment ${ind + 1} because failed`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessVideoDownload'
})
await downloadPart(part, ind)
}
}
for (const [index, part] of parts.entries()) {
await downloadPart(part, index)
}
if (parts[6] !== downloadedParts[6] && dn) {
messageBox('error', ['Cancel'], 2, 'Video Download failed', 'Video Download failed', 'Validation returned downloaded parts are invalid')
server.logger.log({
level: 'error',
message: 'Video Download failed',
error: 'Validation returned downloaded parts are invalid',
parts: parts,
partsdownloaded: downloadedParts,
timestamp: new Date().toISOString(),
section: 'VideoCrunchyrollValidation'
})
await updatePlaylistByID(downloadID, 'failed')
return
}
return await mergeParts(parts, downloadID, path, dir, drmkeys)
}
@ -1167,6 +1194,17 @@ async function mergeParts(parts: { filename: string; url: string }[], downloadID
await concatenateTSFiles(list, concatenatedFile)
if (drmkeys) {
const found = await checkFileExistence(`${tmp}/temp-main.m4s`)
if (!found) {
server.logger.log({
level: 'error',
message: `Temp Videofile not found ${downloadID}`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessVideoMerging'
})
}
await updatePlaylistByID(downloadID, 'decrypting video')
console.log('Video Decryption started')
const inputFilePath = `${tmp}/temp-main.m4s`
@ -1180,6 +1218,17 @@ async function mergeParts(parts: { filename: string; url: string }[], downloadID
concatenatedFile = `${tmp}/main.m4s`
}
const found = await checkFileExistence(`${tmp}/main.m4s`)
if (!found) {
server.logger.log({
level: 'error',
message: `Temp Videofile not found ${downloadID}`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessVideoMerging'
})
}
return new Promise((resolve, reject) => {
if (!ffmpegP.ffmpeg || !ffmpegP.ffprobe) return
Ffmpeg()

View File

@ -1,6 +1,6 @@
import fs from 'fs'
import { Readable } from 'stream'
import { createFolder, deleteFolder } from './folder'
import { checkFileExistence, createFolder, deleteFolder } from './folder'
import { concatenateTSFiles } from './concatenate'
import Ffmpeg from 'fluent-ffmpeg'
import { getFFMPEGPath } from './ffmpeg'
@ -11,6 +11,7 @@ import util from 'util'
import { server } from '../api'
const exec = util.promisify(require('child_process').exec)
import { finished } from 'stream/promises'
import { messageBox } from '../../electron/background'
// Define Downloading Array
var downloading: Array<{
@ -34,6 +35,8 @@ export async function downloadMPDAudio(
downloadID: number,
drmkeys?: { kid: string; key: string }[] | undefined
) {
const downloadedParts: { filename: string; url: string }[] = []
const path = await createFolder()
downloading.push({
@ -44,51 +47,70 @@ export async function downloadMPDAudio(
const dn = downloading.find((i) => i.id === downloadID && i.audio === name)
for (const [index, part] of parts.entries()) {
let success = false
while (!success) {
try {
var stream
async function downloadPart(part: { filename: string; url: string }, ind: number) {
try {
var stream
const response = await fetch(part.url)
console.log(`[${name} DOWNLOAD] Fetching Part ${ind + 1}`)
if (!response.ok) {
throw Error(await response.text())
}
const response = await fetch(part.url)
stream = fs.createWriteStream(`${path}/${part.filename}`)
const readableStream = Readable.from(response.body as any)
await finished(readableStream.pipe(stream))
console.log(`Fragment ${index + 1} downloaded`)
success = true
} catch (error) {
if (dn) {
dn.status = 'failed'
}
console.error(`Error occurred during download of fragment ${index + 1}:`, error)
server.logger.log({
level: 'error',
message: `Error occurred during download of fragment ${index + 1}`,
error: error,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessAudioDownload'
})
console.log(`Retrying download of fragment ${index + 1}...`)
server.logger.log({
level: 'warn',
message: `Retrying download of fragment ${index + 1} because failed`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessAudioDownload'
})
await new Promise((resolve) => setTimeout(resolve, 5000))
if (!response.ok) {
throw Error(await response.text())
}
console.log(`[${name} DOWNLOAD] Writing Part ${ind + 1}`)
stream = fs.createWriteStream(`${path}/${part.filename}`)
const readableStream = Readable.from(response.body as any)
await finished(readableStream.pipe(stream))
console.log(`[${name} DOWNLOAD] Part ${ind + 1} downloaded`)
downloadedParts.push(part)
} catch (error) {
console.error(`Error occurred during download of fragment ${ind + 1}:`, error)
server.logger.log({
level: 'error',
message: `Error occurred during download of fragment ${ind + 1}`,
error: error,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessAudioDownload'
})
console.log(`Retrying download of fragment ${ind + 1}...`)
server.logger.log({
level: 'warn',
message: `Retrying download of fragment ${ind + 1} because failed`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessAudioDownload'
})
await downloadPart(part, ind)
}
}
for (const [index, part] of parts.entries()) {
await downloadPart(part, index)
}
if (parts[6] !== downloadedParts[6] && dn) {
dn.status = 'failed'
messageBox('error', ['Cancel'], 2, 'Audio Download failed', 'Audio Download failed', 'Validation returned downloaded parts are invalid')
server.logger.log({
level: 'error',
message: 'Audio Download failed',
error: 'Validation returned downloaded parts are invalid',
parts: parts,
partsdownloaded: downloadedParts,
timestamp: new Date().toISOString(),
section: 'AudioCrunchyrollValidation'
})
return
}
console.log(`[${name} DOWNLOAD] Parts validated`)
return await mergePartsAudio(parts, path, dir, name, downloadID, drmkeys)
}
@ -124,6 +146,17 @@ async function mergePartsAudio(
await concatenateTSFiles(list, concatenatedFile)
if (drmkeys) {
const found = await checkFileExistence(`${tmp}/temp-main.m4s`)
if (!found) {
server.logger.log({
level: 'error',
message: `Temp Audiofile not found ${name}`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessAudioMerging'
})
}
if (dn) {
dn.status = 'decrypting'
}
@ -139,6 +172,17 @@ async function mergePartsAudio(
console.log(`Audio Decryption finished`)
}
const found = await checkFileExistence(`${tmp}/main.m4s`)
if (!found) {
server.logger.log({
level: 'error',
message: `Audiofile not found ${name}`,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessAudioMerging'
})
}
return new Promise((resolve, reject) => {
if (!ffmpegP.ffmpeg || !ffmpegP.ffprobe) return
Ffmpeg()

View File

@ -1,4 +1,5 @@
import fs from 'fs'
import { finished } from 'stream'
import { server } from '../api'
export async function concatenateTSFiles(inputFiles: Array<string>, outputFile: string) {
@ -16,14 +17,23 @@ export async function concatenateTSFiles(inputFiles: Array<string>, outputFile:
reject(error)
})
writeStream.on('finish', () => {
console.log('TS files concatenated successfully!')
resolve()
})
const processNextFile = (index: number) => {
if (index >= inputFiles.length) {
writeStream.end()
finished(writeStream, (err) => {
if (err) {
server.logger.log({
level: 'error',
message: `Error while finishing write stream`,
error: err,
timestamp: new Date().toISOString(),
section: 'crunchyrollDownloadProcessConcatenate'
})
return reject(err)
}
console.log('TS files concatenated successfully!')
resolve()
})
return
}