From 307f555b708ee698011220a9db3c203e9b1d3db6 Mon Sep 17 00:00:00 2001 From: frost768 Date: Sun, 26 Mar 2023 10:33:24 +0300 Subject: [PATCH 1/4] better subtitle handling --- package.json | 3 +- src/@types/node_webtt.d.ts | 27 -------- src/backend/helpers/captions.ts | 65 ++++++++----------- src/backend/helpers/streams.ts | 2 +- src/setup/locales/en/translation.json | 2 +- src/setup/locales/fr/translation.json | 2 +- .../actions/CaptionRendererAction.tsx | 28 ++++---- .../popouts/CaptionSelectionPopout.tsx | 64 ++++++++---------- tsconfig.json | 1 - yarn.lock | 22 ++----- 10 files changed, 76 insertions(+), 140 deletions(-) delete mode 100644 src/@types/node_webtt.d.ts diff --git a/package.json b/package.json index 7aa795f7..75d5a021 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "json5": "^2.2.0", "lodash.throttle": "^4.1.1", "nanoid": "^4.0.0", - "node-webvtt": "^1.9.4", "ofetch": "^1.0.0", "pako": "^2.1.0", "react": "^17.0.2", @@ -31,7 +30,7 @@ "react-stickynode": "^4.1.0", "react-transition-group": "^4.4.5", "react-use": "^17.4.0", - "srt-webvtt": "^2.0.0", + "subsrt-ts": "^2.1.0", "unpacker": "^1.0.1" }, "scripts": { diff --git a/src/@types/node_webtt.d.ts b/src/@types/node_webtt.d.ts deleted file mode 100644 index 5662d89f..00000000 --- a/src/@types/node_webtt.d.ts +++ /dev/null @@ -1,27 +0,0 @@ -declare module "node-webvtt" { - interface Cue { - identifier: string; - start: number; - end: number; - text: string; - styles: string; - } - interface Options { - meta?: boolean; - strict?: boolean; - } - type ParserError = Error; - interface ParseResult { - valid: boolean; - strict: boolean; - cues: Cue[]; - errors: ParserError[]; - meta?: Map; - } - interface Segment { - duration: number; - cues: Cue[]; - } - function parse(text: string, options: Options): ParseResult; - function segment(input: string, segmentLength?: number): Segment[]; -} diff --git a/src/backend/helpers/captions.ts b/src/backend/helpers/captions.ts index 4bd11052..477f2a68 100644 --- a/src/backend/helpers/captions.ts +++ b/src/backend/helpers/captions.ts @@ -1,48 +1,26 @@ import { mwFetch, proxiedFetch } from "@/backend/helpers/fetch"; -import { MWCaption, MWCaptionType } from "@/backend/helpers/streams"; -import toWebVTT from "srt-webvtt"; +import { MWCaption } from "@/backend/helpers/streams"; import DOMPurify from "dompurify"; +import { list, parse, detect } from "subsrt-ts"; +import { ContentCaption } from "subsrt-ts/dist/types/handler"; +export const subtitleTypeList = list() + .map((v) => `.${v}`) + .join(","); export const sanitize = DOMPurify.sanitize; -export const CUSTOM_CAPTION_ID = "customCaption"; export async function getCaptionUrl(caption: MWCaption): Promise { - if (caption.type === MWCaptionType.SRT) { - let captionBlob: Blob; - - if (caption.needsProxy) { - captionBlob = await proxiedFetch(caption.url, { - responseType: "blob" as any, - }); - } else { - captionBlob = await mwFetch(caption.url, { - responseType: "blob" as any, - }); - } - - return toWebVTT(captionBlob); + if (caption.url.startsWith("blob:")) return caption.url; + let captionBlob: Blob; + if (caption.needsProxy) { + captionBlob = await proxiedFetch(caption.url, { + responseType: "blob" as any, + }); + } else { + captionBlob = await mwFetch(caption.url, { + responseType: "blob" as any, + }); } - - if (caption.type === MWCaptionType.VTT) { - if (caption.needsProxy) { - const blob = await proxiedFetch(caption.url, { - responseType: "blob" as any, - }); - return URL.createObjectURL(blob); - } - - return caption.url; - } - - throw new Error("invalid type"); -} - -export async function convertCustomCaptionFileToWebVTT(file: File) { - const header = await file.slice(0, 6).text(); - const isWebVTT = header === "WEBVTT"; - if (!isWebVTT) { - return toWebVTT(file); - } - return URL.createObjectURL(file); + return URL.createObjectURL(captionBlob); } export function revokeCaptionBlob(url: string | undefined) { @@ -50,3 +28,12 @@ export function revokeCaptionBlob(url: string | undefined) { URL.revokeObjectURL(url); } } + +export function parseSubtitles(text: string): ContentCaption[] { + if (detect(text) === "") { + throw new Error("Invalid subtitle format"); + } + return parse(text) + .filter((cue) => cue.type === "caption") + .map((cue) => cue as ContentCaption); +} diff --git a/src/backend/helpers/streams.ts b/src/backend/helpers/streams.ts index 27e9fddb..aca41fe2 100644 --- a/src/backend/helpers/streams.ts +++ b/src/backend/helpers/streams.ts @@ -20,7 +20,7 @@ export enum MWStreamQuality { export type MWCaption = { needsProxy?: boolean; url: string; - type: MWCaptionType; + type?: MWCaptionType; langIso: string; }; diff --git a/src/setup/locales/en/translation.json b/src/setup/locales/en/translation.json index d1408148..1838b63c 100644 --- a/src/setup/locales/en/translation.json +++ b/src/setup/locales/en/translation.json @@ -80,7 +80,7 @@ "noCaptions": "No captions", "linkedCaptions": "Linked captions", "customCaption": "Custom caption", - "uploadCustomCaption": "Upload caption (SRT, VTT)", + "uploadCustomCaption": "Upload caption", "noEmbeds": "No embeds were found for this source", "errors": { "loadingWentWong": "Something went wrong loading the episodes for {{seasonTitle}}", diff --git a/src/setup/locales/fr/translation.json b/src/setup/locales/fr/translation.json index de40a796..e5c669ce 100644 --- a/src/setup/locales/fr/translation.json +++ b/src/setup/locales/fr/translation.json @@ -62,7 +62,7 @@ "noCaptions": "Pas de sous-titres", "linkedCaptions": "Sous-titres liés", "customCaption": "Sous-titres personnalisés", - "uploadCustomCaption": "Télécharger des sous-titres (SRT, VTT)", + "uploadCustomCaption": "Télécharger des sous-titres", "noEmbeds": "Aucun contenu intégré n'a été trouvé pour cette source", "errors": { "loadingWentWong": "Un problème est survenu lors du chargement des épisodes pour {{seasonTitle}}", diff --git a/src/video/components/actions/CaptionRendererAction.tsx b/src/video/components/actions/CaptionRendererAction.tsx index 6abc18c9..ab7edba0 100644 --- a/src/video/components/actions/CaptionRendererAction.tsx +++ b/src/video/components/actions/CaptionRendererAction.tsx @@ -1,7 +1,7 @@ import { Transition } from "@/components/Transition"; import { useSettings } from "@/state/settings"; -import { sanitize } from "@/backend/helpers/captions"; -import { parse, Cue } from "node-webvtt"; +import { sanitize, parseSubtitles } from "@/backend/helpers/captions"; +import { ContentCaption } from "subsrt-ts/dist/types/handler"; import { useRef } from "react"; import { useAsync } from "react-use"; import { useVideoPlayerDescriptor } from "../../state/hooks"; @@ -48,16 +48,18 @@ export function CaptionRendererAction({ const source = useSource(descriptor).source; const videoTime = useProgress(descriptor).time; const { captionSettings } = useSettings(); - const captions = useRef([]); + const captions = useRef([]); useAsync(async () => { - const url = source?.caption?.url; - if (url) { - // Is there a better way? - const result = await fetch(url); - // Uses UTF-8 by default + const blobUrl = source?.caption?.url; + if (blobUrl) { + const result = await fetch(blobUrl); const text = await result.text(); - captions.current = parse(text, { strict: false }).cues; + try { + captions.current = parseSubtitles(text); + } catch (error) { + captions.current = []; + } } else { captions.current = []; } @@ -65,8 +67,8 @@ export function CaptionRendererAction({ if (!captions.current.length) return null; const isVisible = (start: number, end: number): boolean => { - const delayedStart = start + captionSettings.delay; - const delayedEnd = end + captionSettings.delay; + const delayedStart = start / 1000 + captionSettings.delay; + const delayedEnd = end / 1000 + captionSettings.delay; return ( Math.max(0, delayedStart) <= videoTime && Math.max(0, delayedEnd) >= videoTime @@ -82,9 +84,9 @@ export function CaptionRendererAction({ show > {captions.current.map( - ({ identifier, end, start, text }) => + ({ start, end, content }) => isVisible(start, end) && ( - + ) )} diff --git a/src/video/components/popouts/CaptionSelectionPopout.tsx b/src/video/components/popouts/CaptionSelectionPopout.tsx index 9caaca96..6f056415 100644 --- a/src/video/components/popouts/CaptionSelectionPopout.tsx +++ b/src/video/components/popouts/CaptionSelectionPopout.tsx @@ -1,7 +1,7 @@ import { getCaptionUrl, - convertCustomCaptionFileToWebVTT, - CUSTOM_CAPTION_ID, + parseSubtitles, + subtitleTypeList, } from "@/backend/helpers/captions"; import { MWCaption } from "@/backend/helpers/streams"; import { Icon, Icons } from "@/components/Icon"; @@ -13,10 +13,11 @@ import { useVideoPlayerDescriptor } from "@/video/state/hooks"; import { useControls } from "@/video/state/logic/controls"; import { useMeta } from "@/video/state/logic/meta"; import { useSource } from "@/video/state/logic/source"; -import { ChangeEvent, useMemo, useRef } from "react"; +import { useMemo, useRef } from "react"; import { useTranslation } from "react-i18next"; import { PopoutListEntry, PopoutSection } from "./PopoutUtils"; +const customCaption = "external-custom"; function makeCaptionId(caption: MWCaption, isLinked: boolean): string { return isLinked ? `linked-${caption.langIso}` : `external-${caption.langIso}`; } @@ -41,35 +42,17 @@ export function CaptionSelectionPopout(props: { async (caption: MWCaption, isLinked: boolean) => { const id = makeCaptionId(caption, isLinked); loadingId.current = id; - controls.setCaption(id, await getCaptionUrl(caption)); + const blobUrl = await getCaptionUrl(caption); + const result = await fetch(blobUrl); + const text = await result.text(); + parseSubtitles(text); // This will throw if the file is invalid + controls.setCaption(id, blobUrl); controls.closePopout(); } ); const currentCaption = source.source?.caption?.id; const customCaptionUploadElement = useRef(null); - const [setCustomCaption, loadingCustomCaption, errorCustomCaption] = - useLoading(async (captionFile: File) => { - if ( - !captionFile.name.endsWith(".srt") && - !captionFile.name.endsWith(".vtt") - ) { - throw new Error("Only SRT or VTT files are allowed"); - } - controls.setCaption( - CUSTOM_CAPTION_ID, - await convertCustomCaptionFileToWebVTT(captionFile) - ); - controls.closePopout(); - }); - - async function handleUploadCaption(e: ChangeEvent) { - if (!e.target.files) { - return; - } - const captionFile = e.target.files[0]; - setCustomCaption(captionFile); - } return ( { - customCaptionUploadElement.current?.click(); - }} + key={customCaption} + active={currentCaption === customCaption} + loading={loading && loadingId.current === customCaption} + errored={error && loadingId.current === customCaption} + onClick={() => customCaptionUploadElement.current?.click()} > - {currentCaption === CUSTOM_CAPTION_ID + {currentCaption === customCaption ? t("videoPlayer.popouts.customCaption") : t("videoPlayer.popouts.uploadCustomCaption")} { + if (!e.target.files) return; + const customSubtitle = { + langIso: "custom", + url: URL.createObjectURL(e.target.files[0]), + }; + setCaption(customSubtitle, false); + }} /> diff --git a/tsconfig.json b/tsconfig.json index a00c1a1f..e1004c43 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,7 +16,6 @@ "noEmit": true, "jsx": "react-jsx", "baseUrl": "./src", - "typeRoots": ["./src/@types"], "paths": { "@/*": ["./*"] }, diff --git a/yarn.lock b/yarn.lock index 1c29c291..48e68bc8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2123,11 +2123,6 @@ commander@^2.20.0: resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== -commander@^7.1.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" - integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== - commander@^8.0.0: version "8.3.0" resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66" @@ -3854,13 +3849,6 @@ node-releases@^2.0.8: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f" integrity sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w== -node-webvtt@^1.9.4: - version "1.9.4" - resolved "https://registry.yarnpkg.com/node-webvtt/-/node-webvtt-1.9.4.tgz#b71b98f879c6c88ebeda40c358bd45a882ca5d89" - integrity sha512-EjrJdKdxSyd8j4LMLW6s2Ah4yNoeVXp18Ob04CQl1In18xcUmKzEE8pcsxxnFVqanTyjbGYph2VnvtwIXR4EjA== - dependencies: - commander "^7.1.0" - normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" @@ -4699,11 +4687,6 @@ sourcemap-codec@^1.4.8: resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4" integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== -srt-webvtt@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/srt-webvtt/-/srt-webvtt-2.0.0.tgz#debd2f56dd2b6600894caa11bb78893e5fc6509b" - integrity sha512-G2Z7/Jf2NRKrmLYNSIhSYZZYE6OFlKXFp9Au2/zJBKgrioUzmrAys1x7GT01dwl6d2sEnqr5uahEIOd0JW/Rbw== - stack-generator@^2.0.5: version "2.0.10" resolved "https://registry.yarnpkg.com/stack-generator/-/stack-generator-2.0.10.tgz#8ae171e985ed62287d4f1ed55a1633b3fb53bb4d" @@ -4859,6 +4842,11 @@ subscribe-ui-event@^2.0.6: lodash "^4.17.15" raf "^3.0.0" +subsrt-ts@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/subsrt-ts/-/subsrt-ts-2.1.0.tgz#97b5e0f97800fb08b64465b53c7c4f14f43d6fd4" + integrity sha512-LOdp6A91l/yPLPFuEaYvGzFDusUz0J52ksZjaCFdl347DOhedZOVQEciTaH7KaVDRlb7wstOx4dPFdjf9AyuFw== + supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" From 495222eb105d8f768a1d044f85c499a9aa7bc825 Mon Sep 17 00:00:00 2001 From: frost768 Date: Sat, 1 Apr 2023 12:19:05 +0300 Subject: [PATCH 2/4] export subtitle types as a list --- src/backend/helpers/captions.ts | 12 +++++------- .../components/popouts/CaptionSelectionPopout.tsx | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/backend/helpers/captions.ts b/src/backend/helpers/captions.ts index 477f2a68..83edaa84 100644 --- a/src/backend/helpers/captions.ts +++ b/src/backend/helpers/captions.ts @@ -1,12 +1,10 @@ import { mwFetch, proxiedFetch } from "@/backend/helpers/fetch"; import { MWCaption } from "@/backend/helpers/streams"; import DOMPurify from "dompurify"; -import { list, parse, detect } from "subsrt-ts"; +import { parse, detect, list } from "subsrt-ts"; import { ContentCaption } from "subsrt-ts/dist/types/handler"; -export const subtitleTypeList = list() - .map((v) => `.${v}`) - .join(","); +export const subtitleTypeList = list().map((type) => `.${type}`); export const sanitize = DOMPurify.sanitize; export async function getCaptionUrl(caption: MWCaption): Promise { if (caption.url.startsWith("blob:")) return caption.url; @@ -33,7 +31,7 @@ export function parseSubtitles(text: string): ContentCaption[] { if (detect(text) === "") { throw new Error("Invalid subtitle format"); } - return parse(text) - .filter((cue) => cue.type === "caption") - .map((cue) => cue as ContentCaption); + return parse(text).filter( + (cue) => cue.type === "caption" + ) as ContentCaption[]; } diff --git a/src/video/components/popouts/CaptionSelectionPopout.tsx b/src/video/components/popouts/CaptionSelectionPopout.tsx index 6f056415..b9201024 100644 --- a/src/video/components/popouts/CaptionSelectionPopout.tsx +++ b/src/video/components/popouts/CaptionSelectionPopout.tsx @@ -100,7 +100,7 @@ export function CaptionSelectionPopout(props: { { if (!e.target.files) return; From 156b69346069e18a0b89ba8186497ce799e304eb Mon Sep 17 00:00:00 2001 From: frost768 Date: Sun, 2 Apr 2023 18:14:03 +0300 Subject: [PATCH 3/4] suggested changes --- src/backend/helpers/streams.ts | 3 ++- src/video/components/popouts/CaptionSelectionPopout.tsx | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend/helpers/streams.ts b/src/backend/helpers/streams.ts index aca41fe2..12cbc551 100644 --- a/src/backend/helpers/streams.ts +++ b/src/backend/helpers/streams.ts @@ -6,6 +6,7 @@ export enum MWStreamType { export enum MWCaptionType { VTT = "vtt", SRT = "srt", + UNKNOWN = "unknown", } export enum MWStreamQuality { @@ -20,7 +21,7 @@ export enum MWStreamQuality { export type MWCaption = { needsProxy?: boolean; url: string; - type?: MWCaptionType; + type: MWCaptionType; langIso: string; }; diff --git a/src/video/components/popouts/CaptionSelectionPopout.tsx b/src/video/components/popouts/CaptionSelectionPopout.tsx index b9201024..2ae9fced 100644 --- a/src/video/components/popouts/CaptionSelectionPopout.tsx +++ b/src/video/components/popouts/CaptionSelectionPopout.tsx @@ -3,7 +3,7 @@ import { parseSubtitles, subtitleTypeList, } from "@/backend/helpers/captions"; -import { MWCaption } from "@/backend/helpers/streams"; +import { MWCaption, MWCaptionType } from "@/backend/helpers/streams"; import { Icon, Icons } from "@/components/Icon"; import { FloatingCardView } from "@/components/popout/FloatingCard"; import { FloatingView } from "@/components/popout/FloatingView"; @@ -47,7 +47,10 @@ export function CaptionSelectionPopout(props: { const text = await result.text(); parseSubtitles(text); // This will throw if the file is invalid controls.setCaption(id, blobUrl); - controls.closePopout(); + // sometimes this doesn't work, so we add a small delay + setTimeout(() => { + controls.closePopout(); + }, 100); } ); @@ -107,6 +110,7 @@ export function CaptionSelectionPopout(props: { const customSubtitle = { langIso: "custom", url: URL.createObjectURL(e.target.files[0]), + type: MWCaptionType.UNKNOWN, }; setCaption(customSubtitle, false); }} From 661d995e3b9eb25576b27c5f93c67a784424fc8f Mon Sep 17 00:00:00 2001 From: frost768 Date: Sun, 2 Apr 2023 18:14:26 +0300 Subject: [PATCH 4/4] filter out non subtitle files --- src/backend/providers/superstream/index.ts | 26 ++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/backend/providers/superstream/index.ts b/src/backend/providers/superstream/index.ts index 8abed467..9ebe6262 100644 --- a/src/backend/providers/superstream/index.ts +++ b/src/backend/providers/superstream/index.ts @@ -225,15 +225,23 @@ registerProvider({ const subtitleRes = (await get(subtitleApiQuery)).data; - const mappedCaptions = subtitleRes.list.map((subtitle: any): MWCaption => { - return { - needsProxy: true, - langIso: subtitle.language, - url: subtitle.subtitles[0].file_path, - type: MWCaptionType.SRT, - }; - }); - + const mappedCaptions = subtitleRes.list.map( + (subtitle: any): MWCaption | null => { + const sub = subtitle; + sub.subtitles = subtitle.subtitles.filter((subFile: any) => { + const extension = subFile.file_path.substring( + sub.file_path.length - 3 + ); + return [MWCaptionType.SRT, MWCaptionType.VTT].includes(extension); + }); + return { + needsProxy: true, + langIso: subtitle.language, + url: sub.subtitles[0].file_path, + type: MWCaptionType.SRT, + }; + } + ); return { embeds: [], stream: {