Add embedded subtitles support

package.json

@@ -44,7 +44,7 @@
     "focus-trap-react": "^10.2.3",
     "fscreen": "^1.2.0",
     "fuse.js": "^7.0.0",
-    "hls.js": "^1.4.14",
+    "hls.js": "^1.5.7",
     "i18next": "^23.7.11",
     "immer": "^10.0.3",
     "jwt-decode": "^4.0.0",

pnpm-lock.yaml

@@ -67,8 +67,8 @@ dependencies:
     specifier: ^7.0.0
     version: 7.0.0
   hls.js:
-    specifier: ^1.4.14
+    specifier: ^1.5.7
-    version: 1.4.14
+    version: 1.5.7
   i18next:
     specifier: ^23.7.11
     version: 23.7.11
@@ -274,7 +274,7 @@ devDependencies:
     version: 0.5.9(prettier@3.1.1)
   rollup-plugin-visualizer:
     specifier: ^5.11.0
-    version: 5.11.0(@rollup/wasm-node@4.12.1)
+    version: 5.11.0(@rollup/wasm-node@4.13.0)
   tailwind-scrollbar:
     specifier: ^3.0.5
     version: 3.0.5(tailwindcss@3.4.0)
@@ -2068,7 +2068,7 @@ packages:
     engines: {node: '>=14.0.0'}
     dev: false
 
-  /@rollup/plugin-babel@5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.12.1):
+  /@rollup/plugin-babel@5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.13.0):
     resolution: {integrity: sha512-WFfdLWU/xVWKeRQnKmIAQULUI7Il0gZnBIH/ZFO069wYIfPu+8zrfp/KMW0atmELoRDq8FbiP3VCss9MhCut7Q==}
     engines: {node: '>= 10.0.0'}
     peerDependencies:
@@ -2081,36 +2081,36 @@ packages:
     dependencies:
       '@babel/core': 7.23.6
       '@babel/helper-module-imports': 7.22.15
-      '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.12.1)
+      '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.13.0)
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
     dev: true
 
-  /@rollup/plugin-node-resolve@11.2.1(@rollup/wasm-node@4.12.1):
+  /@rollup/plugin-node-resolve@11.2.1(@rollup/wasm-node@4.13.0):
     resolution: {integrity: sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg==}
     engines: {node: '>= 10.0.0'}
     peerDependencies:
       rollup: npm:@rollup/wasm-node
     dependencies:
-      '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.12.1)
+      '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.13.0)
       '@types/resolve': 1.17.1
       builtin-modules: 3.3.0
       deepmerge: 4.3.1
       is-module: 1.0.0
       resolve: 1.22.4
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
     dev: true
 
-  /@rollup/plugin-replace@2.4.2(@rollup/wasm-node@4.12.1):
+  /@rollup/plugin-replace@2.4.2(@rollup/wasm-node@4.13.0):
     resolution: {integrity: sha512-IGcu+cydlUMZ5En85jxHH4qj2hta/11BHq95iHEyb2sbgiN0eCdzvUcHw5gt9pBL5lTi4JDYJ1acCoMGpTvEZg==}
     peerDependencies:
       rollup: npm:@rollup/wasm-node
     dependencies:
-      '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.12.1)
+      '@rollup/pluginutils': 3.1.0(@rollup/wasm-node@4.13.0)
       magic-string: 0.25.9
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
     dev: true
 
-  /@rollup/pluginutils@3.1.0(@rollup/wasm-node@4.12.1):
+  /@rollup/pluginutils@3.1.0(@rollup/wasm-node@4.13.0):
     resolution: {integrity: sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg==}
     engines: {node: '>= 8.0.0'}
     peerDependencies:
@@ -2119,11 +2119,11 @@ packages:
       '@types/estree': 0.0.39
       estree-walker: 1.0.1
       picomatch: 2.3.1
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
     dev: true
 
-  /@rollup/wasm-node@4.12.1:
+  /@rollup/wasm-node@4.13.0:
-    resolution: {integrity: sha512-5j3BVQEccCzCb8fkl++IbDgAsnlsKBPz049C4C//j5s3pFKxKGlybl63QApdJKl1fNLr7HIwQEJcBImQtA3ZHg==}
+    resolution: {integrity: sha512-oFX11wzU7RTaiW06WBtRpzIVN/oaG0I3XkevNO0brBklYnY9zpLhTfksN4b+TdBt6CfXV/KdVhdWLbb0fQIR7A==}
     engines: {node: '>=18.0.0', npm: '>=8.0.0'}
     hasBin: true
     dependencies:
@@ -4388,8 +4388,8 @@ packages:
       function-bind: 1.1.2
     dev: true
 
-  /hls.js@1.4.14:
+  /hls.js@1.5.7:
-    resolution: {integrity: sha512-UppQjyvPVclg+6t2KY/Rv03h0+bA5u6zwqVoz4LAC/L0fgYmIaCD7ZCrwe8WI1Gv01be1XL0QFsRbSdIHV/Wbw==}
+    resolution: {integrity: sha512-Hnyf7ojTBtXHeOW1/t6wCBJSiK1WpoKF9yg7juxldDx8u3iswrkPt2wbOA/1NiwU4j27DSIVoIEJRAhcdMef/A==}
     dev: false
 
   /hoist-non-react-statics@3.3.2:
@@ -5112,7 +5112,7 @@ packages:
       '@babel/plugin-syntax-typescript': 7.23.3(@babel/core@7.23.6)
       '@babel/types': 7.23.6
       kleur: 4.1.5
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
       unplugin: 1.5.1
     transitivePeerDependencies:
       - supports-color
@@ -6040,7 +6040,7 @@ packages:
       glob: 7.2.3
     dev: true
 
-  /rollup-plugin-terser@7.0.2(@rollup/wasm-node@4.12.1):
+  /rollup-plugin-terser@7.0.2(@rollup/wasm-node@4.13.0):
     resolution: {integrity: sha512-w3iIaU4OxcF52UUXiZNsNeuXIMDvFrr+ZXK6bFZ0Q60qyVfq4uLptoS4bbq3paG3x216eQllFZX7zt6TIImguQ==}
     deprecated: This package has been deprecated and is no longer maintained. Please use @rollup/plugin-terser
     peerDependencies:
@@ -6048,12 +6048,12 @@ packages:
     dependencies:
       '@babel/code-frame': 7.23.5
       jest-worker: 26.6.2
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
       serialize-javascript: 4.0.0
       terser: 5.19.3
     dev: true
 
-  /rollup-plugin-visualizer@5.11.0(@rollup/wasm-node@4.12.1):
+  /rollup-plugin-visualizer@5.11.0(@rollup/wasm-node@4.13.0):
     resolution: {integrity: sha512-exM0Ms2SN3AgTzMeW7y46neZQcyLY7eKwWAop1ZoRTCZwyrIRdMMJ6JjToAJbML77X/9N8ZEpmXG4Z/Clb9k8g==}
     engines: {node: '>=14'}
     hasBin: true
@@ -6065,7 +6065,7 @@ packages:
     dependencies:
       open: 8.4.2
       picomatch: 2.3.1
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
       source-map: 0.7.4
       yargs: 17.7.2
     dev: true
@@ -7051,7 +7051,7 @@ packages:
       '@types/node': 20.10.5
       esbuild: 0.19.10
       postcss: 8.4.32
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
     optionalDependencies:
       fsevents: 2.3.3
     dev: true
@@ -7313,9 +7313,9 @@ packages:
       '@babel/core': 7.23.6
       '@babel/preset-env': 7.23.6(@babel/core@7.23.6)
       '@babel/runtime': 7.23.6
-      '@rollup/plugin-babel': 5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.12.1)
+      '@rollup/plugin-babel': 5.3.1(@babel/core@7.23.6)(@rollup/wasm-node@4.13.0)
-      '@rollup/plugin-node-resolve': 11.2.1(@rollup/wasm-node@4.12.1)
+      '@rollup/plugin-node-resolve': 11.2.1(@rollup/wasm-node@4.13.0)
-      '@rollup/plugin-replace': 2.4.2(@rollup/wasm-node@4.12.1)
+      '@rollup/plugin-replace': 2.4.2(@rollup/wasm-node@4.13.0)
       '@surma/rollup-plugin-off-main-thread': 2.2.3
       ajv: 8.12.0
       common-tags: 1.8.2
@@ -7324,8 +7324,8 @@ packages:
       glob: 7.2.3
       lodash: 4.17.21
       pretty-bytes: 5.6.0
-      rollup: /@rollup/wasm-node@4.12.1
+      rollup: /@rollup/wasm-node@4.13.0
-      rollup-plugin-terser: 7.0.2(@rollup/wasm-node@4.12.1)
+      rollup-plugin-terser: 7.0.2(@rollup/wasm-node@4.13.0)
       source-map: 0.8.0-beta.0
       stringify-object: 3.3.0
       strip-comments: 2.0.1
@@ -7370,6 +7370,7 @@ packages:
 
   /workbox-google-analytics@7.0.0:
     resolution: {integrity: sha512-MEYM1JTn/qiC3DbpvP2BVhyIH+dV/5BjHk756u9VbwuAhu0QHyKscTnisQuz21lfRpOwiS9z4XdqeVAKol0bzg==}
+    deprecated: It is not compatible with newer versions of GA starting with v4, as long as you are using GAv3 it should be ok, but the package is not longer being maintained
     dependencies:
       workbox-background-sync: 7.0.0
       workbox-core: 7.0.0

src/backend/helpers/subs.ts

@@ -51,3 +51,16 @@ export async function downloadCaption(
   downloadCache.set(caption.url, output, expirySeconds);
   return output;
 }
+
+/**
+ * Downloads the WebVTT content. No different than a simple
+ * get request with a cache.
+ */
+export async function downloadWebVTT(url: string): Promise<string> {
+  const cached = downloadCache.get(url);
+  if (cached) return cached;
+
+  // Q: should this use proxiedFetch or sendExtensionRequest?
+  const data = await fetch(url).then((v) => v.text());
+  return data;
+}

src/components/player/atoms/settings/CaptionsView.tsx

@@ -122,9 +122,16 @@ export function CaptionsView({ id }: { id: string }) {
   >(null);
   const { selectCaptionById, disable } = useCaptions();
   const captionList = usePlayerStore((s) => s.captionList);
+  const getHlsCaptionList = usePlayerStore((s) => s.display?.getCaptionList);
+
+  const captions = useMemo(
+    () =>
+      captionList.length !== 0 ? captionList : getHlsCaptionList?.() ?? [],
+    [captionList, getHlsCaptionList],
+  );
 
   const [searchQuery, setSearchQuery] = useState("");
-  const subtitleList = useSubtitleList(captionList, searchQuery);
+  const subtitleList = useSubtitleList(captions, searchQuery);
 
   const [downloadReq, startDownload] = useAsyncFn(
     async (captionId: string) => {

src/components/player/display/base.ts

@@ -67,6 +67,11 @@ export function makeVideoElementDisplayInterface(): DisplayInterface {
   let preferenceQuality: SourceQuality | null = null;
   let lastVolume = 1;
 
+  const languagePromises = new Map<
+    string,
+    (value: void | PromiseLike<void>) => void
+  >();
+
   function reportLevels() {
     if (!hls) return;
     const levels = hls.levels;
@@ -133,6 +138,7 @@ export function makeVideoElementDisplayInterface(): DisplayInterface {
               },
             },
           },
+          renderTextTracksNatively: false,
         });
         hls.on(Hls.Events.ERROR, (event, data) => {
           console.error("HLS error", data);
@@ -173,6 +179,17 @@ export function makeVideoElementDisplayInterface(): DisplayInterface {
           const quality = hlsLevelToQuality(hls.levels[hls.currentLevel]);
           emit("changedquality", quality);
         });
+        hls.on(Hls.Events.SUBTITLE_TRACK_LOADED, () => {
+          for (const [lang, resolve] of languagePromises) {
+            const track = hls?.subtitleTracks.find((t) => t.lang === lang);
+            if (track) {
+              resolve();
+              languagePromises.delete(lang);
+              break;
+            }
+          }
+          console.log("Subtitle tracks loaded", hls?.subtitleTracks);
+        });
       }
 
       hls.attachMedia(vid);
@@ -413,5 +430,40 @@ export function makeVideoElementDisplayInterface(): DisplayInterface {
     setPlaybackRate(rate) {
       if (videoElement) videoElement.playbackRate = rate;
     },
+    getCaptionList() {
+      return (
+        hls?.subtitleTracks.map((track) => {
+          return {
+            id: track.id.toString(),
+            language: track.lang ?? "unknown",
+            url: track.url,
+            needsProxy: false,
+            hls: true,
+          };
+        }) ?? []
+      );
+    },
+    getSubtitleTracks() {
+      return hls?.subtitleTracks ?? [];
+    },
+    async setSubtitlePreference(lang) {
+      // default subtitles are already loaded by hls.js
+      const track = hls?.subtitleTracks.find((t) => t.lang === lang);
+      if (track?.details !== undefined) return Promise.resolve();
+
+      // need to wait a moment before hls loads the subtitles
+      const promise = new Promise<void>((resolve, reject) => {
+        languagePromises.set(lang, resolve);
+
+        // reject after some time, if hls.js fails to load the subtitles
+        // for any reason
+        setTimeout(() => {
+          reject();
+          languagePromises.delete(lang);
+        }, 5000);
+      });
+      hls?.setSubtitleOption({ lang });
+      return promise;
+    },
   };
 }

src/components/player/display/chromecast.ts

@@ -274,5 +274,14 @@ export function makeChromecastDisplayInterface(
       playbackRate = rate;
       setSource();
     },
+    getCaptionList() {
+      return [];
+    },
+    getSubtitleTracks() {
+      return [];
+    },
+    async setSubtitlePreference() {
+      return Promise.resolve();
+    },
   };
 }

src/components/player/display/displayInterface.ts

@@ -1,4 +1,7 @@
+import { MediaPlaylist } from "hls.js";
+
 import { MWMediaType } from "@/backend/metadata/types/mw";
+import { CaptionListItem } from "@/stores/player/slices/source";
 import { LoadableSource, SourceQuality } from "@/stores/player/utils/qualities";
 import { Listener } from "@/utils/events";
 
@@ -70,4 +73,7 @@ export interface DisplayInterface extends Listener<DisplayInterfaceEvents> {
   setMeta(meta: DisplayMeta): void;
   setCaption(caption: DisplayCaption | null): void;
   getType(): DisplayType;
+  getCaptionList(): CaptionListItem[];
+  getSubtitleTracks(): MediaPlaylist[];
+  setSubtitlePreference(lang: string): Promise<void>;
 }

src/components/player/hooks/useCaptions.ts

@@ -1,6 +1,8 @@
-import { useCallback } from "react";
+import { useCallback, useMemo } from "react";
+import subsrt from "subsrt-ts";
+import { ContentCaption } from "subsrt-ts/dist/types/handler";
 
-import { downloadCaption } from "@/backend/helpers/subs";
+import { downloadCaption, downloadWebVTT } from "@/backend/helpers/subs";
 import { usePlayerStore } from "@/stores/player/store";
 import { useSubtitleStore } from "@/stores/subtitles";
 
@@ -12,12 +14,26 @@ export function useCaptions() {
   );
   const setCaption = usePlayerStore((s) => s.setCaption);
   const lastSelectedLanguage = useSubtitleStore((s) => s.lastSelectedLanguage);
+
   const captionList = usePlayerStore((s) => s.captionList);
+  const getHlsCaptionList = usePlayerStore((s) => s.display?.getCaptionList);
+
+  const getSubtitleTracks = usePlayerStore((s) => s.display?.getSubtitleTracks);
+  const setSubtitlePreference = usePlayerStore(
+    (s) => s.display?.setSubtitlePreference,
+  );
+
+  const captions = useMemo(
+    () =>
+      captionList.length !== 0 ? captionList : getHlsCaptionList?.() ?? [],
+    [captionList, getHlsCaptionList],
+  );
 
   const selectCaptionById = useCallback(
     async (captionId: string) => {
-      const caption = captionList.find((v) => v.id === captionId);
+      const caption = captions.find((v) => v.id === captionId);
       if (!caption) return;
+      if (!caption.hls) {
         const srtData = await downloadCaption(caption);
         setCaption({
           id: caption.id,
@@ -27,17 +43,77 @@ export function useCaptions() {
         });
         resetSubtitleSpecificSettings();
         setLanguage(caption.language);
+      } else {
+        // request a language change to hls, so it can load the subtitles
+        await setSubtitlePreference?.(caption.language);
+        const track = getSubtitleTracks?.().find(
+          (t) => t.id.toString() === caption.id && t.details !== undefined,
+        );
+        if (!track) return;
+
+        const fragments =
+          track.details?.fragments?.filter(
+            (frag) => frag !== null && frag.url !== null,
+          ) ?? [];
+
+        const vttCaptions = (
+          await Promise.all(
+            fragments.map(async (frag) => {
+              const vtt = await downloadWebVTT(frag.url);
+              const parsed = subsrt.parse(vtt);
+              return parsed.filter(
+                (c) => c.type === "caption",
+              ) as ContentCaption[];
+            }),
+          )
+        ).flat();
+
+        // for some reason, in some cases there will be captions
+        // with the same start/end times, the same text duplicated
+        const filtered = vttCaptions.reduce(
+          (acc: ContentCaption[], cap: ContentCaption) => {
+            const lastCap = acc[acc.length - 1];
+            const isSameAsLast =
+              lastCap?.start === cap.start &&
+              lastCap?.end === cap.end &&
+              lastCap?.content === cap.content;
+            if (lastCap === undefined || !isSameAsLast) {
+              acc.push(cap);
+            }
+            return acc;
           },
-    [setLanguage, captionList, setCaption, resetSubtitleSpecificSettings],
+          [],
+        );
+
+        const srtData = subsrt.build(filtered, { format: "srt" });
+
+        setCaption({
+          id: caption.id,
+          language: caption.language,
+          srtData,
+          url: caption.url,
+        });
+        resetSubtitleSpecificSettings();
+        setLanguage(caption.language);
+      }
+    },
+    [
+      setLanguage,
+      captions,
+      setCaption,
+      resetSubtitleSpecificSettings,
+      getSubtitleTracks,
+      setSubtitlePreference,
+    ],
   );
 
   const selectLanguage = useCallback(
     async (language: string) => {
-      const caption = captionList.find((v) => v.language === language);
+      const caption = captions.find((v) => v.language === language);
       if (!caption) return;
       return selectCaptionById(caption.id);
     },
-    [captionList, selectCaptionById],
+    [captions, selectCaptionById],
   );
 
   const disable = useCallback(async () => {

src/stores/player/slices/source.ts

@@ -53,6 +53,7 @@ export interface CaptionListItem {
   language: string;
   url: string;
   needsProxy: boolean;
+  hls?: boolean;
 }
 
 export interface SourceSlice {