Merge pull request #322 from frost768/thumbnails

Thumbnails

src/hooks/useProgressBar.ts

@@ -1,10 +1,8 @@
 import React, { RefObject, useCallback, useEffect, useState } from "react";
 
-type ActivityEvent =
-  | React.MouseEvent<HTMLElement>
-  | React.TouchEvent<HTMLElement>
-  | MouseEvent
-  | TouchEvent;
+export type MouseActivity = React.MouseEvent<HTMLElement> | MouseEvent;
+
+type ActivityEvent = MouseActivity | React.TouchEvent<HTMLElement> | TouchEvent;
 
 export function makePercentageString(num: number) {
   return `${num.toFixed(2)}%`;

src/setup/index.css

@@ -34,6 +34,10 @@ body[data-no-select] {
   animation: roll 1s;
 }
 
+.roll-infinite {
+  animation: roll 2s infinite;
+}
+
 @keyframes roll {
   from {
     transform: rotate(0deg);

src/setup/sentry.tsx

@@ -4,6 +4,7 @@ import * as Sentry from "@sentry/react";
 import { conf } from "@/setup/config";
 import { SENTRY_DSN } from "@/setup/constants";
 
+if (process.env.NODE_ENV !== "development")
   Sentry.init({
     dsn: SENTRY_DSN,
     release: `movie-web@${conf().APP_VERSION}`,

src/utils/formatSeconds.ts

@@ -0,0 +1,21 @@
+export function formatSeconds(secs: number, showHours = false): string {
+  if (Number.isNaN(secs)) {
+    if (showHours) return "0:00:00";
+    return "0:00";
+  }
+
+  let time = secs;
+  const seconds = Math.floor(time % 60);
+
+  time /= 60;
+  const minutes = Math.floor(time % 60);
+
+  time /= 60;
+  const hours = Math.floor(time);
+
+  const paddedSecs = seconds.toString().padStart(2, "0");
+  const paddedMins = minutes.toString().padStart(2, "0");
+
+  if (!showHours) return [paddedMins, paddedSecs].join(":");
+  return [hours, paddedMins, paddedSecs].join(":");
+}

src/video/components/VideoPlayerBase.tsx

@@ -7,6 +7,7 @@ import { useInterface } from "@/video/state/logic/interface";
 import { useMeta } from "@/video/state/logic/meta";
 
 import { MetaAction } from "./actions/MetaAction";
+import ThumbnailGeneratorInternal from "./internal/ThumbnailGeneratorInternal";
 import { VideoElementInternal } from "./internal/VideoElementInternal";
 import {
   VideoPlayerContextProvider,
@@ -48,6 +49,7 @@ function VideoPlayerBaseWithState(props: VideoPlayerBaseProps) {
         ].join(" ")}
       >
         <MetaAction />
+        <ThumbnailGeneratorInternal />
         <VideoElementInternal autoPlay={props.autoPlay} />
         <CastingInternal />
         <WrapperRegisterInternal wrapper={ref.current} />

src/video/components/actions/BackdropAction.tsx

@@ -22,7 +22,10 @@ export function BackdropAction(props: BackdropActionProps) {
 
   const lastTouchEnd = useRef<number>(0);
 
-  const handleMouseMove = useCallback(() => {
+  const handleMouseMove = useCallback(
+    (e) => {
+      // to enable thumbnail on mouse hover
+      e.stopPropagation();
       if (!moved) {
         setTimeout(() => {
           // If NOT a touch, set moved to true
@@ -37,7 +40,9 @@ export function BackdropAction(props: BackdropActionProps) {
         setMoved(false);
         timeout.current = null;
       }, 3000);
-  }, [setMoved, moved]);
+    },
+    [setMoved, moved]
+  );
 
   const handleMouseLeave = useCallback(() => {
     setMoved(false);

src/video/components/actions/ProgressAction.tsx

@@ -1,6 +1,7 @@
-import { useCallback, useEffect, useRef } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 
 import {
+  MouseActivity,
   makePercentage,
   makePercentageString,
   useProgressBar,
@@ -10,6 +11,8 @@ import { useVideoPlayerDescriptor } from "@/video/state/hooks";
 import { useControls } from "@/video/state/logic/controls";
 import { useProgress } from "@/video/state/logic/progress";
 
+import ThumbnailAction from "./ThumbnailAction";
+
 export function ProgressAction() {
   const descriptor = useVideoPlayerDescriptor();
   const controls = useControls(descriptor);
@@ -17,7 +20,15 @@ export function ProgressAction() {
   const ref = useRef<HTMLDivElement>(null);
   const dragRef = useRef<boolean>(false);
   const controlRef = useRef<typeof controls>(controls);
-
+  const [hoverPosition, setHoverPosition] = useState<number>(0);
+  const [isThumbnailVisible, setIsThumbnailVisible] = useState<boolean>(false);
+  const onMouseOver = useCallback((e: MouseActivity) => {
+    setHoverPosition(e.clientX);
+    setIsThumbnailVisible(true);
+  }, []);
+  const onMouseLeave = useCallback(() => {
+    setIsThumbnailVisible(false);
+  }, []);
   useEffect(() => {
     controlRef.current = controls;
   }, [controls]);
@@ -59,12 +70,16 @@ export function ProgressAction() {
   );
 
   return (
-    <div className="group pointer-events-auto w-full cursor-pointer rounded-full px-2">
     <div
       ref={ref}
+      className="group pointer-events-auto w-full cursor-pointer rounded-full px-2"
+    >
+      <div
         className="-my-3 flex h-8 items-center"
         onMouseDown={dragMouseDown}
         onTouchStart={dragMouseDown}
+        onMouseMove={onMouseOver}
+        onMouseLeave={onMouseLeave}
       >
         <div
           className={`relative h-1 flex-1 rounded-full bg-gray-500 bg-opacity-50 transition-[height] duration-100 group-hover:h-2 ${
@@ -91,6 +106,13 @@ export function ProgressAction() {
           </div>
         </div>
       </div>
+      {isThumbnailVisible ? (
+        <ThumbnailAction
+          parentRef={ref}
+          videoTime={videoTime}
+          hoverPosition={hoverPosition}
+        />
+      ) : null}
     </div>
   );
 }

src/video/components/actions/ThumbnailAction.tsx

@@ -0,0 +1,121 @@
+import { RefObject, useMemo } from "react";
+
+import { Icon, Icons } from "@/components/Icon";
+import { formatSeconds } from "@/utils/formatSeconds";
+import { useVideoPlayerDescriptor } from "@/video/state/hooks";
+import { VideoProgressEvent } from "@/video/state/logic/progress";
+import { useSource } from "@/video/state/logic/source";
+
+const THUMBNAIL_HEIGHT = 100;
+function position(
+  rectLeft: number,
+  rectWidth: number,
+  thumbnailWidth: number,
+  hoverPos: number
+): number {
+  const relativePosition = hoverPos - rectLeft;
+  if (relativePosition <= thumbnailWidth / 2) {
+    return rectLeft;
+  }
+  if (relativePosition >= rectWidth - thumbnailWidth / 2) {
+    return rectWidth + rectLeft - thumbnailWidth;
+  }
+  return relativePosition + rectLeft - thumbnailWidth / 2;
+}
+function useThumbnailWidth() {
+  const videoEl = useMemo(() => document.getElementsByTagName("video")[0], []);
+  const aspectRatio = videoEl.videoWidth / videoEl.videoHeight;
+  return THUMBNAIL_HEIGHT * aspectRatio;
+}
+
+function LoadingThumbnail({ pos }: { pos: number }) {
+  const videoEl = useMemo(() => document.getElementsByTagName("video")[0], []);
+  const aspectRatio = videoEl.videoWidth / videoEl.videoHeight;
+  const thumbnailWidth = THUMBNAIL_HEIGHT * aspectRatio;
+  return (
+    <div
+      className="absolute bottom-32 flex items-center justify-center rounded bg-black"
+      style={{
+        left: `${pos}px`,
+        width: `${thumbnailWidth}px`,
+        height: `${THUMBNAIL_HEIGHT}px`,
+      }}
+    >
+      <Icon
+        className="roll-infinite text-6xl text-bink-600"
+        icon={Icons.MOVIE_WEB}
+      />
+    </div>
+  );
+}
+
+function ThumbnailTime({ hoverTime, pos }: { hoverTime: number; pos: number }) {
+  const videoEl = useMemo(() => document.getElementsByTagName("video")[0], []);
+  const thumbnailWidth = useThumbnailWidth();
+  return (
+    <div
+      className="absolute bottom-24 text-white"
+      style={{
+        left: `${pos + thumbnailWidth / 2 - 18}px`,
+      }}
+    >
+      {formatSeconds(hoverTime, videoEl.duration > 60 * 60)}
+    </div>
+  );
+}
+
+function ThumbnailImage({ src, pos }: { src: string; pos: number }) {
+  const thumbnailWidth = useThumbnailWidth();
+  return (
+    <img
+      height={THUMBNAIL_HEIGHT}
+      width={thumbnailWidth}
+      className="absolute bottom-32 rounded"
+      src={src}
+      style={{
+        left: `${pos}px`,
+      }}
+    />
+  );
+}
+export default function ThumbnailAction({
+  parentRef,
+  hoverPosition,
+  videoTime,
+}: {
+  parentRef: RefObject<HTMLDivElement>;
+  hoverPosition: number;
+  videoTime: VideoProgressEvent;
+}) {
+  const descriptor = useVideoPlayerDescriptor();
+  const source = useSource(descriptor);
+  const thumbnailWidth = useThumbnailWidth();
+  if (!parentRef.current) return null;
+  const rect = parentRef.current.getBoundingClientRect();
+  if (!rect.width) return null;
+
+  const hoverPercent = (hoverPosition - rect.left) / rect.width;
+  const hoverTime = videoTime.duration * hoverPercent;
+  const src = source.source?.thumbnails.find(
+    (x) => x.from < hoverTime && x.to > hoverTime
+  )?.imgUrl;
+  if (!source.source?.thumbnails.length) return null;
+  return (
+    <div className="pointer-events-none">
+      {!src ? (
+        <LoadingThumbnail
+          pos={position(rect.left, rect.width, thumbnailWidth, hoverPosition)}
+        />
+      ) : (
+        <ThumbnailImage
+          pos={position(rect.left, rect.width, thumbnailWidth, hoverPosition)}
+          src={src}
+        />
+      )}
+      <ThumbnailTime
+        hoverTime={hoverTime}
+        pos={position(rect.left, rect.width, thumbnailWidth, hoverPosition)}
+      />
+    </div>
+  );
+}

src/video/components/actions/TimeAction.tsx

@@ -1,6 +1,7 @@
 import { useTranslation } from "react-i18next";
 
 import { useIsMobile } from "@/hooks/useIsMobile";
+import { formatSeconds } from "@/utils/formatSeconds";
 import { useVideoPlayerDescriptor } from "@/video/state/hooks";
 import { useControls } from "@/video/state/logic/controls";
 import { useInterface } from "@/video/state/logic/interface";
@@ -12,28 +13,6 @@ function durationExceedsHour(secs: number): boolean {
   return secs > 60 * 60;
 }
 
-function formatSeconds(secs: number, showHours = false): string {
-  if (Number.isNaN(secs)) {
-    if (showHours) return "0:00:00";
-    return "0:00";
-  }
-
-  let time = secs;
-  const seconds = Math.floor(time % 60);
-
-  time /= 60;
-  const minutes = Math.floor(time % 60);
-
-  time /= 60;
-  const hours = Math.floor(time);
-
-  const paddedSecs = seconds.toString().padStart(2, "0");
-  const paddedMins = minutes.toString().padStart(2, "0");
-
-  if (!showHours) return [paddedMins, paddedSecs].join(":");
-  return [hours, paddedMins, paddedSecs].join(":");
-}
-
 interface Props {
   className?: string;
   noDuration?: boolean;

src/video/components/internal/ThumbnailGeneratorInternal.tsx

@@ -0,0 +1,110 @@
+import Hls from "hls.js";
+import { RefObject, useCallback, useEffect, useRef, useState } from "react";
+
+import { MWStreamType } from "@/backend/helpers/streams";
+import { getPlayerState } from "@/video/state/cache";
+import { useVideoPlayerDescriptor } from "@/video/state/hooks";
+import { updateSource, useSource } from "@/video/state/logic/source";
+import { Thumbnail } from "@/video/state/types";
+
+async function* generate(
+  videoRef: RefObject<HTMLVideoElement>,
+  canvasRef: RefObject<HTMLCanvasElement>,
+  index = 0,
+  numThumbnails = 20
+): AsyncGenerator<Thumbnail, Thumbnail> {
+  const video = videoRef.current;
+  const canvas = canvasRef.current;
+  if (!video) return { from: -1, to: -1, imgUrl: "" };
+  if (!canvas) return { from: -1, to: -1, imgUrl: "" };
+  await new Promise((resolve, reject) => {
+    video.addEventListener("loadedmetadata", resolve);
+    video.addEventListener("error", reject);
+  });
+
+  canvas.height = video.videoHeight;
+  canvas.width = video.videoWidth;
+  const ctx = canvas.getContext("2d");
+  if (!ctx) return { from: -1, to: -1, imgUrl: "" };
+  let i = index;
+  const limit = numThumbnails - 1;
+  const step = video.duration / limit;
+  while (i < limit && !Number.isNaN(video.duration)) {
+    const from = i * step;
+    const to = (i + 1) * step;
+    video.currentTime = from;
+    await new Promise((resolve) => {
+      video.addEventListener("seeked", resolve);
+    });
+
+    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+    const imgUrl = canvas.toDataURL();
+    i += 1;
+    yield {
+      from,
+      to,
+      imgUrl,
+    };
+  }
+
+  return { from: -1, to: -1, imgUrl: "" };
+}
+
+export default function ThumbnailGeneratorInternal() {
+  const descriptor = useVideoPlayerDescriptor();
+  const source = useSource(descriptor);
+
+  const videoRef = useRef<HTMLVideoElement>(document.createElement("video"));
+  const canvasRef = useRef<HTMLCanvasElement>(document.createElement("canvas"));
+  const hlsRef = useRef<Hls>(new Hls());
+  const thumbnails = useRef<Thumbnail[]>([]);
+  const abortController = useRef<AbortController>(new AbortController());
+
+  const generator = useCallback(
+    async (videoUrl: string, streamType: MWStreamType) => {
+      const prevIndex = thumbnails.current.length;
+      const video = videoRef.current;
+      if (streamType === MWStreamType.HLS) {
+        hlsRef.current.attachMedia(video);
+        hlsRef.current.loadSource(videoUrl);
+      } else {
+        video.crossOrigin = "anonymous";
+        video.src = videoUrl;
+      }
+
+      for await (const thumbnail of generate(videoRef, canvasRef, prevIndex)) {
+        if (abortController.current.signal.aborted) {
+          if (streamType === MWStreamType.HLS) hlsRef.current.detachMedia();
+          abortController.current = new AbortController();
+          const state = getPlayerState(descriptor);
+          if (!state.source) return;
+          const { url, type } = state.source;
+          generator(url, type);
+          break;
+        }
+
+        if (thumbnail.from === -1) continue;
+        thumbnails.current = [...thumbnails.current, thumbnail];
+        const state = getPlayerState(descriptor);
+        if (!state.source) return;
+        state.source.thumbnails = thumbnails.current;
+        updateSource(descriptor, state);
+      }
+    },
+    [descriptor]
+  );
+
+  useEffect(() => {
+    const controller = abortController.current;
+    const state = getPlayerState(descriptor);
+    if (!state.source) return;
+    const { url, type } = state.source;
+    generator(url, type);
+    return () => {
+      if (!source.source?.url) return;
+      controller.abort();
+    };
+  }, [descriptor, generator, source.source?.url]);
+
+  return null;
+}

src/video/components/popouts/CaptionSettingsPopout.tsx

@@ -71,7 +71,7 @@ export function CaptionSettingsPopout(props: {
           </label>
           <div className="flex flex-row gap-2">
             {colors.map((color) => (
-              <CaptionColorSelector color={color} />
+              <CaptionColorSelector key={color} color={color} />
             ))}
           </div>
         </div>

src/video/state/logic/source.ts

@@ -4,7 +4,7 @@ import { MWStreamQuality, MWStreamType } from "@/backend/helpers/streams";
 
 import { getPlayerState } from "../cache";
 import { listenEvent, sendEvent, unlistenEvent } from "../events";
-import { VideoPlayerState } from "../types";
+import { Thumbnail, VideoPlayerState } from "../types";
 
 export type VideoSourceEvent = {
   source: null | {
@@ -17,6 +17,7 @@ export type VideoSourceEvent = {
       id: string;
       url: string;
     };
+    thumbnails: Thumbnail[];
   };
 };
 

src/video/state/providers/castingStateProvider.ts

@@ -154,6 +154,7 @@ export function createCastingStateProvider(
         caption: null,
         embedId: source.embedId,
         providerId: source.providerId,
+        thumbnails: [],
       };
       resetStateForSource(descriptor, state);
       updateSource(descriptor, state);

src/video/state/providers/videoStateProvider.ts

@@ -63,7 +63,6 @@ export function createVideoStateProvider(
 ): VideoPlayerStateProvider {
   const player = playerEl;
   const state = getPlayerState(descriptor);
-
   return {
     getId() {
       return "video";
@@ -147,6 +146,16 @@ export function createVideoStateProvider(
 
       // reset before assign new one so the old HLS instance gets destroyed
       resetStateForSource(descriptor, state);
+      // update state
+      state.source = {
+        quality: source.quality,
+        type: source.type,
+        url: source.source,
+        caption: null,
+        embedId: source.embedId,
+        providerId: source.providerId,
+        thumbnails: [],
+      };
 
       if (source?.type === MWStreamType.HLS) {
         if (player.canPlayType("application/vnd.apple.mpegurl")) {
@@ -185,15 +194,6 @@ export function createVideoStateProvider(
         player.src = source.source;
       }
 
-      // update state
-      state.source = {
-        quality: source.quality,
-        type: source.type,
-        url: source.source,
-        caption: null,
-        embedId: source.embedId,
-        providerId: source.providerId,
-      };
       updateSource(descriptor, state);
     },
     setCaption(id, url) {

src/video/state/types.ts

@@ -9,6 +9,11 @@ import { DetailedMeta } from "@/backend/metadata/getmeta";
 
 import { VideoPlayerStateProvider } from "./providers/providerTypes";
 
+export interface Thumbnail {
+  from: number;
+  to: number;
+  imgUrl: string;
+}
 export type VideoPlayerMeta = {
   meta: DetailedMeta;
   captions: MWCaption[];
@@ -75,6 +80,7 @@ export type VideoPlayerState = {
       url: string;
       id: string;
     };
+    thumbnails: Thumbnail[];
   };
 
   // casting state

src/views/SettingsModal.tsx

@@ -122,7 +122,7 @@ export default function SettingsModal(props: {
                   </label>
                   <div className="flex flex-row gap-2">
                     {colors.map((color) => (
-                      <CaptionColorSelector color={color} />
+                      <CaptionColorSelector key={color} color={color} />
                     ))}
                   </div>
                 </div>