thumbnail scraping

This commit is contained in:
mrjvs 2023-10-21 04:50:14 +02:00
parent 6395d75d78
commit 32f031ab23
6 changed files with 336 additions and 34 deletions

View File

@ -1,8 +1,48 @@
import { useCallback, useEffect, useRef } from "react";
import {
MouseEvent,
RefObject,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { useProgressBar } from "@/hooks/useProgressBar";
import { nearestImageAt } from "@/stores/player/slices/thumbnails";
import { usePlayerStore } from "@/stores/player/store";
function ThumbnailDisplay(props: { at: number }) {
const thumbnailImages = usePlayerStore((s) => s.thumbnails.images);
const currentThumbnail = useMemo(() => {
return nearestImageAt(thumbnailImages, props.at)?.image;
}, [thumbnailImages, props.at]);
if (!currentThumbnail) return null;
return <img src={currentThumbnail.data} className="h-12" />;
}
function useMouseHoverPosition(barRef: RefObject<HTMLDivElement>) {
const [mousePos, setMousePos] = useState(-1);
const mouseMove = useCallback(
(e: MouseEvent<HTMLDivElement>) => {
const bar = barRef.current;
if (!bar) return;
const rect = barRef.current.getBoundingClientRect();
const pos = (e.pageX - rect.left) / barRef.current.offsetWidth;
setMousePos(pos * 100);
},
[setMousePos, barRef]
);
const mouseLeave = useCallback(() => {
setMousePos(-1);
}, [setMousePos]);
return { mousePos, mouseMove, mouseLeave };
}
export function ProgressBar() {
const { duration, time, buffered } = usePlayerStore((s) => s.progress);
const display = usePlayerStore((s) => s.display);
@ -18,6 +58,7 @@ export function ProgressBar() {
);
const ref = useRef<HTMLDivElement>(null);
const { mouseMove, mouseLeave, mousePos } = useMouseHoverPosition(ref);
const { dragging, dragPercentage, dragMouseDown } = useProgressBar(
ref,
@ -31,12 +72,30 @@ export function ProgressBar() {
setDraggingTime((dragPercentage / 100) * duration);
}, [setDraggingTime, duration, dragPercentage]);
const mousePosition = Math.floor(dragPercentage * duration);
return (
<div className="w-full relative">
<div className="top-0 absolute inset-x-0">
{mousePos > -1 ? (
<div
className="absolute bottom-0"
style={{
left: `${mousePos}%`,
}}
>
<ThumbnailDisplay at={mousePosition} />
</div>
) : null}
</div>
<div className="w-full" ref={ref}>
<div
className="group w-full h-8 flex items-center cursor-pointer"
onMouseDown={dragMouseDown}
onTouchStart={dragMouseDown}
onMouseLeave={mouseLeave}
onMouseMove={mouseMove}
>
<div
className={[
@ -59,7 +118,10 @@ export function ProgressBar() {
width: `${
Math.max(
0,
Math.min(1, dragging ? dragPercentage / 100 : time / duration)
Math.min(
1,
dragging ? dragPercentage / 100 : time / duration
)
) * 100
}%`,
}}
@ -74,5 +136,6 @@ export function ProgressBar() {
</div>
</div>
</div>
</div>
);
}

View File

@ -5,6 +5,7 @@ import { CastingInternal } from "@/components/player/internals/CastingInternal";
import { HeadUpdater } from "@/components/player/internals/HeadUpdater";
import { KeyboardEvents } from "@/components/player/internals/KeyboardEvents";
import { ProgressSaver } from "@/components/player/internals/ProgressSaver";
import { ThumbnailScraper } from "@/components/player/internals/ThumbnailScraper";
import { VideoClickTarget } from "@/components/player/internals/VideoClickTarget";
import { VideoContainer } from "@/components/player/internals/VideoContainer";
import { PlayerHoverState } from "@/stores/player/slices/interface";
@ -82,6 +83,7 @@ export function Container(props: PlayerProps) {
return (
<div className="relative">
<BaseContainer>
<ThumbnailScraper />
<CastingInternal />
<VideoContainer />
<ProgressSaver />

View File

@ -0,0 +1,127 @@
import Hls from "hls.js";
import { useEffect, useMemo, useRef } from "react";
import { ThumbnailImage } from "@/stores/player/slices/thumbnails";
import { usePlayerStore } from "@/stores/player/store";
import { LoadableSource, selectQuality } from "@/stores/player/utils/qualities";
class ThumnbnailWorker {
interrupted: boolean;
videoEl: HTMLVideoElement | null = null;
canvasEl: HTMLCanvasElement | null = null;
hls: Hls | null = null;
cb: (img: ThumbnailImage) => void;
constructor(ops: { addImage: (img: ThumbnailImage) => void }) {
this.cb = ops.addImage;
this.interrupted = false;
}
start(source: LoadableSource) {
const el = document.createElement("video");
const canvas = document.createElement("canvas");
this.hls = new Hls();
if (source.type === "mp4") {
el.src = source.url;
el.crossOrigin = "anonymous";
} else if (source.type === "hls") {
this.hls.attachMedia(el);
this.hls.loadSource(source.url);
} else throw new Error("Invalid loadable source type");
this.videoEl = el;
this.canvasEl = canvas;
this.begin().catch((err) => console.error(err));
}
destroy() {
this.hls?.detachMedia();
this.hls?.destroy();
this.hls = null;
this.interrupted = true;
this.videoEl = null;
this.canvasEl = null;
}
private async initVideo() {
if (!this.videoEl || !this.canvasEl) return;
await new Promise((resolve, reject) => {
this.videoEl?.addEventListener("loadedmetadata", resolve);
this.videoEl?.addEventListener("error", reject);
});
if (!this.videoEl || !this.canvasEl) return;
this.canvasEl.height = this.videoEl.videoHeight;
this.canvasEl.width = this.videoEl.videoWidth;
}
private async takeSnapshot(at: number) {
if (!this.videoEl || !this.canvasEl) return;
this.videoEl.currentTime = at;
await new Promise((resolve) => {
this.videoEl?.addEventListener("seeked", resolve);
});
if (!this.videoEl || !this.canvasEl) return;
const ctx = this.canvasEl.getContext("2d");
if (!ctx) return;
ctx.drawImage(
this.videoEl,
0,
0,
this.canvasEl.width,
this.canvasEl.height
);
const imgUrl = this.canvasEl.toDataURL();
this.cb({
at,
data: imgUrl,
});
}
private async begin() {
const vid = this.videoEl;
if (!vid) return;
await this.initVideo();
if (this.interrupted) return;
await this.takeSnapshot(vid.duration / 2);
}
}
export function ThumbnailScraper() {
const addImage = usePlayerStore((s) => s.thumbnails.addImage);
const source = usePlayerStore((s) => s.source);
const workerRef = useRef<ThumnbnailWorker | null>(null);
const inputStream = useMemo(() => {
if (!source) return null;
return selectQuality(source, {
automaticQuality: false,
lastChosenQuality: "360",
});
}, [source]);
// TODO stop worker on meta change
// start worker with the stream
useEffect(() => {
// dont interrupt existing working
if (workerRef.current) return;
if (!inputStream) return;
const ins = new ThumnbnailWorker({
addImage,
});
workerRef.current = ins;
ins.start(inputStream.stream);
}, [inputStream, addImage]);
// destroy worker on unmount
useEffect(() => {
return () => {
if (workerRef.current) workerRef.current.destroy();
};
}, []);
return null;
}

View File

@ -0,0 +1,106 @@
import { MakeSlice } from "@/stores/player/slices/types";
export interface ThumbnailImage {
at: number;
data: string;
}
export interface ThumbnailSlice {
thumbnails: {
images: ThumbnailImage[];
addImage(img: ThumbnailImage): void;
};
}
export interface ThumbnailImagePosition {
index: number;
image: ThumbnailImage;
}
/**
* get nearest image at the timestamp provided
* @param images images, must be sorted
*/
export function nearestImageAt(
images: ThumbnailImage[],
at: number
): ThumbnailImagePosition | null {
// no images, early return
if (images.length === 0) return null;
const indexPastTimestamp = images.findIndex((v) => v.at < at);
// no image found past timestamp, so last image must be closest
if (indexPastTimestamp === -1)
return {
index: images.length - 1,
image: images[images.length - 1],
};
const imagePastTimestamp = images[indexPastTimestamp];
// if past timestamp is first image, just return that image
if (indexPastTimestamp === 0)
return {
index: indexPastTimestamp,
image: imagePastTimestamp,
};
// distance before distance past
// | |
// [before] --------------------- [at] --------------------- [past]
const imageBeforeTimestamp = images[indexPastTimestamp - 1];
const distanceBefore = at - imageBeforeTimestamp.at;
const distancePast = imagePastTimestamp.at - at;
// if distance of before timestamp is smaller than the distance past
// before is closer, return that
// [before] --X-------------- [past]
if (distanceBefore < distancePast)
return {
index: indexPastTimestamp - 1,
image: imageBeforeTimestamp,
};
// must be closer to past here, return past
// [before] --------------X-- [past]
return {
index: indexPastTimestamp,
image: imagePastTimestamp,
};
}
export const createThumbnailSlice: MakeSlice<ThumbnailSlice> = (set, get) => ({
thumbnails: {
images: [],
addImage(img) {
const store = get();
const exactOrPastImageIndex = store.thumbnails.images.findIndex(
(v) => v.at <= img.at
);
// not found past or exact, so just append to the end
if (exactOrPastImageIndex === -1) {
set((s) => {
s.thumbnails.images.push(img);
});
return;
}
const exactOrPastImage = store.thumbnails.images[exactOrPastImageIndex];
// found exact, replace data
if (exactOrPastImage.at === img.at) {
set((s) => {
s.thumbnails.images[exactOrPastImageIndex] = img;
});
return;
}
// found one past, insert right before it
set((s) => {
s.thumbnails.images.splice(exactOrPastImageIndex, 0, img);
});
},
},
});

View File

@ -6,13 +6,15 @@ import { InterfaceSlice } from "@/stores/player/slices/interface";
import { PlayingSlice } from "@/stores/player/slices/playing";
import { ProgressSlice } from "@/stores/player/slices/progress";
import { SourceSlice } from "@/stores/player/slices/source";
import { ThumbnailSlice } from "@/stores/player/slices/thumbnails";
export type AllSlices = InterfaceSlice &
PlayingSlice &
ProgressSlice &
SourceSlice &
DisplaySlice &
CastingSlice;
CastingSlice &
ThumbnailSlice;
export type MakeSlice<Slice> = StateCreator<
AllSlices,
[["zustand/immer", never]],

View File

@ -7,6 +7,7 @@ import { createInterfaceSlice } from "@/stores/player/slices/interface";
import { createPlayingSlice } from "@/stores/player/slices/playing";
import { createProgressSlice } from "@/stores/player/slices/progress";
import { createSourceSlice } from "@/stores/player/slices/source";
import { createThumbnailSlice } from "@/stores/player/slices/thumbnails";
import { AllSlices } from "@/stores/player/slices/types";
export const usePlayerStore = create(
@ -17,5 +18,6 @@ export const usePlayerStore = create(
...createSourceSlice(...a),
...createDisplaySlice(...a),
...createCastingSlice(...a),
...createThumbnailSlice(...a),
}))
);