Merge pull request #169 from movie-web/feature-developer-tooling

Development tooling, round robin and better settings

index.html

@@ -25,7 +25,7 @@
     />
 
     <script src="config.js"></script>
-    <script src="https://cdn.jsdelivr.net/gh/movie-web/6C6F6C7A@3744edbc5f64a77985b6421ea5040e688663634b/out.js"></script>
+    <script src="https://cdn.jsdelivr.net/gh/movie-web/6C6F6C7A@8b821f445b83d51ef1b8f42c99b7346f6b47dce5/out.js"></script>
 
     <!-- prevent darkreader extension from messing with our already dark site -->
     <meta name="darkreader-lock" />

package.json

@@ -32,7 +32,7 @@
   "scripts": {
     "dev": "vite",
     "build": "vite build",
-    "preview": "vite preview",
+    "test": "vitest run",
     "lint": "eslint --ext .tsx,.ts src",
     "lint:fix": "eslint --fix --ext .tsx,.ts src",
     "lint:report": "eslint --ext .tsx,.ts --output-file eslint_report.json --format json src"
@@ -76,6 +76,7 @@
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-react": "7.29.4",
     "eslint-plugin-react-hooks": "4.3.0",
+    "jsdom": "^21.1.0",
     "postcss": "^8.4.20",
     "prettier": "^2.5.1",
     "prettier-plugin-tailwindcss": "^0.1.7",
@@ -84,6 +85,7 @@
     "typescript": "^4.6.4",
     "vite": "^4.0.1",
     "vite-plugin-checker": "^0.5.6",
-    "vite-plugin-package-version": "^1.0.2"
+    "vite-plugin-package-version": "^1.0.2",
+    "vitest": "^0.28.5"
   }
 }

src/__tests__/providers/providers.test.ts

@@ -0,0 +1,51 @@
+import { describe, it } from "vitest";
+import "@/backend";
+import { getProviders } from "@/backend/helpers/register";
+import { MWMediaType } from "@/backend/metadata/types";
+import { runProvider } from "@/backend/helpers/run";
+import { testData } from "@/__tests__/providers/testdata";
+
+describe("providers", () => {
+  const providers = getProviders();
+
+  it("have at least one provider", ({ expect }) => {
+    expect(providers.length).toBeGreaterThan(0);
+  });
+
+  for (const provider of providers) {
+    describe(provider.displayName, () => {
+      it("must have at least one type", async ({ expect }) => {
+        expect(provider.type.length).toBeGreaterThan(0);
+      });
+
+      if (provider.type.includes(MWMediaType.MOVIE)) {
+        it("must work with movies", async ({ expect }) => {
+          const movie = testData.find((v) => v.meta.type === MWMediaType.MOVIE);
+          if (!movie) throw new Error("no movie to test with");
+          const results = await runProvider(provider, {
+            media: movie,
+            progress() {},
+            type: movie.meta.type as any,
+          });
+          expect(results).toBeTruthy();
+        });
+      }
+
+      if (provider.type.includes(MWMediaType.SERIES)) {
+        it("must work with series", async ({ expect }) => {
+          const show = testData.find((v) => v.meta.type === MWMediaType.SERIES);
+          if (show?.meta.type !== MWMediaType.SERIES)
+            throw new Error("no show to test with");
+          const results = await runProvider(provider, {
+            media: show,
+            progress() {},
+            type: show.meta.type as MWMediaType.SERIES,
+            episode: show.meta.seasonData.episodes[0].id,
+            season: show.meta.seasons[0].id,
+          });
+          expect(results).toBeTruthy();
+        });
+      }
+    });
+  }
+});

src/__tests__/providers/testdata.ts

@@ -0,0 +1,45 @@
+import { DetailedMeta } from "@/backend/metadata/getmeta";
+import { MWMediaType } from "@/backend/metadata/types";
+
+export const testData: DetailedMeta[] = [
+  {
+    imdbId: "tt10954562",
+    tmdbId: "572716",
+    meta: {
+      id: "439596",
+      title: "Hamilton",
+      type: MWMediaType.MOVIE,
+      year: "2020",
+      seasons: undefined,
+    },
+  },
+  {
+    imdbId: "tt11126994",
+    tmdbId: "94605",
+    meta: {
+      id: "222333",
+      title: "Arcane",
+      type: MWMediaType.SERIES,
+      year: "2021",
+      seasons: [
+        {
+          id: "230301",
+          number: 1,
+          title: "Season 1",
+        },
+      ],
+      seasonData: {
+        id: "230301",
+        number: 1,
+        title: "Season 1",
+        episodes: [
+          {
+            id: "4243445",
+            number: 1,
+            title: "Welcome to the Playground",
+          },
+        ],
+      },
+    },
+  },
+];

src/backend/helpers/fetch.ts

@@ -1,6 +1,15 @@
 import { conf } from "@/setup/config";
 import { ofetch } from "ofetch";
 
+let proxyUrlIndex = Math.floor(Math.random() * conf().PROXY_URLS.length);
+
+// round robins all proxy urls
+function getProxyUrl(): string {
+  const url = conf().PROXY_URLS[proxyUrlIndex];
+  proxyUrlIndex = (proxyUrlIndex + 1) % conf().PROXY_URLS.length;
+  return url;
+}
+
 type P<T> = Parameters<typeof ofetch<T>>;
 type R<T> = ReturnType<typeof ofetch<T>>;
 
@@ -41,7 +50,7 @@ export function proxiedFetch<T>(url: string, ops: P<T>[1] = {}): R<T> {
     parsedUrl.searchParams.set(k, v);
   });
 
-  return baseFetch<T>(conf().BASE_PROXY_URL, {
+  return baseFetch<T>(getProxyUrl(), {
     ...ops,
     baseURL: undefined,
     params: {

src/index.tsx

@@ -16,7 +16,7 @@ import { initializeStores } from "./utils/storage";
 const key =
   (window as any)?.__CONFIG__?.VITE_KEY ?? import.meta.env.VITE_KEY ?? null;
 if (key) {
-  (window as any).initMW(conf().BASE_PROXY_URL, key);
+  (window as any).initMW(conf().PROXY_URLS, key);
 }
 initializeChromecast();
 

src/setup/App.tsx

@@ -7,16 +7,23 @@ import { MediaView } from "@/views/media/MediaView";
 import { SearchView } from "@/views/search/SearchView";
 import { MWMediaType } from "@/backend/metadata/types";
 import { V2MigrationView } from "@/views/other/v2Migration";
+import { DeveloperView } from "@/views/developer/DeveloperView";
+import { VideoTesterView } from "@/views/developer/VideoTesterView";
+import { ProviderTesterView } from "@/views/developer/ProviderTesterView";
+import { EmbedTesterView } from "@/views/developer/EmbedTesterView";
 
 function App() {
   return (
     <WatchedContextProvider>
       <BookmarkContextProvider>
         <Switch>
+          {/* functional routes */}
           <Route exact path="/v2-migration" component={V2MigrationView} />
           <Route exact path="/">
             <Redirect to={`/search/${MWMediaType.MOVIE}`} />
           </Route>
+
+          {/* pages */}
           <Route exact path="/media/:media" component={MediaView} />
           <Route
             exact
@@ -24,6 +31,12 @@ function App() {
             component={MediaView}
           />
           <Route exact path="/search/:type/:query?" component={SearchView} />
+
+          {/* other */}
+          <Route exact path="/dev" component={DeveloperView} />
+          <Route exact path="/dev/video" component={VideoTesterView} />
+          <Route exact path="/dev/providers" component={ProviderTesterView} />
+          <Route exact path="/dev/embeds" component={EmbedTesterView} />
           <Route path="*" component={NotFoundPage} />
         </Switch>
       </BookmarkContextProvider>

src/setup/config.ts

@@ -10,8 +10,14 @@ interface Config {
   NORMAL_ROUTER: boolean;
 }
 
-export interface RuntimeConfig extends Config {
+export interface RuntimeConfig {
-  BASE_PROXY_URL: string;
+  APP_VERSION: string;
+  GITHUB_LINK: string;
+  DISCORD_LINK: string;
+  OMDB_API_KEY: string;
+  TMDB_API_KEY: string;
+  NORMAL_ROUTER: boolean;
+  PROXY_URLS: string[];
 }
 
 const env: Record<keyof Config, undefined | string> = {
@@ -27,12 +33,13 @@ const env: Record<keyof Config, undefined | string> = {
 const alerts = [] as string[];
 
 // loads from different locations, in order: environment (VITE_{KEY}), window (public/config.js)
-function getKey(key: keyof Config): string {
+function getKey(key: keyof Config, defaultString?: string): string {
   let windowValue = (window as any)?.__CONFIG__?.[`VITE_${key}`];
   if (windowValue !== undefined && windowValue.length === 0)
     windowValue = undefined;
   const value = env[key] ?? windowValue ?? undefined;
   if (value === undefined) {
+    if (defaultString) return defaultString;
     if (!alerts.includes(key)) {
       // eslint-disable-next-line no-alert
       window.alert(`Misconfigured instance, missing key: ${key}`);
@@ -51,8 +58,9 @@ export function conf(): RuntimeConfig {
     DISCORD_LINK,
     OMDB_API_KEY: getKey("OMDB_API_KEY"),
     TMDB_API_KEY: getKey("TMDB_API_KEY"),
-    BASE_PROXY_URL: getKey("CORS_PROXY_URL"),
+    PROXY_URLS: getKey("CORS_PROXY_URL")
-    CORS_PROXY_URL: `${getKey("CORS_PROXY_URL")}/?destination=`,
+      .split(",")
-    NORMAL_ROUTER: (getKey("NORMAL_ROUTER") ?? "false") === "true",
+      .map((v) => v.trim()),
+    NORMAL_ROUTER: getKey("NORMAL_ROUTER", "false") === "true",
   };
 }

src/views/developer/DeveloperView.tsx

@@ -0,0 +1,26 @@
+import { Navigation } from "@/components/layout/Navigation";
+import { ThinContainer } from "@/components/layout/ThinContainer";
+import { ArrowLink } from "@/components/text/ArrowLink";
+import { Title } from "@/components/text/Title";
+
+export function DeveloperView() {
+  return (
+    <div className="py-48">
+      <Navigation />
+      <ThinContainer classNames="flex flex-col space-y-4">
+        <Title className="mb-8">Developer tools</Title>
+        <ArrowLink
+          to="/dev/providers"
+          direction="right"
+          linkText="Provider tester"
+        />
+        <ArrowLink
+          to="/dev/embeds"
+          direction="right"
+          linkText="Embed scraper tester"
+        />
+        <ArrowLink to="/dev/video" direction="right" linkText="Video tester" />
+      </ThinContainer>
+    </div>
+  );
+}

src/views/developer/EmbedTesterView.tsx

@@ -0,0 +1,136 @@
+import { MWEmbed, MWEmbedScraper, MWEmbedType } from "@/backend/helpers/embed";
+import { getEmbeds } from "@/backend/helpers/register";
+import { runEmbedScraper } from "@/backend/helpers/run";
+import { MWStream } from "@/backend/helpers/streams";
+import { Button } from "@/components/Button";
+import { Navigation } from "@/components/layout/Navigation";
+import { ArrowLink } from "@/components/text/ArrowLink";
+import { Title } from "@/components/text/Title";
+import { useLoading } from "@/hooks/useLoading";
+import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
+
+interface MediaSelectorProps {
+  embedType: MWEmbedType;
+  onSelect: (meta: MWEmbed) => void;
+}
+
+interface EmbedScraperSelectorProps {
+  onSelect: (embedScraperId: string) => void;
+}
+
+interface MediaScraperProps {
+  embed: MWEmbed;
+  scraper: MWEmbedScraper;
+}
+
+function MediaSelector(props: MediaSelectorProps) {
+  const [url, setUrl] = useState("");
+
+  const select = useCallback(
+    (urlSt: string) => {
+      props.onSelect({
+        type: props.embedType,
+        url: urlSt,
+      });
+    },
+    [props]
+  );
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Input embed url</Title>
+      <div className="mb-4 flex gap-4">
+        <input
+          type="text"
+          placeholder="embed url here..."
+          value={url}
+          onChange={(e) => setUrl(e.target.value)}
+        />
+        <Button onClick={() => select(url)}>Run scraper</Button>
+      </div>
+    </div>
+  );
+}
+
+function MediaScraper(props: MediaScraperProps) {
+  const [results, setResults] = useState<MWStream | null>(null);
+  const [percentage, setPercentage] = useState(0);
+
+  const [scrape, loading, error] = useLoading(async (url: string) => {
+    const data = await runEmbedScraper(props.scraper, {
+      url,
+      progress(num) {
+        console.log(`SCRAPING AT ${num}%`);
+        setPercentage(num);
+      },
+    });
+    console.log("got data", data);
+    setResults(data);
+  });
+
+  useEffect(() => {
+    if (props.embed) {
+      scrape(props.embed.url);
+    }
+  }, [props.embed, scrape]);
+
+  if (loading) return <p>Scraping... ({percentage}%)</p>;
+  if (error) return <p>Errored, check console</p>;
+
+  return (
+    <div>
+      <Title className="mb-8">Output data</Title>
+      <code>
+        <pre>{JSON.stringify(results, null, 2)}</pre>
+      </code>
+    </div>
+  );
+}
+
+function EmbedScraperSelector(props: EmbedScraperSelectorProps) {
+  const embedScrapers = getEmbeds();
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Choose embed scraper</Title>
+      {embedScrapers.map((v) => (
+        <ArrowLink
+          key={v.id}
+          onClick={() => props.onSelect(v.id)}
+          direction="right"
+          linkText={v.displayName}
+        />
+      ))}
+    </div>
+  );
+}
+
+export function EmbedTesterView() {
+  const [embed, setEmbed] = useState<MWEmbed | null>(null);
+  const [embedScraperId, setEmbedScraperId] = useState<string | null>(null);
+  const embedScraper = useMemo(
+    () => getEmbeds().find((v) => v.id === embedScraperId),
+    [embedScraperId]
+  );
+
+  let content: ReactNode = null;
+  if (!embedScraperId || !embedScraper) {
+    content = <EmbedScraperSelector onSelect={(id) => setEmbedScraperId(id)} />;
+  } else if (!embed) {
+    content = (
+      <MediaSelector
+        embedType={embedScraper.for}
+        onSelect={(v) => setEmbed(v)}
+      />
+    );
+  } else {
+    content = <MediaScraper scraper={embedScraper} embed={embed} />;
+  }
+
+  return (
+    <div className="py-48">
+      <Navigation />
+      <div className="mx-8 overflow-x-auto">{content}</div>
+    </div>
+  );
+}

src/views/developer/ProviderTesterView.tsx

@@ -0,0 +1,118 @@
+import { MWProviderScrapeResult } from "@/backend/helpers/provider";
+import { getProviders } from "@/backend/helpers/register";
+import { runProvider } from "@/backend/helpers/run";
+import { DetailedMeta } from "@/backend/metadata/getmeta";
+import { Navigation } from "@/components/layout/Navigation";
+import { ArrowLink } from "@/components/text/ArrowLink";
+import { Title } from "@/components/text/Title";
+import { useLoading } from "@/hooks/useLoading";
+import { testData } from "@/__tests__/providers/testdata";
+import { ReactNode, useEffect, useState } from "react";
+
+interface MediaSelectorProps {
+  onSelect: (meta: DetailedMeta) => void;
+}
+
+interface ProviderSelectorProps {
+  onSelect: (providerId: string) => void;
+}
+
+interface MediaScraperProps {
+  media: DetailedMeta | null;
+  id: string;
+}
+
+function MediaSelector(props: MediaSelectorProps) {
+  const options: DetailedMeta[] = testData;
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Choose media</Title>
+      {options.map((v) => (
+        <ArrowLink
+          key={v.imdbId}
+          onClick={() => props.onSelect(v)}
+          direction="right"
+          linkText={`${v.meta.title} (${v.meta.type})`}
+        />
+      ))}
+    </div>
+  );
+}
+
+function MediaScraper(props: MediaScraperProps) {
+  const [results, setResults] = useState<MWProviderScrapeResult | null>(null);
+  const [percentage, setPercentage] = useState(0);
+
+  const [scrape, loading, error] = useLoading(async (media: DetailedMeta) => {
+    const provider = getProviders().find((v) => v.id === props.id);
+    if (!provider) throw new Error("provider not found");
+    const data = await runProvider(provider, {
+      progress(num) {
+        console.log(`SCRAPING AT ${num}%`);
+        setPercentage(num);
+      },
+      media,
+      type: media.meta.type as any,
+    });
+    console.log("got data", data);
+    setResults(data);
+  });
+
+  useEffect(() => {
+    if (props.media) {
+      scrape(props.media);
+    }
+  }, [props.media, scrape]);
+
+  if (loading) return <p>Scraping... ({percentage}%)</p>;
+  if (error) return <p>Errored, check console</p>;
+
+  return (
+    <div>
+      <Title className="mb-8">Output data</Title>
+      <code>
+        <pre>{JSON.stringify(results, null, 2)}</pre>
+      </code>
+    </div>
+  );
+}
+
+function ProviderSelector(props: ProviderSelectorProps) {
+  const providers = getProviders();
+
+  return (
+    <div className="flex flex-col space-y-4">
+      <Title className="mb-8">Choose provider</Title>
+      {providers.map((v) => (
+        <ArrowLink
+          key={v.id}
+          onClick={() => props.onSelect(v.id)}
+          direction="right"
+          linkText={v.displayName}
+        />
+      ))}
+    </div>
+  );
+}
+
+export function ProviderTesterView() {
+  const [media, setMedia] = useState<DetailedMeta | null>(null);
+  const [providerId, setProviderId] = useState<string | null>(null);
+
+  let content: ReactNode = null;
+  if (!providerId) {
+    content = <ProviderSelector onSelect={(id) => setProviderId(id)} />;
+  } else if (!media) {
+    content = <MediaSelector onSelect={(v) => setMedia(v)} />;
+  } else {
+    content = <MediaScraper id={providerId} media={media} />;
+  }
+
+  return (
+    <div className="py-48">
+      <Navigation />
+      <div className="mx-8 overflow-x-auto">{content}</div>
+    </div>
+  );
+}

src/views/developer/VideoTesterView.tsx

@@ -0,0 +1,111 @@
+import { MWStreamQuality, MWStreamType } from "@/backend/helpers/streams";
+import { DetailedMeta } from "@/backend/metadata/getmeta";
+import { MWMediaType } from "@/backend/metadata/types";
+import { Button } from "@/components/Button";
+import { Dropdown } from "@/components/Dropdown";
+import { Navigation } from "@/components/layout/Navigation";
+import { ThinContainer } from "@/components/layout/ThinContainer";
+import { MetaController } from "@/video/components/controllers/MetaController";
+import { SourceController } from "@/video/components/controllers/SourceController";
+import { VideoPlayer } from "@/video/components/VideoPlayer";
+import { useCallback, useState } from "react";
+import { Helmet } from "react-helmet";
+
+interface VideoData {
+  streamUrl: string;
+  type: MWStreamType;
+}
+
+const testData: VideoData = {
+  streamUrl:
+    "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4",
+  type: MWStreamType.MP4,
+};
+const testMeta: DetailedMeta = {
+  imdbId: "",
+  tmdbId: "",
+  meta: {
+    id: "hello-world",
+    title: "Big Buck Bunny",
+    type: MWMediaType.MOVIE,
+    seasons: undefined,
+    year: "2000",
+  },
+};
+
+export function VideoTesterView() {
+  const [video, setVideo] = useState<VideoData | null>(null);
+  const [videoType, setVideoType] = useState<MWStreamType>(MWStreamType.MP4);
+  const [url, setUrl] = useState("");
+
+  const playVideo = useCallback(
+    (streamUrl: string) => {
+      setVideo({
+        streamUrl,
+        type: videoType,
+      });
+    },
+    [videoType]
+  );
+
+  if (video) {
+    return (
+      <div className="fixed top-0 left-0 h-[100dvh] w-screen">
+        <Helmet>
+          <html data-full="true" />
+        </Helmet>
+        <VideoPlayer includeSafeArea autoPlay onGoBack={() => setVideo(null)}>
+          <MetaController
+            data={{
+              captions: [],
+              meta: testMeta,
+            }}
+            linkedCaptions={[]}
+          />
+          <SourceController
+            source={video.streamUrl}
+            type={MWStreamType.MP4}
+            quality={MWStreamQuality.Q720P}
+          />
+        </VideoPlayer>
+      </div>
+    );
+  }
+
+  return (
+    <div className="py-64">
+      <Navigation />
+      <ThinContainer classNames="flex items-start flex-col space-y-4">
+        <div className="w-48">
+          <Dropdown
+            options={[
+              { id: MWStreamType.MP4, name: "Mp4" },
+              { id: MWStreamType.HLS, name: "hls/m3u8" },
+            ]}
+            selectedItem={{ id: videoType, name: videoType }}
+            setSelectedItem={(a) => setVideoType(a.id as MWStreamType)}
+          />
+        </div>
+        <div className="mb-4 flex gap-4">
+          <input
+            type="text"
+            placeholder="stream url here..."
+            value={url}
+            onChange={(e) => setUrl(e.target.value)}
+          />
+          <Button onClick={() => playVideo(url)}>Play video</Button>
+        </div>
+        <Button
+          onClick={() =>
+            setVideo({
+              streamUrl: testData.streamUrl,
+              type: testData.type,
+            })
+          }
+        >
+          Play default video
+        </Button>
+      </ThinContainer>
+    </div>
+  );
+}

tsconfig.json

@@ -1,25 +1,25 @@
-{
+{
-  "compilerOptions": {
+  "compilerOptions": {
-    "target": "ES2020",
+    "target": "ES2020",
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "lib": ["dom", "dom.iterable", "esnext"],
-    "allowJs": true,
+    "allowJs": true,
-    "skipLibCheck": true,
+    "skipLibCheck": true,
-    "esModuleInterop": true,
+    "esModuleInterop": true,
-    "allowSyntheticDefaultImports": true,
+    "allowSyntheticDefaultImports": true,
-    "strict": true,
+    "strict": true,
-    "forceConsistentCasingInFileNames": true,
+    "forceConsistentCasingInFileNames": true,
-    "noFallthroughCasesInSwitch": true,
+    "noFallthroughCasesInSwitch": true,
-    "module": "esnext",
+    "module": "esnext",
-    "moduleResolution": "node",
+    "moduleResolution": "node",
-    "resolveJsonModule": true,
+    "resolveJsonModule": true,
-    "isolatedModules": true,
+    "isolatedModules": true,
-    "noEmit": true,
+    "noEmit": true,
-    "jsx": "react-jsx",
+    "jsx": "react-jsx",
-    "baseUrl": "./src",
+    "baseUrl": "./src",
-    "paths": {
+    "paths": {
-      "@/*": ["./*"]
+      "@/*": ["./*"]
-    },
+    },
-    "types": ["vite/client"]
+    "types": ["vite/client"]
-  },
+  },
-  "include": ["src"]
+  "include": ["src"]
-}
+}

vite.config.ts

@@ -1,4 +1,4 @@
-import { defineConfig } from "vite";
+import { defineConfig } from "vitest/config";
 import react from "@vitejs/plugin-react-swc";
 import loadVersion from "vite-plugin-package-version";
 import checker from "vite-plugin-checker";
@@ -24,4 +24,8 @@ export default defineConfig({
       "@": path.resolve(__dirname, "./src"),
     },
   },
+
+  test: {
+    environment: "jsdom",
+  },
 });