From f1257973e70ff15f9b1e98bcf8c52cfe10297b3d Mon Sep 17 00:00:00 2001 From: Jelle van Snik Date: Wed, 11 Jan 2023 21:16:48 +0100 Subject: [PATCH] new backend interfaces --- src/backend/embeds/testEmbedScraper.ts | 20 +++++++++ src/backend/helpers/embed.ts | 24 ++++++++++ src/backend/helpers/provider.ts | 23 ++++++++++ src/backend/helpers/register.ts | 61 ++++++++++++++++++++++++++ src/backend/helpers/streams.ts | 9 ++++ src/backend/index.ts | 17 +++++++ src/backend/metadata/search.ts | 13 ++---- src/backend/metadata/types.ts | 13 ++++++ src/backend/providermeta/.gitkeep | 1 - src/backend/providers/.gitkeep | 1 - src/backend/providers/testProvider.ts | 32 ++++++++++++++ src/index.tsx | 1 + 12 files changed, 203 insertions(+), 12 deletions(-) create mode 100644 src/backend/embeds/testEmbedScraper.ts create mode 100644 src/backend/helpers/embed.ts create mode 100644 src/backend/helpers/provider.ts create mode 100644 src/backend/helpers/register.ts create mode 100644 src/backend/helpers/streams.ts create mode 100644 src/backend/index.ts create mode 100644 src/backend/metadata/types.ts delete mode 100644 src/backend/providermeta/.gitkeep delete mode 100644 src/backend/providers/.gitkeep create mode 100644 src/backend/providers/testProvider.ts diff --git a/src/backend/embeds/testEmbedScraper.ts b/src/backend/embeds/testEmbedScraper.ts new file mode 100644 index 00000000..8498c0a2 --- /dev/null +++ b/src/backend/embeds/testEmbedScraper.ts @@ -0,0 +1,20 @@ +import { MWEmbedType } from "../helpers/embed"; +import { registerEmbedScraper } from "../helpers/register"; +import { MWStreamType } from "../helpers/streams"; + +registerEmbedScraper({ + id: "testembed", + rank: 23, + for: MWEmbedType.OPENLOAD, + + async getStream({ progress, url }) { + console.log("scraping url: ", url); + progress(25); + progress(50); + progress(75); + return { + streamUrl: "hello-world", + type: MWStreamType.MP4, + }; + }, +}); diff --git a/src/backend/helpers/embed.ts b/src/backend/helpers/embed.ts new file mode 100644 index 00000000..20cd3b29 --- /dev/null +++ b/src/backend/helpers/embed.ts @@ -0,0 +1,24 @@ +import { MWStream } from "./streams"; + +export enum MWEmbedType { + OPENLOAD = "openload", +} + +export type MWEmbed = { + type: MWEmbedType | null; + url: string; +}; + +export type MWEmbedContext = { + progress(percentage: number): void; + url: string; +}; + +export type MWEmbedScraper = { + id: string; + for: MWEmbedType; + rank: number; + disabled?: boolean; + + getStream(ctx: MWEmbedContext): Promise; +}; diff --git a/src/backend/helpers/provider.ts b/src/backend/helpers/provider.ts new file mode 100644 index 00000000..38e4bbfa --- /dev/null +++ b/src/backend/helpers/provider.ts @@ -0,0 +1,23 @@ +import { MWMediaType } from "../metadata/types"; +import { MWEmbed } from "./embed"; +import { MWStream } from "./streams"; + +export type MWProviderScrapeResult = { + stream?: MWStream; + embeds: MWEmbed[]; +}; + +export type MWProviderContext = { + progress(percentage: number): void; + imdbId: string; + tmdbId: string; +}; + +export type MWProvider = { + id: string; + rank: number; + disabled?: boolean; + type: MWMediaType[]; + + scrape(ctx: MWProviderContext): Promise; +}; diff --git a/src/backend/helpers/register.ts b/src/backend/helpers/register.ts new file mode 100644 index 00000000..001aed16 --- /dev/null +++ b/src/backend/helpers/register.ts @@ -0,0 +1,61 @@ +import { MWEmbedScraper } from "./embed"; +import { MWProvider } from "./provider"; + +let providers: MWProvider[] = []; +let embeds: MWEmbedScraper[] = []; + +export function registerProvider(provider: MWProvider) { + if (provider.disabled) return; + providers.push(provider); +} +export function registerEmbedScraper(embed: MWEmbedScraper) { + if (embed.disabled) return; + embeds.push(embed); +} + +export function initializeScraperStore() { + // sort by ranking + providers = providers.sort((a, b) => a.rank - b.rank); + embeds = embeds.sort((a, b) => a.rank - b.rank); + + // check for invalid ranks + let lastRank: null | number = null; + providers.forEach((v) => { + if (lastRank === null) { + lastRank = v.rank; + return; + } + if (lastRank === v.rank) + throw new Error(`Duplicate rank number for provider ${v.id}`); + lastRank = v.rank; + }); + lastRank = null; + providers.forEach((v) => { + if (lastRank === null) { + lastRank = v.rank; + return; + } + if (lastRank === v.rank) + throw new Error(`Duplicate rank number for embed scraper ${v.id}`); + lastRank = v.rank; + }); + + // check for duplicate ids + const providerIds = providers.map((v) => v.id); + if ( + providerIds.length > 0 && + new Set(providerIds).size !== providerIds.length + ) + throw new Error("Duplicate IDS in providers"); + const embedIds = embeds.map((v) => v.id); + if (embedIds.length > 0 && new Set(embedIds).size !== embedIds.length) + throw new Error("Duplicate IDS in embed scrapers"); +} + +export function getProviders(): MWProvider[] { + return providers; +} + +export function getEmbeds(): MWEmbedScraper[] { + return embeds; +} diff --git a/src/backend/helpers/streams.ts b/src/backend/helpers/streams.ts new file mode 100644 index 00000000..6eb7257b --- /dev/null +++ b/src/backend/helpers/streams.ts @@ -0,0 +1,9 @@ +export enum MWStreamType { + MP4 = "mp4", + HLS = "hls", +} + +export type MWStream = { + streamUrl: string; + type: MWStreamType; +}; diff --git a/src/backend/index.ts b/src/backend/index.ts new file mode 100644 index 00000000..34983fad --- /dev/null +++ b/src/backend/index.ts @@ -0,0 +1,17 @@ +import { initializeScraperStore } from "./helpers/register"; + +// TODO backend system: +// - run providers/embedscrapers in webworkers for multithreading and isolation +// - caption support +// - hooks to run all providers one by one +// - move over old providers to new system +// - implement jons providers/embedscrapers + +// providers +// -- nothing here yet +import "./providers/testProvider"; + +// embeds +// -- nothing here yet + +initializeScraperStore(); diff --git a/src/backend/metadata/search.ts b/src/backend/metadata/search.ts index c4e844d7..fa26b35d 100644 --- a/src/backend/metadata/search.ts +++ b/src/backend/metadata/search.ts @@ -1,4 +1,5 @@ import { MWMediaType, MWQuery } from "@/providers"; +import { MWMediaMeta } from "./types"; const JW_API_BASE = "https://apis.justwatch.com"; @@ -27,18 +28,10 @@ type JWPage = { total_results: number; }; -export type MWSearchResult = { - title: string; - id: string; - year: string; - poster?: string; - type: MWMediaType; -}; - export async function searchForMedia({ searchQuery, type, -}: MWQuery): Promise { +}: MWQuery): Promise { const body: JWSearchQuery = { content_types: [], page: 1, @@ -56,7 +49,7 @@ export async function searchForMedia({ )}` ).then((res) => res.json() as Promise>); - return data.items.map((v) => ({ + return data.items.map((v) => ({ title: v.title, id: v.id.toString(), year: v.original_release_year.toString(), diff --git a/src/backend/metadata/types.ts b/src/backend/metadata/types.ts new file mode 100644 index 00000000..a74e0520 --- /dev/null +++ b/src/backend/metadata/types.ts @@ -0,0 +1,13 @@ +export enum MWMediaType { + MOVIE = "movie", + SERIES = "series", + ANIME = "anime", +} + +export type MWMediaMeta = { + title: string; + id: string; + year: string; + poster?: string; + type: MWMediaType; +}; diff --git a/src/backend/providermeta/.gitkeep b/src/backend/providermeta/.gitkeep deleted file mode 100644 index 37c97987..00000000 --- a/src/backend/providermeta/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -this folder will be used for provider helper methods and the like diff --git a/src/backend/providers/.gitkeep b/src/backend/providers/.gitkeep deleted file mode 100644 index 8fbcff9c..00000000 --- a/src/backend/providers/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -the new list of all providers, the old ones will go and be rewritten diff --git a/src/backend/providers/testProvider.ts b/src/backend/providers/testProvider.ts new file mode 100644 index 00000000..455325ca --- /dev/null +++ b/src/backend/providers/testProvider.ts @@ -0,0 +1,32 @@ +import { MWEmbedType } from "../helpers/embed"; +import { registerProvider } from "../helpers/register"; +import { MWStreamType } from "../helpers/streams"; +import { MWMediaType } from "../metadata/types"; + +registerProvider({ + id: "testprov", + rank: 42, + type: [MWMediaType.MOVIE], + + async scrape({ progress, imdbId, tmdbId }) { + console.log("scraping provider for: ", imdbId, tmdbId); + progress(25); + progress(50); + progress(75); + + // providers can optionally provide a stream themselves, + // incase they host their own streams instead of using embeds + return { + stream: { + streamUrl: "hello-world", + type: MWStreamType.HLS, + }, + embeds: [ + { + type: MWEmbedType.OPENLOAD, + url: "https://google.com", + }, + ], + }; + }, +}); diff --git a/src/index.tsx b/src/index.tsx index d6b93ba0..fbb6e122 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -7,6 +7,7 @@ import { conf } from "@/setup/config"; import App from "@/setup/App"; import "@/setup/i18n"; import "@/setup/index.css"; +import "@/backend"; // initialize const key =