2023-12-18 21:47:19 +01:00
|
|
|
import {
|
|
|
|
FullScraperEvents,
|
|
|
|
RunOutput,
|
|
|
|
ScrapeMedia,
|
|
|
|
} from "@movie-web/providers";
|
2023-10-06 00:20:19 +02:00
|
|
|
import { RefObject, useCallback, useEffect, useRef, useState } from "react";
|
|
|
|
|
2024-01-18 19:21:43 +01:00
|
|
|
import { isExtensionActiveCached } from "@/backend/extension/messaging";
|
2024-01-09 21:56:39 +01:00
|
|
|
import { prepareStream } from "@/backend/extension/streams";
|
2023-12-19 00:10:46 +01:00
|
|
|
import {
|
|
|
|
connectServerSideEvents,
|
|
|
|
getCachedMetadata,
|
|
|
|
makeProviderUrl,
|
|
|
|
} from "@/backend/helpers/providerApi";
|
2024-01-09 20:07:22 +01:00
|
|
|
import { getLoadbalancedProviderApiUrl } from "@/backend/providers/fetchers";
|
|
|
|
import { getProviders } from "@/backend/providers/providers";
|
2023-10-06 00:20:19 +02:00
|
|
|
|
|
|
|
export interface ScrapingItems {
|
|
|
|
id: string;
|
|
|
|
children: string[];
|
|
|
|
}
|
|
|
|
|
|
|
|
export interface ScrapingSegment {
|
|
|
|
name: string;
|
|
|
|
id: string;
|
2023-11-11 16:17:13 +01:00
|
|
|
embedId?: string;
|
2023-10-06 00:20:19 +02:00
|
|
|
status: "failure" | "pending" | "notfound" | "success" | "waiting";
|
|
|
|
reason?: string;
|
2023-12-29 13:02:55 +01:00
|
|
|
error?: any;
|
2023-10-06 00:20:19 +02:00
|
|
|
percentage: number;
|
|
|
|
}
|
|
|
|
|
2023-12-18 21:47:19 +01:00
|
|
|
type ScraperEvent<Event extends keyof FullScraperEvents> = Parameters<
|
|
|
|
NonNullable<FullScraperEvents[Event]>
|
|
|
|
>[0];
|
|
|
|
|
|
|
|
function useBaseScrape() {
|
2023-10-06 00:20:19 +02:00
|
|
|
const [sources, setSources] = useState<Record<string, ScrapingSegment>>({});
|
|
|
|
const [sourceOrder, setSourceOrder] = useState<ScrapingItems[]>([]);
|
|
|
|
const [currentSource, setCurrentSource] = useState<string>();
|
2023-12-18 21:47:19 +01:00
|
|
|
const lastId = useRef<string | null>(null);
|
|
|
|
|
|
|
|
const initEvent = useCallback((evt: ScraperEvent<"init">) => {
|
|
|
|
setSources(
|
|
|
|
evt.sourceIds
|
|
|
|
.map((v) => {
|
2023-12-19 00:10:46 +01:00
|
|
|
const source = getCachedMetadata().find((s) => s.id === v);
|
2023-12-18 21:47:19 +01:00
|
|
|
if (!source) throw new Error("invalid source id");
|
|
|
|
const out: ScrapingSegment = {
|
|
|
|
name: source.name,
|
|
|
|
id: source.id,
|
|
|
|
status: "waiting",
|
|
|
|
percentage: 0,
|
|
|
|
};
|
|
|
|
return out;
|
|
|
|
})
|
|
|
|
.reduce<Record<string, ScrapingSegment>>((a, v) => {
|
|
|
|
a[v.id] = v;
|
|
|
|
return a;
|
2023-12-23 06:24:43 +01:00
|
|
|
}, {}),
|
2023-12-18 21:47:19 +01:00
|
|
|
);
|
|
|
|
setSourceOrder(evt.sourceIds.map((v) => ({ id: v, children: [] })));
|
|
|
|
}, []);
|
|
|
|
|
|
|
|
const startEvent = useCallback((id: ScraperEvent<"start">) => {
|
2023-12-29 21:35:09 +01:00
|
|
|
const lastIdTmp = lastId.current;
|
2023-12-18 21:47:19 +01:00
|
|
|
setSources((s) => {
|
|
|
|
if (s[id]) s[id].status = "pending";
|
2023-12-30 12:45:10 +01:00
|
|
|
if (lastIdTmp && s[lastIdTmp] && s[lastIdTmp].status === "pending")
|
|
|
|
s[lastIdTmp].status = "success";
|
2023-12-18 21:47:19 +01:00
|
|
|
return { ...s };
|
|
|
|
});
|
|
|
|
setCurrentSource(id);
|
|
|
|
lastId.current = id;
|
|
|
|
}, []);
|
|
|
|
|
|
|
|
const updateEvent = useCallback((evt: ScraperEvent<"update">) => {
|
|
|
|
setSources((s) => {
|
|
|
|
if (s[evt.id]) {
|
|
|
|
s[evt.id].status = evt.status;
|
|
|
|
s[evt.id].reason = evt.reason;
|
|
|
|
s[evt.id].error = evt.error;
|
|
|
|
s[evt.id].percentage = evt.percentage;
|
|
|
|
}
|
|
|
|
return { ...s };
|
|
|
|
});
|
|
|
|
}, []);
|
|
|
|
|
|
|
|
const discoverEmbedsEvent = useCallback(
|
|
|
|
(evt: ScraperEvent<"discoverEmbeds">) => {
|
|
|
|
setSources((s) => {
|
|
|
|
evt.embeds.forEach((v) => {
|
2023-12-19 00:10:46 +01:00
|
|
|
const source = getCachedMetadata().find(
|
2023-12-23 06:24:43 +01:00
|
|
|
(src) => src.id === v.embedScraperId,
|
2023-12-19 00:10:46 +01:00
|
|
|
);
|
2023-12-18 21:47:19 +01:00
|
|
|
if (!source) throw new Error("invalid source id");
|
|
|
|
const out: ScrapingSegment = {
|
|
|
|
embedId: v.embedScraperId,
|
|
|
|
name: source.name,
|
|
|
|
id: v.id,
|
|
|
|
status: "waiting",
|
|
|
|
percentage: 0,
|
|
|
|
};
|
|
|
|
s[v.id] = out;
|
|
|
|
});
|
|
|
|
return { ...s };
|
|
|
|
});
|
|
|
|
setSourceOrder((s) => {
|
|
|
|
const source = s.find((v) => v.id === evt.sourceId);
|
|
|
|
if (!source) throw new Error("invalid source id");
|
|
|
|
source.children = evt.embeds.map((v) => v.id);
|
|
|
|
return [...s];
|
|
|
|
});
|
|
|
|
},
|
2023-12-23 06:24:43 +01:00
|
|
|
[],
|
2023-12-18 21:47:19 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
const startScrape = useCallback(() => {
|
|
|
|
lastId.current = null;
|
|
|
|
}, []);
|
|
|
|
|
|
|
|
const getResult = useCallback((output: RunOutput | null) => {
|
|
|
|
if (output && lastId.current) {
|
|
|
|
setSources((s) => {
|
|
|
|
if (!lastId.current) return s;
|
|
|
|
if (s[lastId.current]) s[lastId.current].status = "success";
|
|
|
|
return { ...s };
|
|
|
|
});
|
|
|
|
}
|
|
|
|
return output;
|
|
|
|
}, []);
|
|
|
|
|
|
|
|
return {
|
|
|
|
initEvent,
|
|
|
|
startEvent,
|
|
|
|
updateEvent,
|
|
|
|
discoverEmbedsEvent,
|
|
|
|
startScrape,
|
|
|
|
getResult,
|
|
|
|
sources,
|
|
|
|
sourceOrder,
|
|
|
|
currentSource,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
export function useScrape() {
|
|
|
|
const {
|
|
|
|
sources,
|
|
|
|
sourceOrder,
|
|
|
|
currentSource,
|
|
|
|
updateEvent,
|
|
|
|
discoverEmbedsEvent,
|
|
|
|
initEvent,
|
|
|
|
getResult,
|
|
|
|
startEvent,
|
|
|
|
startScrape,
|
|
|
|
} = useBaseScrape();
|
2023-10-06 00:20:19 +02:00
|
|
|
|
|
|
|
const startScraping = useCallback(
|
|
|
|
async (media: ScrapeMedia) => {
|
2023-12-18 21:47:19 +01:00
|
|
|
const providerApiUrl = getLoadbalancedProviderApiUrl();
|
|
|
|
if (providerApiUrl) {
|
|
|
|
startScrape();
|
2023-12-19 00:10:46 +01:00
|
|
|
const baseUrlMaker = makeProviderUrl(providerApiUrl);
|
2023-12-19 20:41:56 +01:00
|
|
|
const conn = await connectServerSideEvents<RunOutput | "">(
|
2023-12-19 00:10:46 +01:00
|
|
|
baseUrlMaker.scrapeAll(media),
|
2023-12-23 06:24:43 +01:00
|
|
|
["completed", "noOutput"],
|
2023-12-18 21:47:19 +01:00
|
|
|
);
|
2023-12-19 00:10:46 +01:00
|
|
|
conn.on("init", initEvent);
|
|
|
|
conn.on("start", startEvent);
|
|
|
|
conn.on("update", updateEvent);
|
|
|
|
conn.on("discoverEmbeds", discoverEmbedsEvent);
|
|
|
|
const sseOutput = await conn.promise();
|
2024-01-09 21:56:39 +01:00
|
|
|
if (sseOutput) await prepareStream(sseOutput.stream);
|
2023-12-19 00:10:46 +01:00
|
|
|
|
|
|
|
return getResult(sseOutput === "" ? null : sseOutput);
|
2023-12-18 21:47:19 +01:00
|
|
|
}
|
2023-11-11 16:44:53 +01:00
|
|
|
|
2023-12-18 21:47:19 +01:00
|
|
|
startScrape();
|
2024-01-09 20:07:22 +01:00
|
|
|
const providers = getProviders();
|
2023-10-06 00:20:19 +02:00
|
|
|
const output = await providers.runAll({
|
|
|
|
media,
|
|
|
|
events: {
|
2023-12-18 21:47:19 +01:00
|
|
|
init: initEvent,
|
|
|
|
start: startEvent,
|
|
|
|
update: updateEvent,
|
|
|
|
discoverEmbeds: discoverEmbedsEvent,
|
2023-10-06 00:20:19 +02:00
|
|
|
},
|
|
|
|
});
|
2024-01-18 19:21:43 +01:00
|
|
|
if (output && isExtensionActiveCached())
|
|
|
|
await prepareStream(output.stream);
|
2023-12-18 21:47:19 +01:00
|
|
|
return getResult(output);
|
2023-10-06 00:20:19 +02:00
|
|
|
},
|
2023-12-18 21:47:19 +01:00
|
|
|
[
|
|
|
|
initEvent,
|
|
|
|
startEvent,
|
|
|
|
updateEvent,
|
|
|
|
discoverEmbedsEvent,
|
|
|
|
getResult,
|
|
|
|
startScrape,
|
2023-12-23 06:24:43 +01:00
|
|
|
],
|
2023-10-06 00:20:19 +02:00
|
|
|
);
|
|
|
|
|
|
|
|
return {
|
|
|
|
startScraping,
|
|
|
|
sourceOrder,
|
|
|
|
sources,
|
|
|
|
currentSource,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
export function useListCenter(
|
|
|
|
containerRef: RefObject<HTMLDivElement | null>,
|
|
|
|
listRef: RefObject<HTMLDivElement | null>,
|
|
|
|
sourceOrder: ScrapingItems[],
|
2023-12-23 06:24:43 +01:00
|
|
|
currentSource: string | undefined,
|
2023-10-06 00:20:19 +02:00
|
|
|
) {
|
|
|
|
const [renderedOnce, setRenderedOnce] = useState(false);
|
|
|
|
|
|
|
|
const updatePosition = useCallback(() => {
|
|
|
|
if (!containerRef.current) return;
|
|
|
|
if (!listRef.current) return;
|
|
|
|
|
|
|
|
const elements = [
|
|
|
|
...listRef.current.querySelectorAll("div[data-source-id]"),
|
|
|
|
] as HTMLDivElement[];
|
|
|
|
|
|
|
|
const currentIndex = elements.findIndex(
|
2023-12-23 06:24:43 +01:00
|
|
|
(e) => e.getAttribute("data-source-id") === currentSource,
|
2023-10-06 00:20:19 +02:00
|
|
|
);
|
|
|
|
|
|
|
|
const currentElement = elements[currentIndex];
|
|
|
|
|
|
|
|
if (!currentElement) return;
|
|
|
|
|
|
|
|
const containerWidth = containerRef.current.getBoundingClientRect().width;
|
|
|
|
const listWidth = listRef.current.getBoundingClientRect().width;
|
|
|
|
|
|
|
|
const containerHeight = containerRef.current.getBoundingClientRect().height;
|
|
|
|
|
|
|
|
const listTop = listRef.current.getBoundingClientRect().top;
|
|
|
|
|
|
|
|
const currentTop = currentElement.getBoundingClientRect().top;
|
|
|
|
const currentHeight = currentElement.getBoundingClientRect().height;
|
|
|
|
|
|
|
|
const topDifference = currentTop - listTop;
|
|
|
|
|
|
|
|
const listNewLeft = containerWidth / 2 - listWidth / 2;
|
|
|
|
const listNewTop = containerHeight / 2 - topDifference - currentHeight / 2;
|
|
|
|
|
|
|
|
listRef.current.style.transform = `translateY(${listNewTop}px) translateX(${listNewLeft}px)`;
|
|
|
|
setTimeout(() => {
|
|
|
|
setRenderedOnce(true);
|
|
|
|
}, 150);
|
|
|
|
}, [currentSource, containerRef, listRef, setRenderedOnce]);
|
|
|
|
|
|
|
|
const updatePositionRef = useRef(updatePosition);
|
|
|
|
|
|
|
|
useEffect(() => {
|
|
|
|
updatePosition();
|
|
|
|
updatePositionRef.current = updatePosition;
|
|
|
|
}, [updatePosition, sourceOrder]);
|
|
|
|
|
|
|
|
useEffect(() => {
|
|
|
|
function resize() {
|
|
|
|
updatePositionRef.current();
|
|
|
|
}
|
|
|
|
window.addEventListener("resize", resize);
|
|
|
|
return () => {
|
|
|
|
window.removeEventListener("resize", resize);
|
|
|
|
};
|
|
|
|
}, []);
|
|
|
|
|
|
|
|
return renderedOnce;
|
|
|
|
}
|