From 68e81e8bffb2277b224751ff3533d438458177e3 Mon Sep 17 00:00:00 2001 From: Jelle van Snik Date: Thu, 17 Feb 2022 18:25:12 +0100 Subject: [PATCH] added theflix scraper and better search component --- src/components/media/MediaCard.tsx | 11 +-- src/components/media/WatchedMediaCard.tsx | 2 +- src/hooks/useLoading.ts | 39 +++++++++ src/mw_constants.ts | 2 + src/scrapers/README.md | 3 - src/scrapers/index.ts | 65 +++++++++++---- src/scrapers/list/theflix/index.ts | 46 +++++++++++ src/scrapers/list/theflix/portableToMedia.ts | 35 ++++++++ src/scrapers/list/theflix/search.ts | 44 ++++++++++ src/scrapers/list/theflixmovie/index.ts | 23 ------ src/scrapers/list/theflixseries/index.ts | 23 ------ src/scrapers/types.ts | 28 ++++--- src/utils/storage.ts | 13 +-- src/views/SearchView.tsx | 84 ++++++++++++++------ 14 files changed, 308 insertions(+), 110 deletions(-) create mode 100644 src/hooks/useLoading.ts create mode 100644 src/mw_constants.ts delete mode 100644 src/scrapers/README.md create mode 100644 src/scrapers/list/theflix/index.ts create mode 100644 src/scrapers/list/theflix/portableToMedia.ts create mode 100644 src/scrapers/list/theflix/search.ts delete mode 100644 src/scrapers/list/theflixmovie/index.ts delete mode 100644 src/scrapers/list/theflixseries/index.ts diff --git a/src/components/media/MediaCard.tsx b/src/components/media/MediaCard.tsx index 08c26e90..b6d90ffa 100644 --- a/src/components/media/MediaCard.tsx +++ b/src/components/media/MediaCard.tsx @@ -1,4 +1,4 @@ -import { GetProviderFromId, MWMedia, MWMediaType } from "scrapers"; +import { getProviderFromId, MWMedia, MWMediaType } from "scrapers"; import { Link } from "react-router-dom"; import { Icon, Icons } from "components/Icon"; @@ -32,7 +32,7 @@ function MediaCardContent({ linkable, watchedPercentage, }: MediaCardProps) { - const provider = GetProviderFromId(media.providerId); + const provider = getProviderFromId(media.providerId); if (!provider) { return null; @@ -62,7 +62,9 @@ function MediaCardContent({ {/* card content */}

{media.title}

- +
{/* hoverable chevron */} @@ -79,9 +81,8 @@ function MediaCardContent({ } export function MediaCard(props: MediaCardProps) { - const provider = GetProviderFromId(props.media.providerId); let link = "movie"; - if (provider?.type === MWMediaType.SERIES) link = "series"; + if (props.media.mediaType === MWMediaType.MOVIE) link = "series"; const content = ; diff --git a/src/components/media/WatchedMediaCard.tsx b/src/components/media/WatchedMediaCard.tsx index f1bed10d..e3019ff7 100644 --- a/src/components/media/WatchedMediaCard.tsx +++ b/src/components/media/WatchedMediaCard.tsx @@ -6,5 +6,5 @@ export interface WatchedMediaCardProps { } export function WatchedMediaCard(props: WatchedMediaCardProps) { - return ; + return ; } diff --git a/src/hooks/useLoading.ts b/src/hooks/useLoading.ts new file mode 100644 index 00000000..cd87dafe --- /dev/null +++ b/src/hooks/useLoading.ts @@ -0,0 +1,39 @@ +import React, { useState } from "react"; + +export function useLoading Promise>( + action: T +) { + const [loading, setLoading] = useState(false); + const [success, setSuccess] = useState(false); + const [error, setError] = useState(undefined); + let isMounted = true; + + React.useEffect(() => { + isMounted = true; + return () => { + isMounted = false; + }; + }, []); + + const doAction = async (...args: Parameters) => { + setLoading(true); + setSuccess(false); + setError(undefined); + return new Promise((resolve) => { + action(...args) + .then((v) => { + if (!isMounted) return resolve(undefined); + setSuccess(true); + resolve(v); + }) + .catch((err) => { + if (isMounted) { + setError(err); + setSuccess(false); + } + resolve(undefined); + }); + }).finally(() => isMounted && setLoading(false)); + }; + return [doAction, loading, error, success]; +} diff --git a/src/mw_constants.ts b/src/mw_constants.ts new file mode 100644 index 00000000..2274df2f --- /dev/null +++ b/src/mw_constants.ts @@ -0,0 +1,2 @@ +export const CORS_PROXY_URL = + "https://proxy-1.movie-web.workers.dev/?destination="; diff --git a/src/scrapers/README.md b/src/scrapers/README.md deleted file mode 100644 index 0684eef4..00000000 --- a/src/scrapers/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# about scrapers - -TODO - put stuff here later diff --git a/src/scrapers/index.ts b/src/scrapers/index.ts index cf58c1e0..5eb00845 100644 --- a/src/scrapers/index.ts +++ b/src/scrapers/index.ts @@ -1,21 +1,58 @@ -import { theFlixMovieScraper } from "./list/theflixmovie"; -import { theFlixSeriesScraper } from "./list/theflixseries"; -import { MWMediaProvider, MWQuery } from "./types"; +import { theFlixScraper } from "./list/theflix"; +import { MWMedia, MWMediaType, MWPortableMedia, MWQuery } from "./types"; +import { MWWrappedMediaProvider, WrapProvider } from "./wrapper"; export * from "./types"; -const mediaProvidersUnchecked: MWMediaProvider[] = [ - theFlixMovieScraper, - theFlixSeriesScraper, -] -export const mediaProviders: MWMediaProvider[] = mediaProvidersUnchecked.filter(v=>v.enabled); +const mediaProvidersUnchecked: MWWrappedMediaProvider[] = [ + WrapProvider(theFlixScraper), +]; +export const mediaProviders: MWWrappedMediaProvider[] = + mediaProvidersUnchecked.filter((v) => v.enabled); -export async function SearchProviders(query: MWQuery) { - const allQueries = mediaProviders.map(provider => provider.searchForMedia(query)); +/* + ** Fetch all enabled providers for a specific type + */ +export function GetProvidersForType(type: MWMediaType) { + return mediaProviders.filter((v) => v.type.includes(type)); +} + +/* + ** Call search on all providers that matches query type + */ +export async function SearchProviders(query: MWQuery): Promise { + const allQueries = GetProvidersForType(query.type).map((provider) => + provider.searchForMedia(query) + ); const allResults = await Promise.all(allQueries); - - return allResults.flatMap(results => results); + return allResults.flatMap((results) => results); } -export function GetProviderFromId(id: string) { - return mediaProviders.find(v=>v.id===id); +/* + ** Get a provider by a id + */ +export function getProviderFromId(id: string) { + return mediaProviders.find((v) => v.id === id); +} + +/* + ** Turn media object into a portable media object + */ +export function convertMediaToPortable(media: MWMedia): MWPortableMedia { + return { + mediaId: media.mediaId, + providerId: media.providerId, + mediaType: media.mediaType, + episode: media.episode, + season: media.season, + }; +} + +/* + ** Turn portable media into media object + */ +export async function convertPortableToMedia( + portable: MWPortableMedia +): Promise { + const provider = getProviderFromId(portable.providerId); + return await provider?.getMediaFromPortable(portable); } diff --git a/src/scrapers/list/theflix/index.ts b/src/scrapers/list/theflix/index.ts new file mode 100644 index 00000000..b262041e --- /dev/null +++ b/src/scrapers/list/theflix/index.ts @@ -0,0 +1,46 @@ +import { + MWMediaProvider, + MWMediaType, + MWPortableMedia, + MWQuery, +} from "scrapers/types"; + +import { + searchTheFlix, + getDataFromSearch, + turnDataIntoMedia, +} from "scrapers/list/theflix/search"; + +import { getDataFromPortableSearch } from "scrapers/list/theflix/portableToMedia"; +import { MWProviderMediaResult } from "scrapers"; + +export const theFlixScraper: MWMediaProvider = { + id: "theflix", + enabled: true, + type: [MWMediaType.MOVIE, MWMediaType.SERIES], + displayName: "theflix", + + async getMediaFromPortable( + media: MWPortableMedia + ): Promise { + const data: any = await getDataFromPortableSearch(media); + + return { + ...media, + year: new Date(data.releaseDate).getFullYear().toString(), + title: data.name, + }; + }, + + async searchForMedia(query: MWQuery): Promise { + const searchRes = await searchTheFlix(query); + const searchData = await getDataFromSearch(searchRes, 10); + + const results: MWProviderMediaResult[] = []; + for (let item of searchData) { + results.push(turnDataIntoMedia(item)); + } + + return results; + }, +}; diff --git a/src/scrapers/list/theflix/portableToMedia.ts b/src/scrapers/list/theflix/portableToMedia.ts new file mode 100644 index 00000000..935332d1 --- /dev/null +++ b/src/scrapers/list/theflix/portableToMedia.ts @@ -0,0 +1,35 @@ +import { CORS_PROXY_URL } from "mw_constants"; +import { MWMediaType, MWPortableMedia } from "scrapers/types"; + +const getTheFlixUrl = (media: MWPortableMedia, params?: URLSearchParams) => { + if (media.mediaType === MWMediaType.MOVIE) { + return `https://theflix.to/movie/${media.mediaId}?${params}`; + } else if (media.mediaType === MWMediaType.SERIES) { + return `https://theflix.to/tv-show/${media.mediaId}/season-${media.season}/episode-${media.episode}`; + } + + return ""; +}; + +export async function getDataFromPortableSearch( + media: MWPortableMedia +): Promise { + const params = new URLSearchParams(); + params.append("movieInfo", media.mediaId); + + const res = await fetch(CORS_PROXY_URL + getTheFlixUrl(media, params)).then( + (d) => d.text() + ); + + const node: Element = Array.from( + new DOMParser() + .parseFromString(res, "text/html") + .querySelectorAll(`script[id="__NEXT_DATA__"]`) + )[0]; + + if (media.mediaType === MWMediaType.MOVIE) { + return JSON.parse(node.innerHTML).props.pageProps.movie; + } else if (media.mediaType === MWMediaType.SERIES) { + return JSON.parse(node.innerHTML).props.pageProps.selectedTv; + } +} diff --git a/src/scrapers/list/theflix/search.ts b/src/scrapers/list/theflix/search.ts new file mode 100644 index 00000000..6754051e --- /dev/null +++ b/src/scrapers/list/theflix/search.ts @@ -0,0 +1,44 @@ +import { CORS_PROXY_URL } from "mw_constants"; +import { MWMediaType, MWProviderMediaResult, MWQuery } from "scrapers"; + +const getTheFlixUrl = (type: "tv-shows" | "movies", params: URLSearchParams) => + `https://theflix.to/${type}/trending?${params}`; + +export async function searchTheFlix(query: MWQuery): Promise { + const params = new URLSearchParams(); + params.append("search", query.searchQuery); + return await fetch( + CORS_PROXY_URL + + getTheFlixUrl( + query.type === MWMediaType.MOVIE ? "movies" : "tv-shows", + params + ) + ).then((d) => d.text()); +} + +export function getDataFromSearch(page: string, limit: number = 10): any[] { + const node: Element = Array.from( + new DOMParser() + .parseFromString(page, "text/html") + .querySelectorAll(`script[id="__NEXT_DATA__"]`) + )[0]; + const data = JSON.parse(node.innerHTML); + return data.props.pageProps.mainList.docs + .filter((d: any) => d.available) + .slice(0, limit); +} + +export function turnDataIntoMedia(data: any): MWProviderMediaResult { + return { + mediaId: + data.id + + "-" + + data.name + .replace(/[^a-z0-9]+|\s+/gim, " ") + .trim() + .replace(/\s+/g, "-") + .toLowerCase(), + title: data.name, + year: new Date(data.releaseDate).getFullYear().toString(), + }; +} diff --git a/src/scrapers/list/theflixmovie/index.ts b/src/scrapers/list/theflixmovie/index.ts deleted file mode 100644 index feb92053..00000000 --- a/src/scrapers/list/theflixmovie/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { MWMedia, MWMediaProvider, MWMediaType, MWPortableMedia, MWQuery } from "scrapers/types"; - -export const theFlixMovieScraper: MWMediaProvider = { - id: "theflixmovie", - enabled: true, - type: MWMediaType.MOVIE, - displayName: "TheFlix", - - async getMediaFromPortable(media: MWPortableMedia): Promise { - return { - ...media, - title: "title is here" - } - }, - - async searchForMedia(query: MWQuery): Promise { - return [{ - mediaId: "a", - providerId: this.id, - title: `movie testing in progress`, - }]; - }, -} diff --git a/src/scrapers/list/theflixseries/index.ts b/src/scrapers/list/theflixseries/index.ts deleted file mode 100644 index ce5a00c1..00000000 --- a/src/scrapers/list/theflixseries/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { MWMedia, MWMediaProvider, MWMediaType, MWPortableMedia, MWQuery } from "scrapers/types"; - -export const theFlixSeriesScraper: MWMediaProvider = { - id: "theflixseries", - enabled: true, - type: MWMediaType.SERIES, - displayName: "TheFlix", - - async getMediaFromPortable(media: MWPortableMedia): Promise { - return { - ...media, - title: "title here" - } - }, - - async searchForMedia(query: MWQuery): Promise { - return [{ - mediaId: "b", - providerId: this.id, - title: `series test`, - }]; - }, -} diff --git a/src/scrapers/types.ts b/src/scrapers/types.ts index ceb44885..e362f856 100644 --- a/src/scrapers/types.ts +++ b/src/scrapers/types.ts @@ -5,25 +5,31 @@ export enum MWMediaType { } export interface MWPortableMedia { - mediaId: string, - providerId: string, + mediaId: string; + mediaType: MWMediaType; + providerId: string; + season?: number; + episode?: number; } export interface MWMedia extends MWPortableMedia { - title: string, + title: string; + year: string; } +export type MWProviderMediaResult = Omit; + export interface MWQuery { - searchQuery: string, - type: MWMediaType, + searchQuery: string; + type: MWMediaType; } export interface MWMediaProvider { - id: string, // id of provider, must be unique - enabled: boolean, - type: MWMediaType, - displayName: string, + id: string; // id of provider, must be unique + enabled: boolean; + type: MWMediaType[]; + displayName: string; - getMediaFromPortable(media: MWPortableMedia): Promise, - searchForMedia(query: MWQuery): Promise, + getMediaFromPortable(media: MWPortableMedia): Promise; + searchForMedia(query: MWQuery): Promise; } diff --git a/src/utils/storage.ts b/src/utils/storage.ts index e662e74b..add77d96 100644 --- a/src/utils/storage.ts +++ b/src/utils/storage.ts @@ -4,7 +4,6 @@ also type safety is important, this is all spaghetti with "any" everywhere */ - function buildStoreObject(d: any) { const data: any = { versions: d.versions, @@ -23,7 +22,7 @@ function buildStoreObject(d: any) { if (version.constructor !== Number || version < 0) version = -42; // invalid on purpose so it will reset else { - version = (version as number + 1).toString(); + version = ((version as number) + 1).toString(); } // check if version exists @@ -190,15 +189,19 @@ export function versionedStoreBuilder(): any { } // register helper - if (type === "instance") this._data.instanceHelpers[name as string] = helper; - else if (type === "static") this._data.staticHelpers[name as string] = helper; + if (type === "instance") + this._data.instanceHelpers[name as string] = helper; + else if (type === "static") + this._data.staticHelpers[name as string] = helper; return this; }, build() { // check if version list doesnt skip versions - const versionListSorted = this._data.versionList.sort((a: number, b: number) => a - b); + const versionListSorted = this._data.versionList.sort( + (a: number, b: number) => a - b + ); versionListSorted.forEach((v: any, i: number, arr: any[]) => { if (i === 0) return; if (v !== arr[i - 1] + 1) diff --git a/src/views/SearchView.tsx b/src/views/SearchView.tsx index 660e0ae4..5092ef4b 100644 --- a/src/views/SearchView.tsx +++ b/src/views/SearchView.tsx @@ -9,9 +9,57 @@ import { Loading } from "components/layout/Loading"; import { Tagline } from "components/Text/Tagline"; import { Title } from "components/Text/Title"; import { useDebounce } from "hooks/useDebounce"; +import { useLoading } from "hooks/useLoading"; + +function SearchLoading() { + return ; +} + +function SearchResultsView({ searchQuery }: { searchQuery: MWQuery }) { + const [results, setResults] = useState([]); + const [runSearchQuery, loading, error, success] = useLoading( + (query: MWQuery) => SearchProviders(query) + ); + + useEffect(() => { + if (searchQuery.searchQuery !== "") runSearch(searchQuery); + }, [searchQuery]); + + async function runSearch(query: MWQuery) { + const results = await runSearchQuery(query); + if (!results) return; + setResults(results); + } + + return ( +
+ {/* results */} + {success && results.length > 0 ? ( + + {results.map((v) => ( + + ))} + + ) : null} + + {/* no results */} + {success && results.length === 0 ?

No results found

: null} + + {/* error */} + {error ?

All scrapers failed

: null} + + {/* Loading icon */} + {loading ? : null} +
+ ); +} export function SearchView() { - const [results, setResults] = useState([]); + const [searching, setSearching] = useState(false); + const [loading, setLoading] = useState(false); const [search, setSearch] = useState({ searchQuery: "", type: MWMediaType.MOVIE, @@ -19,19 +67,12 @@ export function SearchView() { const debouncedSearch = useDebounce(search, 2000); useEffect(() => { - if (debouncedSearch.searchQuery !== "") runSearch(debouncedSearch); - }, [debouncedSearch]); - useEffect(() => { - setResults([]); + setSearching(search.searchQuery !== ""); + setLoading(search.searchQuery !== ""); }, [search]); - - async function runSearch(query: MWQuery) { - const results = await SearchProviders(query); - setResults(results); - } - - const isLoading = search.searchQuery !== "" && results.length === 0; - const hasResult = results.length > 0; + useEffect(() => { + setLoading(false); + }, [debouncedSearch]); return ( @@ -48,18 +89,11 @@ export function SearchView() { /> - {/* results */} - {hasResult ? ( - - {results.map((v) => ( - - ))} - - ) : null} - - {/* Loading icon */} - {isLoading ? ( - + {/* results view */} + {loading ? ( + + ) : searching ? ( + ) : null} );