added theflix scraper and better search component

This commit is contained in:
Jelle van Snik 2022-02-17 18:25:12 +01:00
parent 80cad8f8f2
commit 68e81e8bff
14 changed files with 308 additions and 110 deletions

View File

@ -1,4 +1,4 @@
import { GetProviderFromId, MWMedia, MWMediaType } from "scrapers";
import { getProviderFromId, MWMedia, MWMediaType } from "scrapers";
import { Link } from "react-router-dom";
import { Icon, Icons } from "components/Icon";
@ -32,7 +32,7 @@ function MediaCardContent({
linkable,
watchedPercentage,
}: MediaCardProps) {
const provider = GetProviderFromId(media.providerId);
const provider = getProviderFromId(media.providerId);
if (!provider) {
return null;
@ -62,7 +62,9 @@ function MediaCardContent({
{/* card content */}
<div className="flex-1">
<h1 className="mb-1 font-bold text-white">{media.title}</h1>
<MediaMeta content={[provider.displayName, provider.type]} />
<MediaMeta
content={[provider.displayName, media.mediaType, media.year]}
/>
</div>
{/* hoverable chevron */}
@ -79,9 +81,8 @@ function MediaCardContent({
}
export function MediaCard(props: MediaCardProps) {
const provider = GetProviderFromId(props.media.providerId);
let link = "movie";
if (provider?.type === MWMediaType.SERIES) link = "series";
if (props.media.mediaType === MWMediaType.MOVIE) link = "series";
const content = <MediaCardContent {...props} />;

View File

@ -6,5 +6,5 @@ export interface WatchedMediaCardProps {
}
export function WatchedMediaCard(props: WatchedMediaCardProps) {
return <MediaCard watchedPercentage={72} media={props.media} linkable />;
return <MediaCard watchedPercentage={0} media={props.media} linkable />;
}

39
src/hooks/useLoading.ts Normal file
View File

@ -0,0 +1,39 @@
import React, { useState } from "react";
export function useLoading<T extends (...args: any) => Promise<any>>(
action: T
) {
const [loading, setLoading] = useState(false);
const [success, setSuccess] = useState(false);
const [error, setError] = useState<any | undefined>(undefined);
let isMounted = true;
React.useEffect(() => {
isMounted = true;
return () => {
isMounted = false;
};
}, []);
const doAction = async (...args: Parameters<T>) => {
setLoading(true);
setSuccess(false);
setError(undefined);
return new Promise((resolve) => {
action(...args)
.then((v) => {
if (!isMounted) return resolve(undefined);
setSuccess(true);
resolve(v);
})
.catch((err) => {
if (isMounted) {
setError(err);
setSuccess(false);
}
resolve(undefined);
});
}).finally(() => isMounted && setLoading(false));
};
return [doAction, loading, error, success];
}

2
src/mw_constants.ts Normal file
View File

@ -0,0 +1,2 @@
export const CORS_PROXY_URL =
"https://proxy-1.movie-web.workers.dev/?destination=";

View File

@ -1,3 +0,0 @@
# about scrapers
TODO - put stuff here later

View File

@ -1,21 +1,58 @@
import { theFlixMovieScraper } from "./list/theflixmovie";
import { theFlixSeriesScraper } from "./list/theflixseries";
import { MWMediaProvider, MWQuery } from "./types";
import { theFlixScraper } from "./list/theflix";
import { MWMedia, MWMediaType, MWPortableMedia, MWQuery } from "./types";
import { MWWrappedMediaProvider, WrapProvider } from "./wrapper";
export * from "./types";
const mediaProvidersUnchecked: MWMediaProvider[] = [
theFlixMovieScraper,
theFlixSeriesScraper,
]
export const mediaProviders: MWMediaProvider[] = mediaProvidersUnchecked.filter(v=>v.enabled);
const mediaProvidersUnchecked: MWWrappedMediaProvider[] = [
WrapProvider(theFlixScraper),
];
export const mediaProviders: MWWrappedMediaProvider[] =
mediaProvidersUnchecked.filter((v) => v.enabled);
export async function SearchProviders(query: MWQuery) {
const allQueries = mediaProviders.map(provider => provider.searchForMedia(query));
/*
** Fetch all enabled providers for a specific type
*/
export function GetProvidersForType(type: MWMediaType) {
return mediaProviders.filter((v) => v.type.includes(type));
}
/*
** Call search on all providers that matches query type
*/
export async function SearchProviders(query: MWQuery): Promise<MWMedia[]> {
const allQueries = GetProvidersForType(query.type).map((provider) =>
provider.searchForMedia(query)
);
const allResults = await Promise.all(allQueries);
return allResults.flatMap(results => results);
return allResults.flatMap((results) => results);
}
export function GetProviderFromId(id: string) {
return mediaProviders.find(v=>v.id===id);
/*
** Get a provider by a id
*/
export function getProviderFromId(id: string) {
return mediaProviders.find((v) => v.id === id);
}
/*
** Turn media object into a portable media object
*/
export function convertMediaToPortable(media: MWMedia): MWPortableMedia {
return {
mediaId: media.mediaId,
providerId: media.providerId,
mediaType: media.mediaType,
episode: media.episode,
season: media.season,
};
}
/*
** Turn portable media into media object
*/
export async function convertPortableToMedia(
portable: MWPortableMedia
): Promise<MWMedia | undefined> {
const provider = getProviderFromId(portable.providerId);
return await provider?.getMediaFromPortable(portable);
}

View File

@ -0,0 +1,46 @@
import {
MWMediaProvider,
MWMediaType,
MWPortableMedia,
MWQuery,
} from "scrapers/types";
import {
searchTheFlix,
getDataFromSearch,
turnDataIntoMedia,
} from "scrapers/list/theflix/search";
import { getDataFromPortableSearch } from "scrapers/list/theflix/portableToMedia";
import { MWProviderMediaResult } from "scrapers";
export const theFlixScraper: MWMediaProvider = {
id: "theflix",
enabled: true,
type: [MWMediaType.MOVIE, MWMediaType.SERIES],
displayName: "theflix",
async getMediaFromPortable(
media: MWPortableMedia
): Promise<MWProviderMediaResult> {
const data: any = await getDataFromPortableSearch(media);
return {
...media,
year: new Date(data.releaseDate).getFullYear().toString(),
title: data.name,
};
},
async searchForMedia(query: MWQuery): Promise<MWProviderMediaResult[]> {
const searchRes = await searchTheFlix(query);
const searchData = await getDataFromSearch(searchRes, 10);
const results: MWProviderMediaResult[] = [];
for (let item of searchData) {
results.push(turnDataIntoMedia(item));
}
return results;
},
};

View File

@ -0,0 +1,35 @@
import { CORS_PROXY_URL } from "mw_constants";
import { MWMediaType, MWPortableMedia } from "scrapers/types";
const getTheFlixUrl = (media: MWPortableMedia, params?: URLSearchParams) => {
if (media.mediaType === MWMediaType.MOVIE) {
return `https://theflix.to/movie/${media.mediaId}?${params}`;
} else if (media.mediaType === MWMediaType.SERIES) {
return `https://theflix.to/tv-show/${media.mediaId}/season-${media.season}/episode-${media.episode}`;
}
return "";
};
export async function getDataFromPortableSearch(
media: MWPortableMedia
): Promise<any> {
const params = new URLSearchParams();
params.append("movieInfo", media.mediaId);
const res = await fetch(CORS_PROXY_URL + getTheFlixUrl(media, params)).then(
(d) => d.text()
);
const node: Element = Array.from(
new DOMParser()
.parseFromString(res, "text/html")
.querySelectorAll(`script[id="__NEXT_DATA__"]`)
)[0];
if (media.mediaType === MWMediaType.MOVIE) {
return JSON.parse(node.innerHTML).props.pageProps.movie;
} else if (media.mediaType === MWMediaType.SERIES) {
return JSON.parse(node.innerHTML).props.pageProps.selectedTv;
}
}

View File

@ -0,0 +1,44 @@
import { CORS_PROXY_URL } from "mw_constants";
import { MWMediaType, MWProviderMediaResult, MWQuery } from "scrapers";
const getTheFlixUrl = (type: "tv-shows" | "movies", params: URLSearchParams) =>
`https://theflix.to/${type}/trending?${params}`;
export async function searchTheFlix(query: MWQuery): Promise<string> {
const params = new URLSearchParams();
params.append("search", query.searchQuery);
return await fetch(
CORS_PROXY_URL +
getTheFlixUrl(
query.type === MWMediaType.MOVIE ? "movies" : "tv-shows",
params
)
).then((d) => d.text());
}
export function getDataFromSearch(page: string, limit: number = 10): any[] {
const node: Element = Array.from(
new DOMParser()
.parseFromString(page, "text/html")
.querySelectorAll(`script[id="__NEXT_DATA__"]`)
)[0];
const data = JSON.parse(node.innerHTML);
return data.props.pageProps.mainList.docs
.filter((d: any) => d.available)
.slice(0, limit);
}
export function turnDataIntoMedia(data: any): MWProviderMediaResult {
return {
mediaId:
data.id +
"-" +
data.name
.replace(/[^a-z0-9]+|\s+/gim, " ")
.trim()
.replace(/\s+/g, "-")
.toLowerCase(),
title: data.name,
year: new Date(data.releaseDate).getFullYear().toString(),
};
}

View File

@ -1,23 +0,0 @@
import { MWMedia, MWMediaProvider, MWMediaType, MWPortableMedia, MWQuery } from "scrapers/types";
export const theFlixMovieScraper: MWMediaProvider = {
id: "theflixmovie",
enabled: true,
type: MWMediaType.MOVIE,
displayName: "TheFlix",
async getMediaFromPortable(media: MWPortableMedia): Promise<MWMedia> {
return {
...media,
title: "title is here"
}
},
async searchForMedia(query: MWQuery): Promise<MWMedia[]> {
return [{
mediaId: "a",
providerId: this.id,
title: `movie testing in progress`,
}];
},
}

View File

@ -1,23 +0,0 @@
import { MWMedia, MWMediaProvider, MWMediaType, MWPortableMedia, MWQuery } from "scrapers/types";
export const theFlixSeriesScraper: MWMediaProvider = {
id: "theflixseries",
enabled: true,
type: MWMediaType.SERIES,
displayName: "TheFlix",
async getMediaFromPortable(media: MWPortableMedia): Promise<MWMedia> {
return {
...media,
title: "title here"
}
},
async searchForMedia(query: MWQuery): Promise<MWMedia[]> {
return [{
mediaId: "b",
providerId: this.id,
title: `series test`,
}];
},
}

View File

@ -5,25 +5,31 @@ export enum MWMediaType {
}
export interface MWPortableMedia {
mediaId: string,
providerId: string,
mediaId: string;
mediaType: MWMediaType;
providerId: string;
season?: number;
episode?: number;
}
export interface MWMedia extends MWPortableMedia {
title: string,
title: string;
year: string;
}
export type MWProviderMediaResult = Omit<MWMedia, "mediaType" | "providerId">;
export interface MWQuery {
searchQuery: string,
type: MWMediaType,
searchQuery: string;
type: MWMediaType;
}
export interface MWMediaProvider {
id: string, // id of provider, must be unique
enabled: boolean,
type: MWMediaType,
displayName: string,
id: string; // id of provider, must be unique
enabled: boolean;
type: MWMediaType[];
displayName: string;
getMediaFromPortable(media: MWPortableMedia): Promise<MWMedia>,
searchForMedia(query: MWQuery): Promise<MWMedia[]>,
getMediaFromPortable(media: MWPortableMedia): Promise<MWProviderMediaResult>;
searchForMedia(query: MWQuery): Promise<MWProviderMediaResult[]>;
}

View File

@ -4,7 +4,6 @@
also type safety is important, this is all spaghetti with "any" everywhere
*/
function buildStoreObject(d: any) {
const data: any = {
versions: d.versions,
@ -23,7 +22,7 @@ function buildStoreObject(d: any) {
if (version.constructor !== Number || version < 0) version = -42;
// invalid on purpose so it will reset
else {
version = (version as number + 1).toString();
version = ((version as number) + 1).toString();
}
// check if version exists
@ -190,15 +189,19 @@ export function versionedStoreBuilder(): any {
}
// register helper
if (type === "instance") this._data.instanceHelpers[name as string] = helper;
else if (type === "static") this._data.staticHelpers[name as string] = helper;
if (type === "instance")
this._data.instanceHelpers[name as string] = helper;
else if (type === "static")
this._data.staticHelpers[name as string] = helper;
return this;
},
build() {
// check if version list doesnt skip versions
const versionListSorted = this._data.versionList.sort((a: number, b: number) => a - b);
const versionListSorted = this._data.versionList.sort(
(a: number, b: number) => a - b
);
versionListSorted.forEach((v: any, i: number, arr: any[]) => {
if (i === 0) return;
if (v !== arr[i - 1] + 1)

View File

@ -9,9 +9,57 @@ import { Loading } from "components/layout/Loading";
import { Tagline } from "components/Text/Tagline";
import { Title } from "components/Text/Title";
import { useDebounce } from "hooks/useDebounce";
import { useLoading } from "hooks/useLoading";
function SearchLoading() {
return <Loading className="my-12" text="Fetching your favourite shows..." />;
}
function SearchResultsView({ searchQuery }: { searchQuery: MWQuery }) {
const [results, setResults] = useState<MWMedia[]>([]);
const [runSearchQuery, loading, error, success] = useLoading(
(query: MWQuery) => SearchProviders(query)
);
useEffect(() => {
if (searchQuery.searchQuery !== "") runSearch(searchQuery);
}, [searchQuery]);
async function runSearch(query: MWQuery) {
const results = await runSearchQuery(query);
if (!results) return;
setResults(results);
}
return (
<div>
{/* results */}
{success && results.length > 0 ? (
<SectionHeading title="Search results" icon={Icons.SEARCH}>
{results.map((v) => (
<WatchedMediaCard
key={[v.mediaId, v.providerId].join("|")}
media={v}
/>
))}
</SectionHeading>
) : null}
{/* no results */}
{success && results.length === 0 ? <p>No results found</p> : null}
{/* error */}
{error ? <p>All scrapers failed</p> : null}
{/* Loading icon */}
{loading ? <SearchLoading /> : null}
</div>
);
}
export function SearchView() {
const [results, setResults] = useState<MWMedia[]>([]);
const [searching, setSearching] = useState<boolean>(false);
const [loading, setLoading] = useState<boolean>(false);
const [search, setSearch] = useState<MWQuery>({
searchQuery: "",
type: MWMediaType.MOVIE,
@ -19,19 +67,12 @@ export function SearchView() {
const debouncedSearch = useDebounce<MWQuery>(search, 2000);
useEffect(() => {
if (debouncedSearch.searchQuery !== "") runSearch(debouncedSearch);
}, [debouncedSearch]);
useEffect(() => {
setResults([]);
setSearching(search.searchQuery !== "");
setLoading(search.searchQuery !== "");
}, [search]);
async function runSearch(query: MWQuery) {
const results = await SearchProviders(query);
setResults(results);
}
const isLoading = search.searchQuery !== "" && results.length === 0;
const hasResult = results.length > 0;
useEffect(() => {
setLoading(false);
}, [debouncedSearch]);
return (
<ThinContainer>
@ -48,18 +89,11 @@ export function SearchView() {
/>
</div>
{/* results */}
{hasResult ? (
<SectionHeading title="Search results" icon={Icons.SEARCH}>
{results.map((v) => (
<WatchedMediaCard media={v} />
))}
</SectionHeading>
) : null}
{/* Loading icon */}
{isLoading ? (
<Loading className="my-12" text="Fetching your favourite shows..." />
{/* results view */}
{loading ? (
<SearchLoading />
) : searching ? (
<SearchResultsView searchQuery={debouncedSearch} />
) : null}
</ThinContainer>
);