implement url based searching + caching of results

Co-authored-by: James Hawkins <jhawki2005@gmail.com>
This commit is contained in:
mrjvs 2022-02-27 20:07:15 +01:00
parent 948ed68086
commit cfb907924e
8 changed files with 250 additions and 84 deletions

View File

@ -1,4 +1,5 @@
import { Route, Switch } from "react-router-dom"; import { MWMediaType } from "providers";
import { Redirect, Route, Switch } from "react-router-dom";
import { WatchedContextProvider } from "state/watched/context"; import { WatchedContextProvider } from "state/watched/context";
import "./index.css"; import "./index.css";
import { MovieView } from "./views/MovieView"; import { MovieView } from "./views/MovieView";
@ -9,9 +10,12 @@ function App() {
return ( return (
<WatchedContextProvider> <WatchedContextProvider>
<Switch> <Switch>
<Route exact path="/" component={SearchView} /> <Route exact path="/">
<Redirect to={`/${MWMediaType.MOVIE}`} />
</Route>
<Route exact path="/media/movie/:media" component={MovieView} /> <Route exact path="/media/movie/:media" component={MovieView} />
<Route exact path="/media/series/:media" component={SeriesView} /> <Route exact path="/media/series/:media" component={SeriesView} />
<Route exact path="/:type/:query?" component={SearchView} />
</Switch> </Switch>
</WatchedContextProvider> </WatchedContextProvider>
); );

View File

@ -0,0 +1,26 @@
import { MWMediaType, MWQuery } from "providers";
import React, { useState } from "react";
import { generatePath, useHistory, useRouteMatch } from "react-router-dom";
export function useSearchQuery(): [MWQuery, (inp: Partial<MWQuery>) => void] {
const history = useHistory()
const { path, params } = useRouteMatch<{ type: string, query: string}>()
const [search, setSearch] = useState<MWQuery>({
searchQuery: "",
type: MWMediaType.MOVIE,
});
const updateParams = (inp: Partial<MWQuery>) => {
const copySearch: MWQuery = {...search};
Object.assign(copySearch, inp);
history.push(generatePath(path, { query: copySearch.searchQuery.length == 0 ? undefined : inp.searchQuery, type: copySearch.type }))
}
React.useEffect(() => {
const type = Object.values(MWMediaType).find(v=>params.type === v) || MWMediaType.MOVIE;
const searchQuery = params.query || "";
setSearch({ type, searchQuery });
}, [params, setSearch])
return [search, updateParams]
}

View File

@ -1,85 +1,13 @@
import Fuse from "fuse.js"; import { getProviderFromId } from "./methods/helpers";
import { tempScraper } from "./list/temp";
import { theFlixScraper } from "./list/theflix";
import { import {
MWMassProviderOutput,
MWMedia, MWMedia,
MWMediaType,
MWPortableMedia, MWPortableMedia,
MWQuery,
MWMediaStream, MWMediaStream,
} from "./types"; } from "./types";
import { MWWrappedMediaProvider, WrapProvider } from "./wrapper";
export * from "./types"; export * from "./types";
export * from "./methods/helpers";
const mediaProvidersUnchecked: MWWrappedMediaProvider[] = [ export * from "./methods/providers";
WrapProvider(theFlixScraper), export * from "./methods/search";
WrapProvider(tempScraper),
];
export const mediaProviders: MWWrappedMediaProvider[] =
mediaProvidersUnchecked.filter((v) => v.enabled);
/*
** Fetch all enabled providers for a specific type
*/
export function GetProvidersForType(type: MWMediaType) {
return mediaProviders.filter((v) => v.type.includes(type));
}
/*
** Call search on all providers that matches query type
*/
export async function SearchProviders(
query: MWQuery
): Promise<MWMassProviderOutput> {
const allQueries = GetProvidersForType(query.type).map<
Promise<{ media: MWMedia[]; success: boolean; id: string }>
>(async (provider) => {
try {
return {
media: await provider.searchForMedia(query),
success: true,
id: provider.id,
};
} catch (err) {
console.error(`Failed running provider ${provider.id}`, err, query);
return {
media: [],
success: false,
id: provider.id,
};
}
});
const allResults = await Promise.all(allQueries);
const providerResults = allResults.map((provider) => ({
success: provider.success,
id: provider.id,
}));
const output = {
results: allResults.flatMap((results) => results.media),
providers: providerResults,
stats: {
total: providerResults.length,
failed: providerResults.filter((v) => !v.success).length,
succeeded: providerResults.filter((v) => v.success).length,
},
};
// sort results
const fuse = new Fuse(output.results, { threshold: 0.3, keys: ["title"] });
output.results = fuse.search(query.searchQuery).map((v) => v.item);
if (output.stats.total === output.stats.failed)
throw new Error("All Scrapers failed");
return output;
}
/*
** Get a provider by a id
*/
export function getProviderFromId(id: string) {
return mediaProviders.find((v) => v.id === id);
}
/* /*
** Turn media object into a portable media object ** Turn media object into a portable media object

View File

@ -0,0 +1,16 @@
import { MWMediaType } from "providers";
import { mediaProviders } from "./providers";
/*
** Fetch all enabled providers for a specific type
*/
export function GetProvidersForType(type: MWMediaType) {
return mediaProviders.filter((v) => v.type.includes(type));
}
/*
** Get a provider by a id
*/
export function getProviderFromId(id: string) {
return mediaProviders.find((v) => v.id === id);
}

View File

@ -0,0 +1,10 @@
import { tempScraper } from "providers/list/temp";
import { theFlixScraper } from "providers/list/theflix";
import { MWWrappedMediaProvider, WrapProvider } from "providers/wrapper";
const mediaProvidersUnchecked: MWWrappedMediaProvider[] = [
WrapProvider(theFlixScraper),
WrapProvider(tempScraper),
];
export const mediaProviders: MWWrappedMediaProvider[] =
mediaProvidersUnchecked.filter((v) => v.enabled);

View File

@ -0,0 +1,87 @@
import Fuse from "fuse.js";
import { MWMassProviderOutput, MWMedia, MWQuery } from "providers";
import { SimpleCache } from "utils/cache";
import { GetProvidersForType } from "./helpers";
// cache
const resultCache = new SimpleCache<MWQuery, MWMassProviderOutput>();
resultCache.setCompare((a,b) => a.searchQuery === b.searchQuery && a.type === b.type);
resultCache.initialize();
/*
** actually call all providers with the search query
*/
async function callProviders(
query: MWQuery
): Promise<MWMassProviderOutput> {
const allQueries = GetProvidersForType(query.type).map<
Promise<{ media: MWMedia[]; success: boolean; id: string }>
>(async (provider) => {
try {
return {
media: await provider.searchForMedia(query),
success: true,
id: provider.id,
};
} catch (err) {
console.error(`Failed running provider ${provider.id}`, err, query);
return {
media: [],
success: false,
id: provider.id,
};
}
});
const allResults = await Promise.all(allQueries);
const providerResults = allResults.map((provider) => ({
success: provider.success,
id: provider.id,
}));
const output: MWMassProviderOutput = {
results: allResults.flatMap((results) => results.media),
providers: providerResults,
stats: {
total: providerResults.length,
failed: providerResults.filter((v) => !v.success).length,
succeeded: providerResults.filter((v) => v.success).length,
},
};
// save in cache if all successfull
if (output.stats.failed === 0) {
resultCache.set(query, output, 60 * 60); // cache for an hour
}
return output;
}
/*
** sort results based on query
*/
function sortResults(query: MWQuery, providerResults: MWMassProviderOutput): MWMassProviderOutput {
const fuse = new Fuse(providerResults.results, { threshold: 0.3, keys: ["title"] });
providerResults.results = fuse.search(query.searchQuery).map((v) => v.item);
return providerResults;
}
/*
** Call search on all providers that matches query type
*/
export async function SearchProviders(
query: MWQuery
): Promise<MWMassProviderOutput> {
// input normalisation
query.searchQuery = query.searchQuery.toLowerCase().trim();
// consult cache first
let output = resultCache.get(query);
if (!output)
output = await callProviders(query);
// sort results
output = sortResults(query, output);
if (output.stats.total === output.stats.failed)
throw new Error("All Scrapers failed");
return output;
}

97
src/utils/cache.ts Normal file
View File

@ -0,0 +1,97 @@
export class SimpleCache<Key, Value> {
protected readonly INTERVAL_MS = 2 * 60 * 1000; // 2 minutes
protected _interval: NodeJS.Timer | null = null;
protected _compare: ((a: Key, b: Key) => boolean) | null = null;
protected _storage: { key: Key; value: Value; expiry: Date }[] = [];
/*
** initialize store, will start the interval
*/
public initialize(): void {
if (this._interval) throw new Error("cache is already initialized");
this._interval = setInterval(() => {
const now = new Date();
this._storage.filter((val) => {
if (val.expiry < now) return false; // remove if expiry date is in the past
return true;
});
}, this.INTERVAL_MS);
}
/*
** destroy cache instance, its not safe to use the instance after calling this
*/
public destroy(): void {
if (this._interval)
clearInterval(this._interval);
this.clear();
}
/*
** Set compare function, function must return true if A & B are equal
*/
public setCompare(compare: (a: Key, b: Key) => boolean): void {
this._compare = compare;
}
/*
** check if cache contains the item
*/
public has(key: Key): boolean {
return !!this.get(key);
}
/*
** get item from cache
*/
public get(key: Key): Value | undefined {
if (!this._compare) throw new Error("Compare function not set");
const foundValue = this._storage.find(item => this._compare && this._compare(item.key, key));
if (!foundValue)
return undefined;
return foundValue.value;
}
/*
** set item from cache, if it already exists, it will overwrite
*/
public set(key: Key, value: Value, expirySeconds: number): void {
if (!this._compare) throw new Error("Compare function not set");
const foundValue = this._storage.find(item => this._compare && this._compare(item.key, key));
const expiry = new Date((new Date().getTime()) + (expirySeconds * 1000));
// overwrite old value
if (foundValue) {
foundValue.key = key;
foundValue.value = value;
foundValue.expiry = expiry;
return;
}
// add new value to storage
this._storage.push({
key,
value,
expiry,
})
}
/*
** remove item from cache
*/
public remove(key: Key): void {
if (!this._compare) throw new Error("Compare function not set");
this._storage.filter((val) => {
if (this._compare && this._compare(val.key, key)) return false; // remove if compare is success
return true;
});
}
/*
** clear entire cache storage
*/
public clear(): void {
this._storage = [];
}
}

View File

@ -17,6 +17,7 @@ import { useDebounce } from "hooks/useDebounce";
import { useLoading } from "hooks/useLoading"; import { useLoading } from "hooks/useLoading";
import { IconPatch } from "components/buttons/IconPatch"; import { IconPatch } from "components/buttons/IconPatch";
import { Navigation } from "components/layout/Navigation"; import { Navigation } from "components/layout/Navigation";
import { useSearchQuery } from "hooks/useSearchQuery";
function SearchLoading() { function SearchLoading() {
return <Loading className="my-12" text="Fetching your favourite shows..." />; return <Loading className="my-12" text="Fetching your favourite shows..." />;
@ -125,10 +126,7 @@ function SearchResultsView({
export function SearchView() { export function SearchView() {
const [searching, setSearching] = useState<boolean>(false); const [searching, setSearching] = useState<boolean>(false);
const [loading, setLoading] = useState<boolean>(false); const [loading, setLoading] = useState<boolean>(false);
const [search, setSearch] = useState<MWQuery>({ const [search, setSearch] = useSearchQuery();
searchQuery: "",
type: MWMediaType.MOVIE,
});
const debouncedSearch = useDebounce<MWQuery>(search, 2000); const debouncedSearch = useDebounce<MWQuery>(search, 2000);
useEffect(() => { useEffect(() => {
@ -162,7 +160,7 @@ export function SearchView() {
) : searching ? ( ) : searching ? (
<SearchResultsView <SearchResultsView
searchQuery={debouncedSearch} searchQuery={debouncedSearch}
clear={() => setSearch((v) => ({ searchQuery: "", type: v.type }))} clear={() => setSearch({ searchQuery: "" })}
/> />
) : null} ) : null}
</ThinContainer> </ThinContainer>