From cfb907924e1010c5b70756b5fd8dcf6ec91763de Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 27 Feb 2022 20:07:15 +0100 Subject: [PATCH] implement url based searching + caching of results Co-authored-by: James Hawkins --- src/App.tsx | 8 ++- src/hooks/useSearchQuery.ts | 26 ++++++++ src/providers/index.ts | 80 ++---------------------- src/providers/methods/helpers.ts | 16 +++++ src/providers/methods/providers.ts | 10 +++ src/providers/methods/search.ts | 87 +++++++++++++++++++++++++++ src/utils/cache.ts | 97 ++++++++++++++++++++++++++++++ src/views/SearchView.tsx | 10 ++- 8 files changed, 250 insertions(+), 84 deletions(-) create mode 100644 src/hooks/useSearchQuery.ts create mode 100644 src/providers/methods/helpers.ts create mode 100644 src/providers/methods/providers.ts create mode 100644 src/providers/methods/search.ts create mode 100644 src/utils/cache.ts diff --git a/src/App.tsx b/src/App.tsx index 1823dc00..0ef231ba 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,4 +1,5 @@ -import { Route, Switch } from "react-router-dom"; +import { MWMediaType } from "providers"; +import { Redirect, Route, Switch } from "react-router-dom"; import { WatchedContextProvider } from "state/watched/context"; import "./index.css"; import { MovieView } from "./views/MovieView"; @@ -9,9 +10,12 @@ function App() { return ( - + + + + ); diff --git a/src/hooks/useSearchQuery.ts b/src/hooks/useSearchQuery.ts new file mode 100644 index 00000000..3501f121 --- /dev/null +++ b/src/hooks/useSearchQuery.ts @@ -0,0 +1,26 @@ +import { MWMediaType, MWQuery } from "providers"; +import React, { useState } from "react"; +import { generatePath, useHistory, useRouteMatch } from "react-router-dom"; + +export function useSearchQuery(): [MWQuery, (inp: Partial) => void] { + const history = useHistory() + const { path, params } = useRouteMatch<{ type: string, query: string}>() + const [search, setSearch] = useState({ + searchQuery: "", + type: MWMediaType.MOVIE, + }); + + const updateParams = (inp: Partial) => { + const copySearch: MWQuery = {...search}; + Object.assign(copySearch, inp); + history.push(generatePath(path, { query: copySearch.searchQuery.length == 0 ? undefined : inp.searchQuery, type: copySearch.type })) + } + + React.useEffect(() => { + const type = Object.values(MWMediaType).find(v=>params.type === v) || MWMediaType.MOVIE; + const searchQuery = params.query || ""; + setSearch({ type, searchQuery }); + }, [params, setSearch]) + + return [search, updateParams] +} diff --git a/src/providers/index.ts b/src/providers/index.ts index 121d0eef..20a32d4b 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,85 +1,13 @@ -import Fuse from "fuse.js"; -import { tempScraper } from "./list/temp"; -import { theFlixScraper } from "./list/theflix"; +import { getProviderFromId } from "./methods/helpers"; import { - MWMassProviderOutput, MWMedia, - MWMediaType, MWPortableMedia, - MWQuery, MWMediaStream, } from "./types"; -import { MWWrappedMediaProvider, WrapProvider } from "./wrapper"; export * from "./types"; - -const mediaProvidersUnchecked: MWWrappedMediaProvider[] = [ - WrapProvider(theFlixScraper), - WrapProvider(tempScraper), -]; -export const mediaProviders: MWWrappedMediaProvider[] = - mediaProvidersUnchecked.filter((v) => v.enabled); - -/* - ** Fetch all enabled providers for a specific type - */ -export function GetProvidersForType(type: MWMediaType) { - return mediaProviders.filter((v) => v.type.includes(type)); -} - -/* - ** Call search on all providers that matches query type - */ -export async function SearchProviders( - query: MWQuery -): Promise { - const allQueries = GetProvidersForType(query.type).map< - Promise<{ media: MWMedia[]; success: boolean; id: string }> - >(async (provider) => { - try { - return { - media: await provider.searchForMedia(query), - success: true, - id: provider.id, - }; - } catch (err) { - console.error(`Failed running provider ${provider.id}`, err, query); - return { - media: [], - success: false, - id: provider.id, - }; - } - }); - const allResults = await Promise.all(allQueries); - const providerResults = allResults.map((provider) => ({ - success: provider.success, - id: provider.id, - })); - const output = { - results: allResults.flatMap((results) => results.media), - providers: providerResults, - stats: { - total: providerResults.length, - failed: providerResults.filter((v) => !v.success).length, - succeeded: providerResults.filter((v) => v.success).length, - }, - }; - - // sort results - const fuse = new Fuse(output.results, { threshold: 0.3, keys: ["title"] }); - output.results = fuse.search(query.searchQuery).map((v) => v.item); - - if (output.stats.total === output.stats.failed) - throw new Error("All Scrapers failed"); - return output; -} - -/* - ** Get a provider by a id - */ -export function getProviderFromId(id: string) { - return mediaProviders.find((v) => v.id === id); -} +export * from "./methods/helpers"; +export * from "./methods/providers"; +export * from "./methods/search"; /* ** Turn media object into a portable media object diff --git a/src/providers/methods/helpers.ts b/src/providers/methods/helpers.ts new file mode 100644 index 00000000..abb09466 --- /dev/null +++ b/src/providers/methods/helpers.ts @@ -0,0 +1,16 @@ +import { MWMediaType } from "providers"; +import { mediaProviders } from "./providers"; + +/* + ** Fetch all enabled providers for a specific type + */ + export function GetProvidersForType(type: MWMediaType) { + return mediaProviders.filter((v) => v.type.includes(type)); +} + +/* + ** Get a provider by a id + */ + export function getProviderFromId(id: string) { + return mediaProviders.find((v) => v.id === id); +} diff --git a/src/providers/methods/providers.ts b/src/providers/methods/providers.ts new file mode 100644 index 00000000..6496add2 --- /dev/null +++ b/src/providers/methods/providers.ts @@ -0,0 +1,10 @@ +import { tempScraper } from "providers/list/temp"; +import { theFlixScraper } from "providers/list/theflix"; +import { MWWrappedMediaProvider, WrapProvider } from "providers/wrapper"; + +const mediaProvidersUnchecked: MWWrappedMediaProvider[] = [ + WrapProvider(theFlixScraper), + WrapProvider(tempScraper), +]; +export const mediaProviders: MWWrappedMediaProvider[] = + mediaProvidersUnchecked.filter((v) => v.enabled); diff --git a/src/providers/methods/search.ts b/src/providers/methods/search.ts new file mode 100644 index 00000000..3353f758 --- /dev/null +++ b/src/providers/methods/search.ts @@ -0,0 +1,87 @@ +import Fuse from "fuse.js"; +import { MWMassProviderOutput, MWMedia, MWQuery } from "providers"; +import { SimpleCache } from "utils/cache"; +import { GetProvidersForType } from "./helpers"; + +// cache +const resultCache = new SimpleCache(); +resultCache.setCompare((a,b) => a.searchQuery === b.searchQuery && a.type === b.type); +resultCache.initialize(); + +/* +** actually call all providers with the search query +*/ +async function callProviders( + query: MWQuery +): Promise { + const allQueries = GetProvidersForType(query.type).map< + Promise<{ media: MWMedia[]; success: boolean; id: string }> + >(async (provider) => { + try { + return { + media: await provider.searchForMedia(query), + success: true, + id: provider.id, + }; + } catch (err) { + console.error(`Failed running provider ${provider.id}`, err, query); + return { + media: [], + success: false, + id: provider.id, + }; + } + }); + const allResults = await Promise.all(allQueries); + const providerResults = allResults.map((provider) => ({ + success: provider.success, + id: provider.id, + })); + const output: MWMassProviderOutput = { + results: allResults.flatMap((results) => results.media), + providers: providerResults, + stats: { + total: providerResults.length, + failed: providerResults.filter((v) => !v.success).length, + succeeded: providerResults.filter((v) => v.success).length, + }, + }; + + // save in cache if all successfull + if (output.stats.failed === 0) { + resultCache.set(query, output, 60 * 60); // cache for an hour + } + + return output; +} + +/* +** sort results based on query +*/ +function sortResults(query: MWQuery, providerResults: MWMassProviderOutput): MWMassProviderOutput { + const fuse = new Fuse(providerResults.results, { threshold: 0.3, keys: ["title"] }); + providerResults.results = fuse.search(query.searchQuery).map((v) => v.item); + return providerResults; +} + +/* + ** Call search on all providers that matches query type + */ +export async function SearchProviders( + query: MWQuery +): Promise { + // input normalisation + query.searchQuery = query.searchQuery.toLowerCase().trim(); + + // consult cache first + let output = resultCache.get(query); + if (!output) + output = await callProviders(query); + + // sort results + output = sortResults(query, output); + + if (output.stats.total === output.stats.failed) + throw new Error("All Scrapers failed"); + return output; +} diff --git a/src/utils/cache.ts b/src/utils/cache.ts new file mode 100644 index 00000000..2e08f0fa --- /dev/null +++ b/src/utils/cache.ts @@ -0,0 +1,97 @@ +export class SimpleCache { + protected readonly INTERVAL_MS = 2 * 60 * 1000; // 2 minutes + + protected _interval: NodeJS.Timer | null = null; + protected _compare: ((a: Key, b: Key) => boolean) | null = null; + protected _storage: { key: Key; value: Value; expiry: Date }[] = []; + + /* + ** initialize store, will start the interval + */ + public initialize(): void { + if (this._interval) throw new Error("cache is already initialized"); + this._interval = setInterval(() => { + const now = new Date(); + this._storage.filter((val) => { + if (val.expiry < now) return false; // remove if expiry date is in the past + return true; + }); + }, this.INTERVAL_MS); + } + + /* + ** destroy cache instance, its not safe to use the instance after calling this + */ + public destroy(): void { + if (this._interval) + clearInterval(this._interval); + this.clear(); + } + + /* + ** Set compare function, function must return true if A & B are equal + */ + public setCompare(compare: (a: Key, b: Key) => boolean): void { + this._compare = compare; + } + + /* + ** check if cache contains the item + */ + public has(key: Key): boolean { + return !!this.get(key); + } + + /* + ** get item from cache + */ + public get(key: Key): Value | undefined { + if (!this._compare) throw new Error("Compare function not set"); + const foundValue = this._storage.find(item => this._compare && this._compare(item.key, key)); + if (!foundValue) + return undefined; + return foundValue.value; + } + + /* + ** set item from cache, if it already exists, it will overwrite + */ + public set(key: Key, value: Value, expirySeconds: number): void { + if (!this._compare) throw new Error("Compare function not set"); + const foundValue = this._storage.find(item => this._compare && this._compare(item.key, key)); + const expiry = new Date((new Date().getTime()) + (expirySeconds * 1000)); + + // overwrite old value + if (foundValue) { + foundValue.key = key; + foundValue.value = value; + foundValue.expiry = expiry; + return; + } + + // add new value to storage + this._storage.push({ + key, + value, + expiry, + }) + } + + /* + ** remove item from cache + */ + public remove(key: Key): void { + if (!this._compare) throw new Error("Compare function not set"); + this._storage.filter((val) => { + if (this._compare && this._compare(val.key, key)) return false; // remove if compare is success + return true; + }); + } + + /* + ** clear entire cache storage + */ + public clear(): void { + this._storage = []; + } +} diff --git a/src/views/SearchView.tsx b/src/views/SearchView.tsx index fef0dc72..1af65fc7 100644 --- a/src/views/SearchView.tsx +++ b/src/views/SearchView.tsx @@ -17,6 +17,7 @@ import { useDebounce } from "hooks/useDebounce"; import { useLoading } from "hooks/useLoading"; import { IconPatch } from "components/buttons/IconPatch"; import { Navigation } from "components/layout/Navigation"; +import { useSearchQuery } from "hooks/useSearchQuery"; function SearchLoading() { return ; @@ -125,10 +126,7 @@ function SearchResultsView({ export function SearchView() { const [searching, setSearching] = useState(false); const [loading, setLoading] = useState(false); - const [search, setSearch] = useState({ - searchQuery: "", - type: MWMediaType.MOVIE, - }); + const [search, setSearch] = useSearchQuery(); const debouncedSearch = useDebounce(search, 2000); useEffect(() => { @@ -153,7 +151,7 @@ export function SearchView() { onChange={setSearch} value={search} placeholder="What movie do you want to watch?" - /> + /> {/* results view */} @@ -162,7 +160,7 @@ export function SearchView() { ) : searching ? ( setSearch((v) => ({ searchQuery: "", type: v.type }))} + clear={() => setSearch({ searchQuery: "" })} /> ) : null}