From aca7827a15c2caf8603a8c3dfbf7e04ac028ef12 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 3 Jan 2024 20:06:08 +0100 Subject: [PATCH] Implement new country code system and new language code system --- package.json | 2 +- pnpm-lock.yaml | 11 +- src/assets/languages.ts | 2 - src/components/FlagIcon.tsx | 41 +--- src/components/form/Dropdown.tsx | 2 +- .../player/atoms/settings/CaptionsView.tsx | 11 +- .../player/atoms/settings/SettingsMenu.tsx | 4 +- src/components/player/utils/language.ts | 14 -- src/index.tsx | 5 +- src/pages/parts/settings/LocalePart.tsx | 10 +- src/setup/i18n.ts | 34 +--- src/stores/language/index.tsx | 17 +- src/utils/language.ts | 188 ++++++++++++++++++ src/utils/sortLangCodes.ts | 12 -- vite.config.mts | 2 +- 15 files changed, 243 insertions(+), 112 deletions(-) delete mode 100644 src/components/player/utils/language.ts create mode 100644 src/utils/language.ts delete mode 100644 src/utils/sortLangCodes.ts diff --git a/package.json b/package.json index 544f7500..c3e1b2d8 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dependencies": { "@formkit/auto-animate": "^0.8.1", "@headlessui/react": "^1.7.17", + "@ladjs/country-language": "^1.0.3", "@movie-web/providers": "^2.0.2", "@noble/hashes": "^1.3.3", "@react-spring/web": "^9.7.3", @@ -44,7 +45,6 @@ "hls.js": "^1.4.14", "i18next": "^23.7.11", "immer": "^10.0.3", - "iso-639-1": "^3.1.0", "jwt-decode": "^4.0.0", "lodash.isequal": "^4.5.0", "million": "^2.6.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6ef1bada..6022bda6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ dependencies: '@headlessui/react': specifier: ^1.7.17 version: 1.7.17(react-dom@18.2.0)(react@18.2.0) + '@ladjs/country-language': + specifier: ^1.0.3 + version: 1.0.3 '@movie-web/providers': specifier: ^2.0.2 version: 2.0.3 @@ -65,9 +68,6 @@ dependencies: immer: specifier: ^10.0.3 version: 10.0.3 - iso-639-1: - specifier: ^3.1.0 - version: 3.1.0 jwt-decode: specifier: ^4.0.0 version: 4.0.0 @@ -1912,6 +1912,11 @@ packages: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.4.15 + /@ladjs/country-language@1.0.3: + resolution: {integrity: sha512-FJROu9/hh4eqVAGDyfL8vpv6Vb0qKHX1ozYLRZ+beUzD5xFf+3r0J+SVIWKviEa7W524Qvqou+ta1WrsRgzxGw==} + engines: {node: '>= 14'} + dev: false + /@movie-web/providers@2.0.3: resolution: {integrity: sha512-6UNk5EebiNjGoFTuyHuu0eZZTreRYv0cdsn52CVYjm6CXG63w4dMbx8ybxcvMUrDF3o8bWlqnlovG142sdOmNw==} dependencies: diff --git a/src/assets/languages.ts b/src/assets/languages.ts index 6af96bf7..baa36531 100644 --- a/src/assets/languages.ts +++ b/src/assets/languages.ts @@ -50,5 +50,3 @@ export const locales = { uk, }; export type Locales = keyof typeof locales; - -export const rtlLocales: Locales[] = ["he", "ar"]; diff --git a/src/components/FlagIcon.tsx b/src/components/FlagIcon.tsx index 58466819..96f9431e 100644 --- a/src/components/FlagIcon.tsx +++ b/src/components/FlagIcon.tsx @@ -1,53 +1,32 @@ import classNames from "classnames"; + +import { getCountryCodeForLocale } from "@/utils/language"; import "flag-icons/css/flag-icons.min.css"; export interface FlagIconProps { - countryCode?: string; + country?: string; + langCode?: string; } -// Country code overrides -const countryOverrides: Record = { - en: "gb", - cs: "cz", - el: "gr", - fa: "ir", - ko: "kr", - he: "il", - ze: "cn", - ar: "sa", - ja: "jp", - bs: "ba", - vi: "vn", - zh: "cn", - sl: "si", - sv: "se", - et: "ee", - ne: "np", - uk: "ua", - hi: "in", -}; - export function FlagIcon(props: FlagIconProps) { - let countryCode = - (props.countryCode || "")?.split("-").pop()?.toLowerCase() || ""; - if (countryOverrides[countryCode]) - countryCode = countryOverrides[countryCode]; + let countryCode: string | null = props.country ?? null; + if (props.langCode) countryCode = getCountryCodeForLocale(props.langCode); - if (countryCode === "tok") + if (props.langCode === "tok") return (
); - if (countryCode === "pirate") + if (props.langCode === "pirate") return (
); - if (countryCode === "minion") + if (props.langCode === "minion") return (
@@ -66,7 +45,7 @@ export function FlagIcon(props: FlagIconProps) { className={classNames( "!w-8 h-6 rounded overflow-hidden bg-cover bg-center block fi", backgroundClass, - props.countryCode ? `fi-${countryCode}` : undefined, + countryCode ? `fi-${countryCode}` : undefined, )} /> ); diff --git a/src/components/form/Dropdown.tsx b/src/components/form/Dropdown.tsx index febe923f..df647b5f 100644 --- a/src/components/form/Dropdown.tsx +++ b/src/components/form/Dropdown.tsx @@ -17,7 +17,7 @@ interface DropdownProps { export function Dropdown(props: DropdownProps) { return ( -
+
{() => ( <> diff --git a/src/components/player/atoms/settings/CaptionsView.tsx b/src/components/player/atoms/settings/CaptionsView.tsx index b640f9cd..dc9a112f 100644 --- a/src/components/player/atoms/settings/CaptionsView.tsx +++ b/src/components/player/atoms/settings/CaptionsView.tsx @@ -10,12 +10,14 @@ import { useCaptions } from "@/components/player/hooks/useCaptions"; import { Menu } from "@/components/player/internals/ContextMenu"; import { Input } from "@/components/player/internals/ContextMenu/Input"; import { SelectableLink } from "@/components/player/internals/ContextMenu/Links"; -import { getLanguageFromIETF } from "@/components/player/utils/language"; import { useOverlayRouter } from "@/hooks/useOverlayRouter"; import { CaptionListItem } from "@/stores/player/slices/source"; import { usePlayerStore } from "@/stores/player/store"; import { useSubtitleStore } from "@/stores/subtitles"; -import { sortLangCodes } from "@/utils/sortLangCodes"; +import { + getPrettyLanguageNameFromLocale, + sortLangCodes, +} from "@/utils/language"; export function CaptionOption(props: { countryCode?: string; @@ -37,7 +39,7 @@ export function CaptionOption(props: { className="flex items-center" > - + {props.children} @@ -89,7 +91,8 @@ function useSubtitleList(subs: CaptionListItem[], searchQuery: string) { return useMemo(() => { const input = subs.map((t) => ({ ...t, - languageName: getLanguageFromIETF(t.language) ?? unknownChoice, + languageName: + getPrettyLanguageNameFromLocale(t.language) ?? unknownChoice, })); const sorted = sortLangCodes(input.map((t) => t.language)); let results = input.sort((a, b) => { diff --git a/src/components/player/atoms/settings/SettingsMenu.tsx b/src/components/player/atoms/settings/SettingsMenu.tsx index c9ecdf52..8321c562 100644 --- a/src/components/player/atoms/settings/SettingsMenu.tsx +++ b/src/components/player/atoms/settings/SettingsMenu.tsx @@ -6,11 +6,11 @@ import { Toggle } from "@/components/buttons/Toggle"; import { Icon, Icons } from "@/components/Icon"; import { useCaptions } from "@/components/player/hooks/useCaptions"; import { Menu } from "@/components/player/internals/ContextMenu"; -import { getLanguageFromIETF } from "@/components/player/utils/language"; import { useOverlayRouter } from "@/hooks/useOverlayRouter"; import { usePlayerStore } from "@/stores/player/store"; import { qualityToString } from "@/stores/player/utils/qualities"; import { useSubtitleStore } from "@/stores/subtitles"; +import { getPrettyLanguageNameFromLocale } from "@/utils/language"; export function SettingsMenu({ id }: { id: string }) { const { t } = useTranslation(); @@ -31,7 +31,7 @@ export function SettingsMenu({ id }: { id: string }) { const { toggleLastUsed } = useCaptions(); const selectedLanguagePretty = selectedCaptionLanguage - ? getLanguageFromIETF(selectedCaptionLanguage) ?? + ? getPrettyLanguageNameFromLocale(selectedCaptionLanguage) ?? t("player.menus.subtitles.unknownLanguage") : undefined; diff --git a/src/components/player/utils/language.ts b/src/components/player/utils/language.ts deleted file mode 100644 index bf31e786..00000000 --- a/src/components/player/utils/language.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { getTag } from "@sozialhelden/ietf-language-tags"; - -export function getLanguageFromIETF(ietf: string): string | null { - const tag = getTag(ietf, true); - - const lang = tag?.language?.Description?.[0] ?? null; - if (!lang) return null; - - const region = tag?.region?.Description?.[0] ?? null; - let regionText = ""; - if (region) regionText = ` (${region})`; - - return `${lang}${regionText}`; -} diff --git a/src/index.tsx b/src/index.tsx index a0376611..d5fffaae 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -23,10 +23,9 @@ import { MigrationPart } from "@/pages/parts/migrations/MigrationPart"; import { LargeTextPart } from "@/pages/parts/util/LargeTextPart"; import App from "@/setup/App"; import { conf } from "@/setup/config"; -import i18n from "@/setup/i18n"; import { useAuthStore } from "@/stores/auth"; import { BookmarkSyncer } from "@/stores/bookmarks/BookmarkSyncer"; -import { useLanguageStore } from "@/stores/language"; +import { changeAppLanguage, useLanguageStore } from "@/stores/language"; import { ProgressSyncer } from "@/stores/progress/ProgressSyncer"; import { SettingsSyncer } from "@/stores/subtitles/SettingsSyncer"; import { ThemeProvider } from "@/stores/theme"; @@ -123,7 +122,7 @@ function AuthWrapper() { function MigrationRunner() { const status = useAsync(async () => { - i18n.changeLanguage(useLanguageStore.getState().language); + changeAppLanguage(useLanguageStore.getState().language); await initializeOldStores(); }, []); const { t } = useTranslation(); diff --git a/src/pages/parts/settings/LocalePart.tsx b/src/pages/parts/settings/LocalePart.tsx index ef9801b9..87819451 100644 --- a/src/pages/parts/settings/LocalePart.tsx +++ b/src/pages/parts/settings/LocalePart.tsx @@ -4,7 +4,7 @@ import { FlagIcon } from "@/components/FlagIcon"; import { Dropdown } from "@/components/form/Dropdown"; import { Heading1 } from "@/components/utils/Text"; import { appLanguageOptions } from "@/setup/i18n"; -import { sortLangCodes } from "@/utils/sortLangCodes"; +import { getLocaleInfo, sortLangCodes } from "@/utils/language"; export function LocalePart(props: { language: string; @@ -17,11 +17,13 @@ export function LocalePart(props: { .sort((a, b) => sorted.indexOf(a.code) - sorted.indexOf(b.code)) .map((opt) => ({ id: opt.code, - name: `${opt.name} — ${opt.nativeName}`, - leftIcon: , + name: `${opt.name}${opt.nativeName ? ` — ${opt.nativeName}` : ""}`, + leftIcon: , })); - const selected = options.find((item) => item.id === props.language); + const selected = options.find( + (item) => item.id === getLocaleInfo(props.language)?.code, + ); return (
diff --git a/src/setup/i18n.ts b/src/setup/i18n.ts index 0110363a..ca263975 100644 --- a/src/setup/i18n.ts +++ b/src/setup/i18n.ts @@ -1,8 +1,8 @@ import i18n from "i18next"; -import ISO6391 from "iso-639-1"; import { initReactI18next } from "react-i18next"; import { locales } from "@/assets/languages"; +import { getLocaleInfo } from "@/utils/language"; // Languages const langCodes = Object.keys(locales); @@ -10,43 +10,15 @@ const resources = Object.fromEntries( Object.entries(locales).map((entry) => [entry[0], { translation: entry[1] }]), ); i18n.use(initReactI18next).init({ - fallbackLng: "en", + fallbackLng: "en-US", resources, interpolation: { escapeValue: false, // not needed for react as it escapes by default }, }); -const extraLanguages: Record< - string, - { - code: string; - name: string; - nativeName: string; - } -> = { - pirate: { - code: "pirate", - name: "Pirate", - nativeName: "Pirate Tongue", - }, - minion: { - code: "minion", - name: "Minion", - nativeName: "Minionese", - }, - tok: { - code: "tok", - name: "Toki pona", - nativeName: "Toki pona", - }, -}; - export const appLanguageOptions = langCodes.map((lang) => { - const extraLang = extraLanguages[lang]; - if (extraLang) return extraLang; - - const [langObj] = ISO6391.getLanguages([lang]); + const langObj = getLocaleInfo(lang); if (!langObj) throw new Error(`Language with code ${lang} cannot be found in database`); return langObj; diff --git a/src/stores/language/index.tsx b/src/stores/language/index.tsx index 9cee9d19..c901fddf 100644 --- a/src/stores/language/index.tsx +++ b/src/stores/language/index.tsx @@ -4,8 +4,8 @@ import { create } from "zustand"; import { persist } from "zustand/middleware"; import { immer } from "zustand/middleware/immer"; -import { rtlLocales } from "@/assets/languages"; import i18n from "@/setup/i18n"; +import { getLocaleInfo } from "@/utils/language"; export interface LanguageStore { language: string; @@ -26,14 +26,25 @@ export const useLanguageStore = create( ), ); +export function changeAppLanguage(language: string) { + const lang = getLocaleInfo(language); + if (lang) i18n.changeLanguage(lang.code); +} + +export function isRightToLeft(language: string) { + const lang = getLocaleInfo(language); + if (!lang) return false; + return lang.isRtl; +} + export function LanguageProvider() { const language = useLanguageStore((s) => s.language); useEffect(() => { - i18n.changeLanguage(language); + changeAppLanguage(language); }, [language]); - const isRtl = rtlLocales.includes(language as any); + const isRtl = isRightToLeft(language); return ( diff --git a/src/utils/language.ts b/src/utils/language.ts new file mode 100644 index 00000000..420c8d93 --- /dev/null +++ b/src/utils/language.ts @@ -0,0 +1,188 @@ +import countryLanguages from "@ladjs/country-language"; +import { getTag } from "@sozialhelden/ietf-language-tags"; + +const languageOrder = ["en", "hi", "fr", "de", "nl", "pt"]; + +// mapping of language code to country code. +// multiple mappings can exist, since languages are spoken in multiple countries. +// This mapping purely exists to prioritize a country over another in languages. +// iso639_1 -> iso3166 Alpha-2 +const countryPriority: Record = { + en: "gb", + nl: "nl", + fr: "fr", + de: "de", + pt: "pt", + ar: "sa", + es: "es", + zh: "cn", +}; + +// list of iso639_1 Alpha-2 codes used as default languages +const defaultLanguageCodes: string[] = [ + "en-US", + "cs-CZ", + "de-DE", + "fr-FR", + "pt-BR", + "it-IT", + "nl-NL", + "pl-PL", + "tr-TR", + "vi-VN", + "zh-CN", + "he-IL", + "sv-SE", + "lv-LV", + "th-TH", + "ne-NP", + "ar-SA", + "es-ES", + "et-EE", +]; + +export interface LocaleInfo { + name: string; + nativeName?: string; + code: string; + isRtl?: boolean; +} + +interface LanguageObj { + countries: Array<{ + code_2: string; + code_3: string; + numCode: string; + }>; + direction: "RTL" | "LTR"; + name: string[]; + nativeName: string[]; + iso639_1: string; +} + +const extraLanguages: Record = { + pirate: { + code: "pirate", + name: "Pirate", + nativeName: "Pirate Tongue", + }, + minion: { + code: "minion", + name: "Minion", + nativeName: "Minionese", + }, + tok: { + code: "tok", + name: "Toki pona", + nativeName: "Toki pona", + }, +}; + +function populateLanguageCode(language: string): string { + if (language.includes("-")) return language; + if (language.length !== 2) return language; + return ( + defaultLanguageCodes.find((v) => v.startsWith(`${language}-`)) ?? language + ); +} + +/** + * @param locale idk what kinda code this takes, anytihhng in ietf format I guess + * @returns pretty format for language, null if it no info can be found for language + */ +export function getPrettyLanguageNameFromLocale(locale: string): string | null { + const tag = getTag(populateLanguageCode(locale), true); + + const lang = tag?.language?.Description?.[0] ?? null; + if (!lang) return null; + + const region = tag?.region?.Description?.[0] ?? null; + let regionText = ""; + if (region) regionText = ` (${region})`; + + return `${lang}${regionText}`; +} + +/** + * Sort locale codes by occurance, rest on alphabetical order + * @param langCodes list language codes to sort + * @returns sorted version of inputted list + */ +export function sortLangCodes(langCodes: string[]) { + const languagesOrder = [...languageOrder].reverse(); // Reverse is neccesary, not sure why + + const results = langCodes.sort((a, b) => { + const langOrderA = languagesOrder.findIndex( + (v) => a.startsWith(`${v}-`) || a === v, + ); + const langOrderB = languagesOrder.findIndex( + (v) => b.startsWith(`${v}-`) || b === v, + ); + if (langOrderA !== -1 || langOrderB !== -1) return langOrderB - langOrderA; + + return a.localeCompare(b); + }); + + return results; +} + +/** + * Get country code for locale + * @param locale input locale + * @returns country code or null + */ +export function getCountryCodeForLocale(locale: string): string | null { + let output: LanguageObj | null = null as any as LanguageObj; + const tag = getTag(locale, true); + if (!tag?.language?.Subtag) return null; + // this function isnt async, so its garuanteed to work like this + countryLanguages.getLanguage( + tag.language.Subtag, + (_err: string, lang: LanguageObj) => { + if (lang) output = lang; + }, + ); + if (!output) return null; + if (output.countries.length === 0) return null; + const priority = countryPriority[output.iso639_1.toLowerCase()]; + if (priority) { + const priotizedCountry = output.countries.find( + (v) => v.code_2.toLowerCase() === priority, + ); + if (priotizedCountry) return priotizedCountry.code_2.toLowerCase(); + } + return output.countries[0].code_2.toLowerCase(); +} + +/** + * Get information for a specific local + * @param locale local code + * @returns locale object + */ +export function getLocaleInfo(locale: string): LocaleInfo | null { + const realLocale = populateLanguageCode(locale); + const extraLang = extraLanguages[realLocale]; + if (extraLang) return extraLang; + + const tag = getTag(realLocale, true); + if (!tag?.language?.Subtag) return null; + + let output: LanguageObj | null = null as any as LanguageObj; + // this function isnt async, so its garuanteed to work like this + countryLanguages.getLanguage( + tag.language.Subtag, + (_err: string, lang: LanguageObj) => { + if (lang) output = lang; + }, + ); + if (!output) return null; + + return { + code: tag.parts.langtag ?? realLocale, + isRtl: output.direction === "RTL", + name: + output.name[0] + + (tag.region?.Description ? ` (${tag.region.Description[0]})` : ""), + nativeName: output.nativeName[0] ?? undefined, + }; +} diff --git a/src/utils/sortLangCodes.ts b/src/utils/sortLangCodes.ts deleted file mode 100644 index 57999e92..00000000 --- a/src/utils/sortLangCodes.ts +++ /dev/null @@ -1,12 +0,0 @@ -export function sortLangCodes(langCodes: string[]) { - const languagesOrder = ["en", "hi", "fr", "de", "nl", "pt"].reverse(); // Reverse is neccesary, not sure why - - const results = langCodes.sort((a, b) => { - if (languagesOrder.indexOf(b) !== -1 || languagesOrder.indexOf(a) !== -1) - return languagesOrder.indexOf(b) - languagesOrder.indexOf(a); - - return a.localeCompare(b); - }); - - return results; -} diff --git a/vite.config.mts b/vite.config.mts index 3d18e359..c77a27c3 100644 --- a/vite.config.mts +++ b/vite.config.mts @@ -25,7 +25,7 @@ export default defineConfig(({ mode }) => { const env = loadEnv(mode, process.cwd()); return { plugins: [ - million.vite({ auto: true }), + million.vite({ auto: true, mute: true }), handlebars({ vars: { opensearchEnabled: env.VITE_OPENSEARCH_ENABLED === "true",