Implement new country code system and new language code system

This commit is contained in:
mrjvs 2024-01-03 20:06:08 +01:00
parent bc5e2d6f30
commit aca7827a15
15 changed files with 243 additions and 112 deletions

View File

@ -28,6 +28,7 @@
"dependencies": {
"@formkit/auto-animate": "^0.8.1",
"@headlessui/react": "^1.7.17",
"@ladjs/country-language": "^1.0.3",
"@movie-web/providers": "^2.0.2",
"@noble/hashes": "^1.3.3",
"@react-spring/web": "^9.7.3",
@ -44,7 +45,6 @@
"hls.js": "^1.4.14",
"i18next": "^23.7.11",
"immer": "^10.0.3",
"iso-639-1": "^3.1.0",
"jwt-decode": "^4.0.0",
"lodash.isequal": "^4.5.0",
"million": "^2.6.4",

11
pnpm-lock.yaml generated
View File

@ -17,6 +17,9 @@ dependencies:
'@headlessui/react':
specifier: ^1.7.17
version: 1.7.17(react-dom@18.2.0)(react@18.2.0)
'@ladjs/country-language':
specifier: ^1.0.3
version: 1.0.3
'@movie-web/providers':
specifier: ^2.0.2
version: 2.0.3
@ -65,9 +68,6 @@ dependencies:
immer:
specifier: ^10.0.3
version: 10.0.3
iso-639-1:
specifier: ^3.1.0
version: 3.1.0
jwt-decode:
specifier: ^4.0.0
version: 4.0.0
@ -1912,6 +1912,11 @@ packages:
'@jridgewell/resolve-uri': 3.1.1
'@jridgewell/sourcemap-codec': 1.4.15
/@ladjs/country-language@1.0.3:
resolution: {integrity: sha512-FJROu9/hh4eqVAGDyfL8vpv6Vb0qKHX1ozYLRZ+beUzD5xFf+3r0J+SVIWKviEa7W524Qvqou+ta1WrsRgzxGw==}
engines: {node: '>= 14'}
dev: false
/@movie-web/providers@2.0.3:
resolution: {integrity: sha512-6UNk5EebiNjGoFTuyHuu0eZZTreRYv0cdsn52CVYjm6CXG63w4dMbx8ybxcvMUrDF3o8bWlqnlovG142sdOmNw==}
dependencies:

View File

@ -50,5 +50,3 @@ export const locales = {
uk,
};
export type Locales = keyof typeof locales;
export const rtlLocales: Locales[] = ["he", "ar"];

View File

@ -1,53 +1,32 @@
import classNames from "classnames";
import { getCountryCodeForLocale } from "@/utils/language";
import "flag-icons/css/flag-icons.min.css";
export interface FlagIconProps {
countryCode?: string;
country?: string;
langCode?: string;
}
// Country code overrides
const countryOverrides: Record<string, string> = {
en: "gb",
cs: "cz",
el: "gr",
fa: "ir",
ko: "kr",
he: "il",
ze: "cn",
ar: "sa",
ja: "jp",
bs: "ba",
vi: "vn",
zh: "cn",
sl: "si",
sv: "se",
et: "ee",
ne: "np",
uk: "ua",
hi: "in",
};
export function FlagIcon(props: FlagIconProps) {
let countryCode =
(props.countryCode || "")?.split("-").pop()?.toLowerCase() || "";
if (countryOverrides[countryCode])
countryCode = countryOverrides[countryCode];
let countryCode: string | null = props.country ?? null;
if (props.langCode) countryCode = getCountryCodeForLocale(props.langCode);
if (countryCode === "tok")
if (props.langCode === "tok")
return (
<div className="w-8 h-6 rounded bg-[#c8e1ed] flex justify-center items-center">
<img src="/tokiPona.svg" className="w-7 h-5" />
</div>
);
if (countryCode === "pirate")
if (props.langCode === "pirate")
return (
<div className="w-8 h-6 rounded bg-[#2E3439] flex justify-center items-center">
<img src="/skull.svg" className="w-4 h-4" />
</div>
);
if (countryCode === "minion")
if (props.langCode === "minion")
return (
<div className="w-8 h-6 rounded bg-[#ffff1a] flex justify-center items-center">
<div className="w-4 h-4 border-2 border-gray-500 rounded-full bg-white flex justify-center items-center">
@ -66,7 +45,7 @@ export function FlagIcon(props: FlagIconProps) {
className={classNames(
"!w-8 h-6 rounded overflow-hidden bg-cover bg-center block fi",
backgroundClass,
props.countryCode ? `fi-${countryCode}` : undefined,
countryCode ? `fi-${countryCode}` : undefined,
)}
/>
);

View File

@ -17,7 +17,7 @@ interface DropdownProps {
export function Dropdown(props: DropdownProps) {
return (
<div className="relative my-4 max-w-[18rem]">
<div className="relative my-4 max-w-[25rem]">
<Listbox value={props.selectedItem} onChange={props.setSelectedItem}>
{() => (
<>

View File

@ -10,12 +10,14 @@ import { useCaptions } from "@/components/player/hooks/useCaptions";
import { Menu } from "@/components/player/internals/ContextMenu";
import { Input } from "@/components/player/internals/ContextMenu/Input";
import { SelectableLink } from "@/components/player/internals/ContextMenu/Links";
import { getLanguageFromIETF } from "@/components/player/utils/language";
import { useOverlayRouter } from "@/hooks/useOverlayRouter";
import { CaptionListItem } from "@/stores/player/slices/source";
import { usePlayerStore } from "@/stores/player/store";
import { useSubtitleStore } from "@/stores/subtitles";
import { sortLangCodes } from "@/utils/sortLangCodes";
import {
getPrettyLanguageNameFromLocale,
sortLangCodes,
} from "@/utils/language";
export function CaptionOption(props: {
countryCode?: string;
@ -37,7 +39,7 @@ export function CaptionOption(props: {
className="flex items-center"
>
<span data-code={props.countryCode} className="mr-3 inline-flex">
<FlagIcon countryCode={props.countryCode} />
<FlagIcon langCode={props.countryCode} />
</span>
<span>{props.children}</span>
</span>
@ -89,7 +91,8 @@ function useSubtitleList(subs: CaptionListItem[], searchQuery: string) {
return useMemo(() => {
const input = subs.map((t) => ({
...t,
languageName: getLanguageFromIETF(t.language) ?? unknownChoice,
languageName:
getPrettyLanguageNameFromLocale(t.language) ?? unknownChoice,
}));
const sorted = sortLangCodes(input.map((t) => t.language));
let results = input.sort((a, b) => {

View File

@ -6,11 +6,11 @@ import { Toggle } from "@/components/buttons/Toggle";
import { Icon, Icons } from "@/components/Icon";
import { useCaptions } from "@/components/player/hooks/useCaptions";
import { Menu } from "@/components/player/internals/ContextMenu";
import { getLanguageFromIETF } from "@/components/player/utils/language";
import { useOverlayRouter } from "@/hooks/useOverlayRouter";
import { usePlayerStore } from "@/stores/player/store";
import { qualityToString } from "@/stores/player/utils/qualities";
import { useSubtitleStore } from "@/stores/subtitles";
import { getPrettyLanguageNameFromLocale } from "@/utils/language";
export function SettingsMenu({ id }: { id: string }) {
const { t } = useTranslation();
@ -31,7 +31,7 @@ export function SettingsMenu({ id }: { id: string }) {
const { toggleLastUsed } = useCaptions();
const selectedLanguagePretty = selectedCaptionLanguage
? getLanguageFromIETF(selectedCaptionLanguage) ??
? getPrettyLanguageNameFromLocale(selectedCaptionLanguage) ??
t("player.menus.subtitles.unknownLanguage")
: undefined;

View File

@ -1,14 +0,0 @@
import { getTag } from "@sozialhelden/ietf-language-tags";
export function getLanguageFromIETF(ietf: string): string | null {
const tag = getTag(ietf, true);
const lang = tag?.language?.Description?.[0] ?? null;
if (!lang) return null;
const region = tag?.region?.Description?.[0] ?? null;
let regionText = "";
if (region) regionText = ` (${region})`;
return `${lang}${regionText}`;
}

View File

@ -23,10 +23,9 @@ import { MigrationPart } from "@/pages/parts/migrations/MigrationPart";
import { LargeTextPart } from "@/pages/parts/util/LargeTextPart";
import App from "@/setup/App";
import { conf } from "@/setup/config";
import i18n from "@/setup/i18n";
import { useAuthStore } from "@/stores/auth";
import { BookmarkSyncer } from "@/stores/bookmarks/BookmarkSyncer";
import { useLanguageStore } from "@/stores/language";
import { changeAppLanguage, useLanguageStore } from "@/stores/language";
import { ProgressSyncer } from "@/stores/progress/ProgressSyncer";
import { SettingsSyncer } from "@/stores/subtitles/SettingsSyncer";
import { ThemeProvider } from "@/stores/theme";
@ -123,7 +122,7 @@ function AuthWrapper() {
function MigrationRunner() {
const status = useAsync(async () => {
i18n.changeLanguage(useLanguageStore.getState().language);
changeAppLanguage(useLanguageStore.getState().language);
await initializeOldStores();
}, []);
const { t } = useTranslation();

View File

@ -4,7 +4,7 @@ import { FlagIcon } from "@/components/FlagIcon";
import { Dropdown } from "@/components/form/Dropdown";
import { Heading1 } from "@/components/utils/Text";
import { appLanguageOptions } from "@/setup/i18n";
import { sortLangCodes } from "@/utils/sortLangCodes";
import { getLocaleInfo, sortLangCodes } from "@/utils/language";
export function LocalePart(props: {
language: string;
@ -17,11 +17,13 @@ export function LocalePart(props: {
.sort((a, b) => sorted.indexOf(a.code) - sorted.indexOf(b.code))
.map((opt) => ({
id: opt.code,
name: `${opt.name}${opt.nativeName}`,
leftIcon: <FlagIcon countryCode={opt.code} />,
name: `${opt.name}${opt.nativeName ? `${opt.nativeName}` : ""}`,
leftIcon: <FlagIcon langCode={opt.code} />,
}));
const selected = options.find((item) => item.id === props.language);
const selected = options.find(
(item) => item.id === getLocaleInfo(props.language)?.code,
);
return (
<div>

View File

@ -1,8 +1,8 @@
import i18n from "i18next";
import ISO6391 from "iso-639-1";
import { initReactI18next } from "react-i18next";
import { locales } from "@/assets/languages";
import { getLocaleInfo } from "@/utils/language";
// Languages
const langCodes = Object.keys(locales);
@ -10,43 +10,15 @@ const resources = Object.fromEntries(
Object.entries(locales).map((entry) => [entry[0], { translation: entry[1] }]),
);
i18n.use(initReactI18next).init({
fallbackLng: "en",
fallbackLng: "en-US",
resources,
interpolation: {
escapeValue: false, // not needed for react as it escapes by default
},
});
const extraLanguages: Record<
string,
{
code: string;
name: string;
nativeName: string;
}
> = {
pirate: {
code: "pirate",
name: "Pirate",
nativeName: "Pirate Tongue",
},
minion: {
code: "minion",
name: "Minion",
nativeName: "Minionese",
},
tok: {
code: "tok",
name: "Toki pona",
nativeName: "Toki pona",
},
};
export const appLanguageOptions = langCodes.map((lang) => {
const extraLang = extraLanguages[lang];
if (extraLang) return extraLang;
const [langObj] = ISO6391.getLanguages([lang]);
const langObj = getLocaleInfo(lang);
if (!langObj)
throw new Error(`Language with code ${lang} cannot be found in database`);
return langObj;

View File

@ -4,8 +4,8 @@ import { create } from "zustand";
import { persist } from "zustand/middleware";
import { immer } from "zustand/middleware/immer";
import { rtlLocales } from "@/assets/languages";
import i18n from "@/setup/i18n";
import { getLocaleInfo } from "@/utils/language";
export interface LanguageStore {
language: string;
@ -26,14 +26,25 @@ export const useLanguageStore = create(
),
);
export function changeAppLanguage(language: string) {
const lang = getLocaleInfo(language);
if (lang) i18n.changeLanguage(lang.code);
}
export function isRightToLeft(language: string) {
const lang = getLocaleInfo(language);
if (!lang) return false;
return lang.isRtl;
}
export function LanguageProvider() {
const language = useLanguageStore((s) => s.language);
useEffect(() => {
i18n.changeLanguage(language);
changeAppLanguage(language);
}, [language]);
const isRtl = rtlLocales.includes(language as any);
const isRtl = isRightToLeft(language);
return (
<Helmet>

188
src/utils/language.ts Normal file
View File

@ -0,0 +1,188 @@
import countryLanguages from "@ladjs/country-language";
import { getTag } from "@sozialhelden/ietf-language-tags";
const languageOrder = ["en", "hi", "fr", "de", "nl", "pt"];
// mapping of language code to country code.
// multiple mappings can exist, since languages are spoken in multiple countries.
// This mapping purely exists to prioritize a country over another in languages.
// iso639_1 -> iso3166 Alpha-2
const countryPriority: Record<string, string> = {
en: "gb",
nl: "nl",
fr: "fr",
de: "de",
pt: "pt",
ar: "sa",
es: "es",
zh: "cn",
};
// list of iso639_1 Alpha-2 codes used as default languages
const defaultLanguageCodes: string[] = [
"en-US",
"cs-CZ",
"de-DE",
"fr-FR",
"pt-BR",
"it-IT",
"nl-NL",
"pl-PL",
"tr-TR",
"vi-VN",
"zh-CN",
"he-IL",
"sv-SE",
"lv-LV",
"th-TH",
"ne-NP",
"ar-SA",
"es-ES",
"et-EE",
];
export interface LocaleInfo {
name: string;
nativeName?: string;
code: string;
isRtl?: boolean;
}
interface LanguageObj {
countries: Array<{
code_2: string;
code_3: string;
numCode: string;
}>;
direction: "RTL" | "LTR";
name: string[];
nativeName: string[];
iso639_1: string;
}
const extraLanguages: Record<string, LocaleInfo> = {
pirate: {
code: "pirate",
name: "Pirate",
nativeName: "Pirate Tongue",
},
minion: {
code: "minion",
name: "Minion",
nativeName: "Minionese",
},
tok: {
code: "tok",
name: "Toki pona",
nativeName: "Toki pona",
},
};
function populateLanguageCode(language: string): string {
if (language.includes("-")) return language;
if (language.length !== 2) return language;
return (
defaultLanguageCodes.find((v) => v.startsWith(`${language}-`)) ?? language
);
}
/**
* @param locale idk what kinda code this takes, anytihhng in ietf format I guess
* @returns pretty format for language, null if it no info can be found for language
*/
export function getPrettyLanguageNameFromLocale(locale: string): string | null {
const tag = getTag(populateLanguageCode(locale), true);
const lang = tag?.language?.Description?.[0] ?? null;
if (!lang) return null;
const region = tag?.region?.Description?.[0] ?? null;
let regionText = "";
if (region) regionText = ` (${region})`;
return `${lang}${regionText}`;
}
/**
* Sort locale codes by occurance, rest on alphabetical order
* @param langCodes list language codes to sort
* @returns sorted version of inputted list
*/
export function sortLangCodes(langCodes: string[]) {
const languagesOrder = [...languageOrder].reverse(); // Reverse is neccesary, not sure why
const results = langCodes.sort((a, b) => {
const langOrderA = languagesOrder.findIndex(
(v) => a.startsWith(`${v}-`) || a === v,
);
const langOrderB = languagesOrder.findIndex(
(v) => b.startsWith(`${v}-`) || b === v,
);
if (langOrderA !== -1 || langOrderB !== -1) return langOrderB - langOrderA;
return a.localeCompare(b);
});
return results;
}
/**
* Get country code for locale
* @param locale input locale
* @returns country code or null
*/
export function getCountryCodeForLocale(locale: string): string | null {
let output: LanguageObj | null = null as any as LanguageObj;
const tag = getTag(locale, true);
if (!tag?.language?.Subtag) return null;
// this function isnt async, so its garuanteed to work like this
countryLanguages.getLanguage(
tag.language.Subtag,
(_err: string, lang: LanguageObj) => {
if (lang) output = lang;
},
);
if (!output) return null;
if (output.countries.length === 0) return null;
const priority = countryPriority[output.iso639_1.toLowerCase()];
if (priority) {
const priotizedCountry = output.countries.find(
(v) => v.code_2.toLowerCase() === priority,
);
if (priotizedCountry) return priotizedCountry.code_2.toLowerCase();
}
return output.countries[0].code_2.toLowerCase();
}
/**
* Get information for a specific local
* @param locale local code
* @returns locale object
*/
export function getLocaleInfo(locale: string): LocaleInfo | null {
const realLocale = populateLanguageCode(locale);
const extraLang = extraLanguages[realLocale];
if (extraLang) return extraLang;
const tag = getTag(realLocale, true);
if (!tag?.language?.Subtag) return null;
let output: LanguageObj | null = null as any as LanguageObj;
// this function isnt async, so its garuanteed to work like this
countryLanguages.getLanguage(
tag.language.Subtag,
(_err: string, lang: LanguageObj) => {
if (lang) output = lang;
},
);
if (!output) return null;
return {
code: tag.parts.langtag ?? realLocale,
isRtl: output.direction === "RTL",
name:
output.name[0] +
(tag.region?.Description ? ` (${tag.region.Description[0]})` : ""),
nativeName: output.nativeName[0] ?? undefined,
};
}

View File

@ -1,12 +0,0 @@
export function sortLangCodes(langCodes: string[]) {
const languagesOrder = ["en", "hi", "fr", "de", "nl", "pt"].reverse(); // Reverse is neccesary, not sure why
const results = langCodes.sort((a, b) => {
if (languagesOrder.indexOf(b) !== -1 || languagesOrder.indexOf(a) !== -1)
return languagesOrder.indexOf(b) - languagesOrder.indexOf(a);
return a.localeCompare(b);
});
return results;
}

View File

@ -25,7 +25,7 @@ export default defineConfig(({ mode }) => {
const env = loadEnv(mode, process.cwd());
return {
plugins: [
million.vite({ auto: true }),
million.vite({ auto: true, mute: true }),
handlebars({
vars: {
opensearchEnabled: env.VITE_OPENSEARCH_ENABLED === "true",