Merge pull request #698 from movie-web/locale-codes

Revamp locale codes

package.json

@@ -28,6 +28,7 @@
   "dependencies": {
     "@formkit/auto-animate": "^0.8.1",
     "@headlessui/react": "^1.7.17",
+    "@ladjs/country-language": "^1.0.3",
     "@movie-web/providers": "^2.0.2",
     "@noble/hashes": "^1.3.3",
     "@react-spring/web": "^9.7.3",
@@ -44,7 +45,6 @@
     "hls.js": "^1.4.14",
     "i18next": "^23.7.11",
     "immer": "^10.0.3",
-    "iso-639-1": "^3.1.0",
     "jwt-decode": "^4.0.0",
     "lodash.isequal": "^4.5.0",
     "million": "^2.6.4",

pnpm-lock.yaml

@@ -17,6 +17,9 @@ dependencies:
   '@headlessui/react':
     specifier: ^1.7.17
     version: 1.7.17(react-dom@18.2.0)(react@18.2.0)
+  '@ladjs/country-language':
+    specifier: ^1.0.3
+    version: 1.0.3
   '@movie-web/providers':
     specifier: ^2.0.2
     version: 2.0.3
@@ -65,9 +68,6 @@ dependencies:
   immer:
     specifier: ^10.0.3
     version: 10.0.3
-  iso-639-1:
-    specifier: ^3.1.0
-    version: 3.1.0
   jwt-decode:
     specifier: ^4.0.0
     version: 4.0.0
@@ -1912,6 +1912,11 @@ packages:
       '@jridgewell/resolve-uri': 3.1.1
       '@jridgewell/sourcemap-codec': 1.4.15
 
+  /@ladjs/country-language@1.0.3:
+    resolution: {integrity: sha512-FJROu9/hh4eqVAGDyfL8vpv6Vb0qKHX1ozYLRZ+beUzD5xFf+3r0J+SVIWKviEa7W524Qvqou+ta1WrsRgzxGw==}
+    engines: {node: '>= 14'}
+    dev: false
+
   /@movie-web/providers@2.0.3:
     resolution: {integrity: sha512-6UNk5EebiNjGoFTuyHuu0eZZTreRYv0cdsn52CVYjm6CXG63w4dMbx8ybxcvMUrDF3o8bWlqnlovG142sdOmNw==}
     dependencies:

src/@types/country-language.d.ts

@@ -0,0 +1,21 @@
+declare module "@ladjs/country-language" {
+  export interface LanguageObj {
+    countries: Array<{
+      code_2: string;
+      code_3: string;
+      numCode: string;
+    }>;
+    direction: "RTL" | "LTR";
+    name: string[];
+    nativeName: string[];
+    iso639_1: string;
+  }
+
+  type Callback<T> = (err: null | string, result: null | T) => void;
+
+  declare namespace lib {
+    function getLanguage(locale: string, cb: Callback<LanguageObj>): void;
+  }
+
+  export = lib;
+}

src/assets/README.md

@@ -0,0 +1,9 @@
+# About the languages
+
+Locales are difficult, here is some guidance.
+
+## Process on adding new languages
+1. Use weblate to add translations, see contributing guidelines.
+2. Add your language to `@/assets/languages.ts`. Must be in ISO format (ISO-639 for language and ISO-3166 for country/region). For joke languages, use any format.
+3. If your language doesn't have a region specified (Such as in `pt-BR`, `BR` being the region). Add a default region in `@/utils/language.ts` at `defaultLanguageCodes`
+4. If the flag in the language dropdown doesn't match the correct one. Add a default country in `@/utils/language.ts` at `countryPriority`.

src/assets/languages.ts

@@ -1,13 +1,21 @@
 import ar from "@/assets/locales/ar.json";
+import bg from "@/assets/locales/bg.json";
+import bn from "@/assets/locales/bn.json";
 import cs from "@/assets/locales/cs.json";
 import de from "@/assets/locales/de.json";
+import el from "@/assets/locales/el.json";
 import en from "@/assets/locales/en.json";
 import es from "@/assets/locales/es.json";
 import et from "@/assets/locales/et.json";
+import fa from "@/assets/locales/fa.json";
 import fr from "@/assets/locales/fr.json";
+import gu from "@/assets/locales/gu.json";
 import he from "@/assets/locales/he.json";
 import hi from "@/assets/locales/hi.json";
+import id from "@/assets/locales/id.json";
 import it from "@/assets/locales/it.json";
+import ja from "@/assets/locales/ja.json";
+import ko from "@/assets/locales/ko.json";
 import lv from "@/assets/locales/lv.json";
 import minion from "@/assets/locales/minion.json";
 import ne from "@/assets/locales/ne.json";
@@ -15,12 +23,16 @@ import nl from "@/assets/locales/nl.json";
 import pirate from "@/assets/locales/pirate.json";
 import pl from "@/assets/locales/pl.json";
 import ptbr from "@/assets/locales/pt-BR.json";
+import ru from "@/assets/locales/ru.json";
+import sl from "@/assets/locales/sl.json";
 import sv from "@/assets/locales/sv.json";
+import ta from "@/assets/locales/ta.json";
 import th from "@/assets/locales/th.json";
 import tok from "@/assets/locales/tok.json";
 import tr from "@/assets/locales/tr.json";
 import uk from "@/assets/locales/uk.json";
 import vi from "@/assets/locales/vi.json";
+import zhhant from "@/assets/locales/zh-Hant.json";
 import zh from "@/assets/locales/zh.json";
 
 export const locales = {
@@ -46,9 +58,19 @@ export const locales = {
   et,
   tok,
   hi,
-  pt: ptbr,
+  "pt-BR": ptbr,
   uk,
+  bg,
+  bn,
+  el,
+  fa,
+  gu,
+  id,
+  ja,
+  ko,
+  sl,
+  ta,
+  "zh-HANT": zhhant,
+  ru,
 };
 export type Locales = keyof typeof locales;
-
-export const rtlLocales: Locales[] = ["he", "ar"];

src/components/FlagIcon.tsx

@@ -1,53 +1,32 @@
 import classNames from "classnames";
+
+import { getCountryCodeForLocale } from "@/utils/language";
 import "flag-icons/css/flag-icons.min.css";
 
 export interface FlagIconProps {
-  countryCode?: string;
+  country?: string;
+  langCode?: string;
 }
 
-// Country code overrides
-const countryOverrides: Record<string, string> = {
-  en: "gb",
-  cs: "cz",
-  el: "gr",
-  fa: "ir",
-  ko: "kr",
-  he: "il",
-  ze: "cn",
-  ar: "sa",
-  ja: "jp",
-  bs: "ba",
-  vi: "vn",
-  zh: "cn",
-  sl: "si",
-  sv: "se",
-  et: "ee",
-  ne: "np",
-  uk: "ua",
-  hi: "in",
-};
-
 export function FlagIcon(props: FlagIconProps) {
-  let countryCode =
-    (props.countryCode || "")?.split("-").pop()?.toLowerCase() || "";
-  if (countryOverrides[countryCode])
-    countryCode = countryOverrides[countryCode];
+  let countryCode: string | null = props.country ?? null;
+  if (props.langCode) countryCode = getCountryCodeForLocale(props.langCode);
 
-  if (countryCode === "tok")
+  if (props.langCode === "tok")
     return (
       <div className="w-8 h-6 rounded bg-[#c8e1ed] flex justify-center items-center">
         <img src="/tokiPona.svg" className="w-7 h-5" />
       </div>
     );
 
-  if (countryCode === "pirate")
+  if (props.langCode === "pirate")
     return (
       <div className="w-8 h-6 rounded bg-[#2E3439] flex justify-center items-center">
         <img src="/skull.svg" className="w-4 h-4" />
       </div>
     );
 
-  if (countryCode === "minion")
+  if (props.langCode === "minion")
     return (
       <div className="w-8 h-6 rounded bg-[#ffff1a] flex justify-center items-center">
         <div className="w-4 h-4 border-2 border-gray-500 rounded-full bg-white flex justify-center items-center">
@@ -64,9 +43,9 @@ export function FlagIcon(props: FlagIconProps) {
   return (
     <span
       className={classNames(
-        "!w-8 h-6 rounded overflow-hidden bg-cover bg-center block fi",
+        "!w-8 min-w-8 h-6 rounded overflow-hidden bg-cover bg-center block fi",
         backgroundClass,
-        props.countryCode ? `fi-${countryCode}` : undefined,
+        countryCode ? `fi-${countryCode}` : undefined,
       )}
     />
   );

src/components/form/Dropdown.tsx

@@ -17,7 +17,7 @@ interface DropdownProps {
 
 export function Dropdown(props: DropdownProps) {
   return (
-    <div className="relative my-4 max-w-[18rem]">
+    <div className="relative my-4 max-w-[25rem]">
       <Listbox value={props.selectedItem} onChange={props.setSelectedItem}>
         {() => (
           <>

src/components/player/atoms/settings/CaptionsView.tsx

@@ -10,12 +10,14 @@ import { useCaptions } from "@/components/player/hooks/useCaptions";
 import { Menu } from "@/components/player/internals/ContextMenu";
 import { Input } from "@/components/player/internals/ContextMenu/Input";
 import { SelectableLink } from "@/components/player/internals/ContextMenu/Links";
-import { getLanguageFromIETF } from "@/components/player/utils/language";
 import { useOverlayRouter } from "@/hooks/useOverlayRouter";
 import { CaptionListItem } from "@/stores/player/slices/source";
 import { usePlayerStore } from "@/stores/player/store";
 import { useSubtitleStore } from "@/stores/subtitles";
-import { sortLangCodes } from "@/utils/sortLangCodes";
+import {
+  getPrettyLanguageNameFromLocale,
+  sortLangCodes,
+} from "@/utils/language";
 
 export function CaptionOption(props: {
   countryCode?: string;
@@ -37,7 +39,7 @@ export function CaptionOption(props: {
         className="flex items-center"
       >
         <span data-code={props.countryCode} className="mr-3 inline-flex">
-          <FlagIcon countryCode={props.countryCode} />
+          <FlagIcon langCode={props.countryCode} />
         </span>
         <span>{props.children}</span>
       </span>
@@ -89,7 +91,8 @@ function useSubtitleList(subs: CaptionListItem[], searchQuery: string) {
   return useMemo(() => {
     const input = subs.map((t) => ({
       ...t,
-      languageName: getLanguageFromIETF(t.language) ?? unknownChoice,
+      languageName:
+        getPrettyLanguageNameFromLocale(t.language) ?? unknownChoice,
     }));
     const sorted = sortLangCodes(input.map((t) => t.language));
     let results = input.sort((a, b) => {

src/components/player/atoms/settings/SettingsMenu.tsx

@@ -6,11 +6,11 @@ import { Toggle } from "@/components/buttons/Toggle";
 import { Icon, Icons } from "@/components/Icon";
 import { useCaptions } from "@/components/player/hooks/useCaptions";
 import { Menu } from "@/components/player/internals/ContextMenu";
-import { getLanguageFromIETF } from "@/components/player/utils/language";
 import { useOverlayRouter } from "@/hooks/useOverlayRouter";
 import { usePlayerStore } from "@/stores/player/store";
 import { qualityToString } from "@/stores/player/utils/qualities";
 import { useSubtitleStore } from "@/stores/subtitles";
+import { getPrettyLanguageNameFromLocale } from "@/utils/language";
 
 export function SettingsMenu({ id }: { id: string }) {
   const { t } = useTranslation();
@@ -31,7 +31,7 @@ export function SettingsMenu({ id }: { id: string }) {
   const { toggleLastUsed } = useCaptions();
 
   const selectedLanguagePretty = selectedCaptionLanguage
-    ? getLanguageFromIETF(selectedCaptionLanguage) ??
+    ? getPrettyLanguageNameFromLocale(selectedCaptionLanguage) ??
       t("player.menus.subtitles.unknownLanguage")
     : undefined;
 

src/components/player/utils/language.ts

@@ -1,14 +0,0 @@
-import { getTag } from "@sozialhelden/ietf-language-tags";
-
-export function getLanguageFromIETF(ietf: string): string | null {
-  const tag = getTag(ietf, true);
-
-  const lang = tag?.language?.Description?.[0] ?? null;
-  if (!lang) return null;
-
-  const region = tag?.region?.Description?.[0] ?? null;
-  let regionText = "";
-  if (region) regionText = ` (${region})`;
-
-  return `${lang}${regionText}`;
-}

src/index.tsx

@@ -23,10 +23,9 @@ import { MigrationPart } from "@/pages/parts/migrations/MigrationPart";
 import { LargeTextPart } from "@/pages/parts/util/LargeTextPart";
 import App from "@/setup/App";
 import { conf } from "@/setup/config";
-import i18n from "@/setup/i18n";
 import { useAuthStore } from "@/stores/auth";
 import { BookmarkSyncer } from "@/stores/bookmarks/BookmarkSyncer";
-import { useLanguageStore } from "@/stores/language";
+import { changeAppLanguage, useLanguageStore } from "@/stores/language";
 import { ProgressSyncer } from "@/stores/progress/ProgressSyncer";
 import { SettingsSyncer } from "@/stores/subtitles/SettingsSyncer";
 import { ThemeProvider } from "@/stores/theme";
@@ -123,7 +122,7 @@ function AuthWrapper() {
 
 function MigrationRunner() {
   const status = useAsync(async () => {
-    i18n.changeLanguage(useLanguageStore.getState().language);
+    changeAppLanguage(useLanguageStore.getState().language);
     await initializeOldStores();
   }, []);
   const { t } = useTranslation();

src/pages/parts/settings/LocalePart.tsx

@@ -4,7 +4,7 @@ import { FlagIcon } from "@/components/FlagIcon";
 import { Dropdown } from "@/components/form/Dropdown";
 import { Heading1 } from "@/components/utils/Text";
 import { appLanguageOptions } from "@/setup/i18n";
-import { sortLangCodes } from "@/utils/sortLangCodes";
+import { getLocaleInfo, sortLangCodes } from "@/utils/language";
 
 export function LocalePart(props: {
   language: string;
@@ -17,11 +17,13 @@ export function LocalePart(props: {
     .sort((a, b) => sorted.indexOf(a.code) - sorted.indexOf(b.code))
     .map((opt) => ({
       id: opt.code,
-      name: `${opt.name} — ${opt.nativeName}`,
-      leftIcon: <FlagIcon countryCode={opt.code} />,
+      name: `${opt.name}${opt.nativeName ? ` — ${opt.nativeName}` : ""}`,
+      leftIcon: <FlagIcon langCode={opt.code} />,
     }));
 
-  const selected = options.find((item) => item.id === props.language);
+  const selected = options.find(
+    (item) => item.id === getLocaleInfo(props.language)?.code,
+  );
 
   return (
     <div>

src/setup/i18n.ts

@@ -1,8 +1,8 @@
 import i18n from "i18next";
-import ISO6391 from "iso-639-1";
 import { initReactI18next } from "react-i18next";
 
 import { locales } from "@/assets/languages";
+import { getLocaleInfo } from "@/utils/language";
 
 // Languages
 const langCodes = Object.keys(locales);
@@ -10,43 +10,15 @@ const resources = Object.fromEntries(
   Object.entries(locales).map((entry) => [entry[0], { translation: entry[1] }]),
 );
 i18n.use(initReactI18next).init({
-  fallbackLng: "en",
+  fallbackLng: "en-US",
   resources,
   interpolation: {
     escapeValue: false, // not needed for react as it escapes by default
   },
 });
 
-const extraLanguages: Record<
-  string,
-  {
-    code: string;
-    name: string;
-    nativeName: string;
-  }
-> = {
-  pirate: {
-    code: "pirate",
-    name: "Pirate",
-    nativeName: "Pirate Tongue",
-  },
-  minion: {
-    code: "minion",
-    name: "Minion",
-    nativeName: "Minionese",
-  },
-  tok: {
-    code: "tok",
-    name: "Toki pona",
-    nativeName: "Toki pona",
-  },
-};
-
 export const appLanguageOptions = langCodes.map((lang) => {
-  const extraLang = extraLanguages[lang];
-  if (extraLang) return extraLang;
-
-  const [langObj] = ISO6391.getLanguages([lang]);
+  const langObj = getLocaleInfo(lang);
   if (!langObj)
     throw new Error(`Language with code ${lang} cannot be found in database`);
   return langObj;

src/stores/language/index.tsx

@@ -4,8 +4,8 @@ import { create } from "zustand";
 import { persist } from "zustand/middleware";
 import { immer } from "zustand/middleware/immer";
 
-import { rtlLocales } from "@/assets/languages";
 import i18n from "@/setup/i18n";
+import { getLocaleInfo } from "@/utils/language";
 
 export interface LanguageStore {
   language: string;
@@ -26,14 +26,25 @@ export const useLanguageStore = create(
   ),
 );
 
+export function changeAppLanguage(language: string) {
+  const lang = getLocaleInfo(language);
+  if (lang) i18n.changeLanguage(lang.code);
+}
+
+export function isRightToLeft(language: string) {
+  const lang = getLocaleInfo(language);
+  if (!lang) return false;
+  return lang.isRtl;
+}
+
 export function LanguageProvider() {
   const language = useLanguageStore((s) => s.language);
 
   useEffect(() => {
-    i18n.changeLanguage(language);
+    changeAppLanguage(language);
   }, [language]);
 
-  const isRtl = rtlLocales.includes(language as any);
+  const isRtl = isRightToLeft(language);
 
   return (
     <Helmet>

src/utils/language.ts

@@ -0,0 +1,185 @@
+import countryLanguages, { LanguageObj } from "@ladjs/country-language";
+import { getTag } from "@sozialhelden/ietf-language-tags";
+
+const languageOrder = ["en", "hi", "fr", "de", "nl", "pt"];
+
+// mapping of language code to country code.
+// multiple mappings can exist, since languages are spoken in multiple countries.
+// This mapping purely exists to prioritize a country over another in languages.
+// iso639_1 -> iso3166 Alpha-2
+const countryPriority: Record<string, string> = {
+  en: "gb",
+  nl: "nl",
+  fr: "fr",
+  de: "de",
+  pt: "pt",
+  ar: "sa",
+  es: "es",
+  zh: "cn",
+  ko: "kr",
+  ta: "lk",
+};
+
+// list of iso639_1 Alpha-2 codes used as default languages
+const defaultLanguageCodes: string[] = [
+  "en-US",
+  "cs-CZ",
+  "de-DE",
+  "fr-FR",
+  "pt-BR",
+  "it-IT",
+  "nl-NL",
+  "pl-PL",
+  "tr-TR",
+  "vi-VN",
+  "zh-CN",
+  "he-IL",
+  "sv-SE",
+  "lv-LV",
+  "th-TH",
+  "ne-NP",
+  "ar-SA",
+  "es-ES",
+  "et-EE",
+  "bg-BG",
+  "bn-BD",
+  "el-GR",
+  "fa-IR",
+  "gu-IN",
+  "id-ID",
+  "ja-JP",
+  "ko-KR",
+  "sl-SI",
+  "ta-LK",
+  "ru-RU",
+];
+
+export interface LocaleInfo {
+  name: string;
+  nativeName?: string;
+  code: string;
+  isRtl?: boolean;
+}
+
+const extraLanguages: Record<string, LocaleInfo> = {
+  pirate: {
+    code: "pirate",
+    name: "Pirate",
+    nativeName: "Pirate Tongue",
+  },
+  minion: {
+    code: "minion",
+    name: "Minion",
+    nativeName: "Minionese",
+  },
+  tok: {
+    code: "tok",
+    name: "Toki pona",
+    nativeName: "Toki pona",
+  },
+};
+
+function populateLanguageCode(language: string): string {
+  if (language.includes("-")) return language;
+  if (language.length !== 2) return language;
+  return (
+    defaultLanguageCodes.find((v) => v.startsWith(`${language}-`)) ?? language
+  );
+}
+
+/**
+ * @param locale idk what kinda code this takes, anytihhng in ietf format I guess
+ * @returns pretty format for language, null if it no info can be found for language
+ */
+export function getPrettyLanguageNameFromLocale(locale: string): string | null {
+  const tag = getTag(populateLanguageCode(locale), true);
+  const lang = tag?.language?.Description?.[0] ?? null;
+  if (!lang) return null;
+
+  const region = tag?.region?.Description?.[0] ?? null;
+  let regionText = "";
+  if (region) regionText = ` (${region})`;
+
+  return `${lang}${regionText}`;
+}
+
+/**
+ * Sort locale codes by occurance, rest on alphabetical order
+ * @param langCodes list language codes to sort
+ * @returns sorted version of inputted list
+ */
+export function sortLangCodes(langCodes: string[]) {
+  const languagesOrder = [...languageOrder].reverse(); // Reverse is neccesary, not sure why
+
+  const results = langCodes.sort((a, b) => {
+    const langOrderA = languagesOrder.findIndex(
+      (v) => a.startsWith(`${v}-`) || a === v,
+    );
+    const langOrderB = languagesOrder.findIndex(
+      (v) => b.startsWith(`${v}-`) || b === v,
+    );
+    if (langOrderA !== -1 || langOrderB !== -1) return langOrderB - langOrderA;
+
+    return a.localeCompare(b);
+  });
+
+  return results;
+}
+
+/**
+ * Get country code for locale
+ * @param locale input locale
+ * @returns country code or null
+ */
+export function getCountryCodeForLocale(locale: string): string | null {
+  let output: LanguageObj | null = null as any as LanguageObj;
+  const tag = getTag(locale, true);
+  if (!tag?.language?.Subtag) return null;
+  // this function isnt async, so its garuanteed to work like this
+  countryLanguages.getLanguage(tag.language.Subtag, (_err, lang) => {
+    if (lang) output = lang;
+  });
+  if (!output) return null;
+  if (output.countries.length === 0) return null;
+  const priority = countryPriority[output.iso639_1.toLowerCase()];
+  if (priority) {
+    const priotizedCountry = output.countries.find(
+      (v) => v.code_2.toLowerCase() === priority,
+    );
+    if (priotizedCountry) return priotizedCountry.code_2.toLowerCase();
+  }
+  return output.countries[0].code_2.toLowerCase();
+}
+
+/**
+ * Get information for a specific local
+ * @param locale local code
+ * @returns locale object
+ */
+export function getLocaleInfo(locale: string): LocaleInfo | null {
+  const realLocale = populateLanguageCode(locale);
+  const extraLang = extraLanguages[realLocale];
+  if (extraLang) return extraLang;
+
+  const tag = getTag(realLocale, true);
+  if (!tag?.language?.Subtag) return null;
+
+  let output: LanguageObj | null = null as any as LanguageObj;
+  // this function isnt async, so its garuanteed to work like this
+  countryLanguages.getLanguage(tag.language.Subtag, (_err, lang) => {
+    if (lang) output = lang;
+  });
+  if (!output) return null;
+
+  const extras = [];
+  if (tag.region?.Description) extras.push(tag.region.Description[0]);
+  if (tag.script?.Description) extras.push(tag.script.Description[0]);
+  const extraStringified = extras.map((v) => `(${v})`).join(" ");
+
+  return {
+    code: tag.parts.langtag ?? realLocale,
+    isRtl: output.direction === "RTL",
+    name: output.name[0] + (extraStringified ? ` ${extraStringified}` : ""),
+    nativeName: output.nativeName[0] ?? undefined,
+  };
+}

src/utils/sortLangCodes.ts

@@ -1,12 +0,0 @@
-export function sortLangCodes(langCodes: string[]) {
-  const languagesOrder = ["en", "hi", "fr", "de", "nl", "pt"].reverse(); // Reverse is neccesary, not sure why
-
-  const results = langCodes.sort((a, b) => {
-    if (languagesOrder.indexOf(b) !== -1 || languagesOrder.indexOf(a) !== -1)
-      return languagesOrder.indexOf(b) - languagesOrder.indexOf(a);
-
-    return a.localeCompare(b);
-  });
-
-  return results;
-}

vite.config.mts

@@ -25,7 +25,7 @@ export default defineConfig(({ mode }) => {
   const env = loadEnv(mode, process.cwd());
   return {
     plugins: [
-      million.vite({ auto: true }),
+      million.vite({ auto: true, mute: true }),
       handlebars({
         vars: {
           opensearchEnabled: env.VITE_OPENSEARCH_ENABLED === "true",
@@ -124,8 +124,8 @@ export default defineConfig(({ mode }) => {
       rollupOptions: {
         output: {
           manualChunks(id: string) {
-            if (id.includes("@sozialhelden+ietf-language-tags")) {
-              return "ietf-language-tags";
+            if (id.includes("@sozialhelden+ietf-language-tags") || id.includes("country-language")) {
+              return "language-db";
             }
             if (id.includes("hls.js")) {
               return "hls";