Update custom translations to support nested fields in structured JSON (#11685)

* Update matrix-web-i18n Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix custom translations for structured JSON nested fields Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix import Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix export Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Update @matrix-org/react-sdk-module-api Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Delint Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Update matrix-web-i18n Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Update matrix-web-i18n Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

Author: t3chguy

package.json

@@ -63,7 +63,7 @@
  "@matrix-org/analytics-events": "^0.7.0",
  "@matrix-org/emojibase-bindings": "^1.1.2",
  "@matrix-org/matrix-wysiwyg": "^2.4.1",
- "@matrix-org/react-sdk-module-api": "^2.1.0",
+ "@matrix-org/react-sdk-module-api": "^2.1.1",
  "@matrix-org/spec": "^1.7.0",
  "@sentry/browser": "^7.0.0",
  "@sentry/tracing": "^7.0.0",
@@ -214,7 +214,7 @@
  "jsqr": "^1.4.0",
  "mailhog": "^4.16.0",
  "matrix-mock-request": "^2.5.0",
- "matrix-web-i18n": "^2.1.0",
+ "matrix-web-i18n": "^3.1.3",
  "mocha-junit-reporter": "^2.2.0",
  "node-fetch": "2",
  "postcss-scss": "^4.0.4",

src/@types/common.ts

@@ -23,40 +23,7 @@ export type Writeable<T> = { -readonly [P in keyof T]: T[P] };
 
 export type ComponentClass = keyof JSX.IntrinsicElements | JSXElementConstructor<any>;
 
-/**
- * Utility type for string dot notation for accessing nested object properties.
- * Based on https://stackoverflow.com/a/58436959
- * @example
- * {
- * "a": {
- * "b": {
- * "c": "value"
- * },
- * "d": "foobar"
- * }
- * }
- * will yield a type of `"a.b.c" | "a.d"` with Separator="."
- * @typeParam Target the target type to generate leaf keys for
- * @typeParam Separator the separator to use between key segments when accessing nested objects
- * @typeParam LeafType the type which leaves of this object extend, used to determine when to stop recursion
- * @typeParam MaxDepth the maximum depth to recurse to
- * @returns a union type representing all dot (Separator) string notation keys which can access a Leaf (of LeafType)
- */
-export type Leaves<Target, Separator extends string = ".", LeafType = string, MaxDepth extends number = 3> = [
- MaxDepth,
-] extends [never]
- ? never
- : Target extends LeafType
- ? ""
- : {
- [K in keyof Target]-?: Join<K, Leaves<Target[K], Separator, LeafType, Prev[MaxDepth]>, Separator>;
- }[keyof Target];
-type Prev = [never, 0, 1, 2, 3, ...0[]];
-type Join<K, P, S extends string = "."> = K extends string | number
- ? P extends string | number
- ? `${K}${"" extends P ? "" : S}${P}`
- : never
- : never;
+export type { Leaves } from "matrix-web-i18n";
 
 export type RecursivePartial<T> = {
  [P in keyof T]?: T[P] extends (infer U)[]

src/languageHandler.tsx

@@ -21,7 +21,10 @@ import counterpart from "counterpart";
 import React from "react";
 import { logger } from "matrix-js-sdk/src/logger";
 import { Optional } from "matrix-events-sdk";
-import { MapWithDefault, safeSet } from "matrix-js-sdk/src/utils";
+import { MapWithDefault } from "matrix-js-sdk/src/utils";
+import { normalizeLanguageKey, TranslationKey as _TranslationKey, KEY_SEPARATOR } from "matrix-web-i18n";
+import { TranslationStringsObject } from "@matrix-org/react-sdk-module-api";
+import _ from "lodash";
 
 import type Translations from "./i18n/strings/en_EN.json";
 import SettingsStore from "./settings/SettingsStore";
@@ -30,19 +33,20 @@ import { SettingLevel } from "./settings/SettingLevel";
 import { retry } from "./utils/promise";
 import SdkConfig from "./SdkConfig";
 import { ModuleRunner } from "./modules/ModuleRunner";
-import { Leaves } from "./@types/common";
 
 // @ts-ignore - $webapp is a webpack resolve alias pointing to the output directory, see webpack config
 import webpackLangJsonUrl from "$webapp/i18n/languages.json";
 
+export { normalizeLanguageKey, getNormalizedLanguageKeys } from "matrix-web-i18n";
+
 const i18nFolder = "i18n/";
 
 // Control whether to also return original, untranslated strings
 // Useful for debugging and testing
 const ANNOTATE_STRINGS = false;
 
 // We use english strings as keys, some of which contain full stops
-counterpart.setSeparator("|");
+counterpart.setSeparator(KEY_SEPARATOR);
 
 // see `translateWithFallback` for an explanation of fallback handling
 const FALLBACK_LOCALE = "en";
@@ -110,7 +114,7 @@ export function getUserLanguage(): string {
  * }
  * }
  */
-export type TranslationKey = Leaves<typeof Translations, "|", string | { other: string }, 4>;
+export type TranslationKey = _TranslationKey<typeof Translations>;
 
 // Function which only purpose is to mark that a string is translatable
 // Does not actually do anything. It's helpful for automatic extraction of translatable strings
@@ -541,41 +545,6 @@ export function getLanguageFromBrowser(): string {
  return getLanguagesFromBrowser()[0];
 }
 
-/**
- * Turns a language string, normalises it,
- * (see normalizeLanguageKey) into an array of language strings
- * with fallback to generic languages
- * (eg. 'pt-BR' => ['pt-br', 'pt'])
- *
- * @param {string} language The input language string
- * @return {string[]} List of normalised languages
- */
-export function getNormalizedLanguageKeys(language: string): string[] {
- const languageKeys: string[] = [];
- const normalizedLanguage = normalizeLanguageKey(language);
- const languageParts = normalizedLanguage.split("-");
- if (languageParts.length === 2 && languageParts[0] === languageParts[1]) {
- languageKeys.push(languageParts[0]);
- } else {
- languageKeys.push(normalizedLanguage);
- if (languageParts.length === 2) {
- languageKeys.push(languageParts[0]);
- }
- }
- return languageKeys;
-}
-
-/**
- * Returns a language string with underscores replaced with
- * hyphens, and lowercased.
- *
- * @param {string} language The language string to be normalized
- * @returns {string} The normalized language string
- */
-export function normalizeLanguageKey(language: string): string {
- return language.toLowerCase().replace("_", "-");
-}
-
 export function getCurrentLanguage(): string {
  return counterpart.getLocale();
 }
@@ -662,34 +631,26 @@ async function getLanguage(langPath: string): Promise<ICounterpartTranslation> {
  return res.json();
 }
 
-export interface ICustomTranslations {
- // Format is a map of english string to language to override
- [str: string]: {
- [lang: string]: string;
- };
-}
-
-let cachedCustomTranslations: Optional<ICustomTranslations> = null;
+let cachedCustomTranslations: Optional<TranslationStringsObject> = null;
 let cachedCustomTranslationsExpire = 0; // zero to trigger expiration right away
 
 // This awkward class exists so the test runner can get at the function. It is
 // not intended for practical or realistic usage.
 export class CustomTranslationOptions {
- public static lookupFn?: (url: string) => ICustomTranslations;
+ public static lookupFn?: (url: string) => TranslationStringsObject;
 
  private constructor() {
  // static access for tests only
  }
 }
 
-function doRegisterTranslations(customTranslations: ICustomTranslations): void {
- // We convert the operator-friendly version into something counterpart can
- // consume.
+function doRegisterTranslations(customTranslations: TranslationStringsObject): void {
+ // We convert the operator-friendly version into something counterpart can consume.
  // Map: lang → Record: string → translation
  const langs: MapWithDefault<string, Record<string, string>> = new MapWithDefault(() => ({}));
- for (const [str, translations] of Object.entries(customTranslations)) {
- for (const [lang, newStr] of Object.entries(translations)) {
- safeSet(langs.getOrCreate(lang), str, newStr);
+ for (const [translationKey, translations] of Object.entries(customTranslations)) {
+ for (const [lang, translation] of Object.entries(translations)) {
+ _.set(langs.getOrCreate(lang), translationKey.split(KEY_SEPARATOR), translation);
  }
  }
 
@@ -719,11 +680,11 @@ export async function registerCustomTranslations({
  if (!lookupUrl) return; // easy - nothing to do
 
  try {
- let json: Optional<ICustomTranslations>;
+ let json: Optional<TranslationStringsObject>;
  if (testOnlyIgnoreCustomTranslationsCache || Date.now() >= cachedCustomTranslationsExpire) {
  json = CustomTranslationOptions.lookupFn
  ? CustomTranslationOptions.lookupFn(lookupUrl)
- : ((await (await fetch(lookupUrl)).json()) as ICustomTranslations);
+ : ((await (await fetch(lookupUrl)).json()) as TranslationStringsObject);
  cachedCustomTranslations = json;
 
  // Set expiration to the future, but not too far. Just trying to avoid

test/languageHandler-test.tsx

@@ -16,14 +16,15 @@ limitations under the License.
 
 import React from "react";
 import fetchMock from "fetch-mock-jest";
+import { Translation } from "matrix-web-i18n";
+import { TranslationStringsObject } from "@matrix-org/react-sdk-module-api";
 
 import SdkConfig from "../src/SdkConfig";
 import {
  _t,
  _tDom,
  CustomTranslationOptions,
  getAllLanguagesWithLabels,
- ICustomTranslations,
  registerCustomTranslations,
  setLanguage,
  setMissingEntryGenerator,
@@ -35,9 +36,9 @@ import {
 import { stubClient } from "./test-utils";
 import { setupLanguageMock } from "./setup/setupLanguage";
 
-async function setupTranslationOverridesForTests(overrides: ICustomTranslations) {
+async function setupTranslationOverridesForTests(overrides: TranslationStringsObject) {
  const lookupUrl = "/translations.json";
- const fn = (url: string): ICustomTranslations => {
+ const fn = (url: string): TranslationStringsObject => {
  expect(url).toEqual(lookupUrl);
  return overrides;
  };
@@ -62,15 +63,15 @@ describe("languageHandler", () => {
  });
 
  it("should support overriding translations", async () => {
- const str = "This is a test string that does not exist in the app." as TranslationKey;
- const enOverride = "This is the English version of a custom string." as TranslationKey;
- const deOverride = "This is the German version of a custom string." as TranslationKey;
+ const str: TranslationKey = "power_level|default";
+ const enOverride: Translation = "Visitor";
+ const deOverride: Translation = "Besucher";
 
  // First test that overrides aren't being used
  await setLanguage("en");
- expect(_t(str)).toEqual(str);
+ expect(_t(str)).toMatchInlineSnapshot(`"Default"`);
  await setLanguage("de");
- expect(_t(str)).toEqual(str);
+ expect(_t(str)).toMatchInlineSnapshot(`"Standard"`);
 
  await setupTranslationOverridesForTests({
  [str]: {
@@ -87,6 +88,42 @@ describe("languageHandler", () => {
  expect(_t(str)).toEqual(deOverride);
  });
 
+ it("should support overriding plural translations", async () => {
+ const str: TranslationKey = "voip|n_people_joined";
+ const enOverride: Translation = {
+ other: "%(count)s people in the call",
+ one: "%(count)s person in the call",
+ };
+ const deOverride: Translation = {
+ other: "%(count)s Personen im Anruf",
+ one: "%(count)s Person im Anruf",
+ };
+
+ // First test that overrides aren't being used
+ await setLanguage("en");
+ expect(_t(str, { count: 1 })).toMatchInlineSnapshot(`"1 person joined"`);
+ expect(_t(str, { count: 5 })).toMatchInlineSnapshot(`"5 people joined"`);
+ await setLanguage("de");
+ expect(_t(str, { count: 1 })).toMatchInlineSnapshot(`"1 Person beigetreten"`);
+ expect(_t(str, { count: 5 })).toMatchInlineSnapshot(`"5 Personen beigetreten"`);
+
+ await setupTranslationOverridesForTests({
+ [str]: {
+ en: enOverride,
+ de: deOverride,
+ },
+ });
+
+ // Now test that they *are* being used
+ await setLanguage("en");
+ expect(_t(str, { count: 1 })).toMatchInlineSnapshot(`"1 person in the call"`);
+ expect(_t(str, { count: 5 })).toMatchInlineSnapshot(`"5 people in the call"`);
+
+ await setLanguage("de");
+ expect(_t(str, { count: 1 })).toMatchInlineSnapshot(`"1 Person im Anruf"`);
+ expect(_t(str, { count: 5 })).toMatchInlineSnapshot(`"5 Personen im Anruf"`);
+ });
+
  describe("UserFriendlyError", () => {
  const testErrorMessage = "This email address is already in use (%(email)s)" as TranslationKey;
  beforeEach(async () => {

yarn.lock

@@ -1916,10 +1916,10 @@
  version "3.2.14"
  resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz#acd96c00a881d0f462e1f97a56c73742c8dbc984"
 
-"@matrix-org/react-sdk-module-api@^2.1.0":
- version "2.1.0"
- resolved "https://registry.yarnpkg.com/@matrix-org/react-sdk-module-api/-/react-sdk-module-api-2.1.0.tgz#ca9d67853512fda1df2786810b90be31dd8dc7b1"
- integrity sha512-SARD5BsmZYv1hvuezLfBUafJ9+rPLbk5WO0S3vZgkLH3jJQrk7f/65qBB5fLKF2ljprfZ1GTpuBeq04wn7Tnmg==
+"@matrix-org/react-sdk-module-api@^2.1.1":
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/@matrix-org/react-sdk-module-api/-/react-sdk-module-api-2.1.1.tgz#54e8617c15185010d608c0325ecaec8d1574d12b"
+ integrity sha512-dYPY3aXtNwPrg2aEmFeWddMdohus/Ha17XES2QH+WMCawt+hH+uq28jH1EmW1RUOOzxVcdY36lRGOwqRtAJbhA==
  dependencies:
  "@babel/runtime" "^7.17.9"
 
@@ -7676,14 +7676,15 @@ matrix-mock-request@^2.5.0:
  dependencies:
  expect "^28.1.0"
 
-matrix-web-i18n@^2.1.0:
- version "2.1.0"
- resolved "https://registry.yarnpkg.com/matrix-web-i18n/-/matrix-web-i18n-2.1.0.tgz#bab2db9ac462773de829053b4b8d43c11154a85b"
- integrity sha512-z+B9D/PkWYB4O9SP4lsG4KNA2V3ypMWstP+lreft1c1wz6L5R1U3ennp+cs3yOsylBfcK+xLRvkwLNZsU6QEUA==
+matrix-web-i18n@^3.1.3:
+ version "3.1.3"
+ resolved "https://registry.yarnpkg.com/matrix-web-i18n/-/matrix-web-i18n-3.1.3.tgz#b462015b138ebdd288ed945507abea42c896f52d"
+ integrity sha512-9JUUTifqS/Xe6YQr5uDbX04xvr5Pxg8aU7tRKx49/ZLqm4dZoJKo4SKpyLEwCQeNjAvjcKuXibWO+2hkZ2/Ojw==
  dependencies:
  "@babel/parser" "^7.18.5"
  "@babel/traverse" "^7.18.5"
  lodash "^4.17.21"
+ minimist "^1.2.8"
  walk "^2.3.15"
 
 matrix-widget-api@^1.5.0, matrix-widget-api@^1.6.0: