Emoji: Fix path resolution for emoji worker (#36897)

app/javascript/mastodon/features/emoji/index.ts

@@ -1,6 +1,7 @@
 import { initialState } from '@/mastodon/initial_state';
 
 import { toSupportedLocale } from './locale';
+import type { LocaleOrCustom } from './types';
 import { emojiLogger } from './utils';
 // eslint-disable-next-line import/default -- Importing via worker loader.
 import EmojiWorker from './worker?worker&inline';
@@ -24,19 +25,17 @@ export function initializeEmoji() {
   }
 
   if (worker) {
-    // Assign worker to const to make TS happy inside the event listener.
-    const thisWorker = worker;
     const timeoutId = setTimeout(() => {
       log('worker is not ready after timeout');
       worker = null;
       void fallbackLoad();
     }, WORKER_TIMEOUT);
-    thisWorker.addEventListener('message', (event: MessageEvent<string>) => {
+    worker.addEventListener('message', (event: MessageEvent<string>) => {
       const { data: message } = event;
       if (message === 'ready') {
         log('worker ready, loading data');
         clearTimeout(timeoutId);
-        thisWorker.postMessage('custom');
+        messageWorker('custom');
         void loadEmojiLocale(userLocale);
         // Load English locale as well, because people are still used to
         // using it from before we supported other locales.
@@ -55,20 +54,35 @@ export function initializeEmoji() {
 async function fallbackLoad() {
   log('falling back to main thread for loading');
   const { importCustomEmojiData } = await import('./loader');
-  await importCustomEmojiData();
+  const emojis = await importCustomEmojiData();
+  if (emojis) {
+    log('loaded %d custom emojis', emojis.length);
+  }
   await loadEmojiLocale(userLocale);
   if (userLocale !== 'en') {
     await loadEmojiLocale('en');
   }
 }
 
-export async function loadEmojiLocale(localeString: string) {
+async function loadEmojiLocale(localeString: string) {
   const locale = toSupportedLocale(localeString);
+  const { importEmojiData, localeToPath } = await import('./loader');
 
   if (worker) {
-    worker.postMessage(locale);
+    const path = await localeToPath(locale);
+    log('asking worker to load locale %s from %s', locale, path);
+    messageWorker(locale, path);
   } else {
-    const { importEmojiData } = await import('./loader');
+    const emojis = await importEmojiData(locale);
-    await importEmojiData(locale);
+    if (emojis) {
+      log('loaded %d emojis to locale %s', emojis.length, locale);
+    }
+  }
+}
+
+function messageWorker(locale: LocaleOrCustom, path?: string) {
+  if (!worker) {
+    return;
   }
+  worker.postMessage({ locale, path });
 }

app/javascript/mastodon/features/emoji/loader.ts

@@ -8,44 +8,64 @@ import {
   putLatestEtag,
 } from './database';
 import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
-import type { CustomEmojiData, LocaleOrCustom } from './types';
+import type { CustomEmojiData } from './types';
-import { emojiLogger } from './utils';
 
-const log = emojiLogger('loader');
+export async function importEmojiData(localeString: string, path?: string) {
-
-export async function importEmojiData(localeString: string) {
   const locale = toSupportedLocale(localeString);
-  const emojis = await fetchAndCheckEtag<CompactEmoji[]>(locale);
+
+  // Validate the provided path.
+  if (path && !/^[/a-z]*\/packs\/assets\/compact-\w+\.json$/.test(path)) {
+    throw new Error('Invalid path for emoji data');
+  } else {
+    // Otherwise get the path if not provided.
+    path ??= await localeToPath(locale);
+  }
+
+  const emojis = await fetchAndCheckEtag<CompactEmoji[]>(locale, path);
   if (!emojis) {
     return;
   }
   const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData(emojis);
-  log('loaded %d for %s locale', flattenedEmojis.length, locale);
   await putEmojiData(flattenedEmojis, locale);
+  return flattenedEmojis;
 }
 
 export async function importCustomEmojiData() {
-  const emojis = await fetchAndCheckEtag<CustomEmojiData[]>('custom');
+  const emojis = await fetchAndCheckEtag<CustomEmojiData[]>(
+    'custom',
+    '/api/v1/custom_emojis',
+  );
   if (!emojis) {
     return;
   }
-  log('loaded %d custom emojis', emojis.length);
   await putCustomEmojiData(emojis);
+  return emojis;
+}
+
+const modules = import.meta.glob<string>(
+  '../../../../../node_modules/emojibase-data/**/compact.json',
+  {
+    query: '?url',
+    import: 'default',
+  },
+);
+
+export function localeToPath(locale: Locale) {
+  const key = `../../../../../node_modules/emojibase-data/${locale}/compact.json`;
+  if (!modules[key] || typeof modules[key] !== 'function') {
+    throw new Error(`Unsupported locale: ${locale}`);
+  }
+  return modules[key]();
 }
 
-async function fetchAndCheckEtag<ResultType extends object[]>(
+export async function fetchAndCheckEtag<ResultType extends object[]>(
-  localeOrCustom: LocaleOrCustom,
+  localeString: string,
+  path: string,
 ): Promise<ResultType | null> {
-  const locale = toSupportedLocaleOrCustom(localeOrCustom);
+  const locale = toSupportedLocaleOrCustom(localeString);
 
   // Use location.origin as this script may be loaded from a CDN domain.
-  const url = new URL(location.origin);
+  const url = new URL(path, location.origin);
-  if (locale === 'custom') {
-    url.pathname = '/api/v1/custom_emojis';
-  } else {
-    const modulePath = await localeToPath(locale);
-    url.pathname = modulePath;
-  }
 
   const oldEtag = await loadLatestEtag(locale);
   const response = await fetch(url, {
@@ -60,38 +80,20 @@ async function fetchAndCheckEtag<ResultType extends object[]>(
   }
   if (!response.ok) {
     throw new Error(
-      `Failed to fetch emoji data for ${localeOrCustom}: ${response.statusText}`,
+      `Failed to fetch emoji data for ${locale}: ${response.statusText}`,
     );
   }
 
   const data = (await response.json()) as ResultType;
   if (!Array.isArray(data)) {
-    throw new Error(
+    throw new Error(`Unexpected data format for ${locale}: expected an array`);
-      `Unexpected data format for ${localeOrCustom}: expected an array`,
-    );
   }
 
   // Store the ETag for future requests
   const etag = response.headers.get('ETag');
   if (etag) {
-    await putLatestEtag(etag, localeOrCustom);
+    await putLatestEtag(etag, localeString);
   }
 
   return data;
 }
-
-const modules = import.meta.glob<string>(
-  '../../../../../node_modules/emojibase-data/**/compact.json',
-  {
-    query: '?url',
-    import: 'default',
-  },
-);
-
-function localeToPath(locale: Locale) {
-  const key = `../../../../../node_modules/emojibase-data/${locale}/compact.json`;
-  if (!modules[key] || typeof modules[key] !== 'function') {
-    throw new Error(`Unsupported locale: ${locale}`);
-  }
-  return modules[key]();
-}

app/javascript/mastodon/features/emoji/render.test.ts

@@ -162,7 +162,7 @@ describe('loadEmojiDataToState', () => {
     const dbCall = vi
       .spyOn(db, 'loadEmojiByHexcode')
       .mockRejectedValue(new db.LocaleNotLoadedError('en'));
-    vi.spyOn(loader, 'importEmojiData').mockResolvedValueOnce();
+    vi.spyOn(loader, 'importEmojiData').mockResolvedValueOnce(undefined);
     const consoleCall = vi
       .spyOn(console, 'warn')
       .mockImplementationOnce(() => null);

app/javascript/mastodon/features/emoji/worker.ts

@@ -1,18 +1,25 @@
-import { importEmojiData, importCustomEmojiData } from './loader';
+import { importCustomEmojiData, importEmojiData } from './loader';
 
 addEventListener('message', handleMessage);
 self.postMessage('ready'); // After the worker is ready, notify the main thread
 
-function handleMessage(event: MessageEvent<string>) {
+function handleMessage(event: MessageEvent<{ locale: string; path?: string }>) {
-  const { data: locale } = event;
+  const {
-  void loadData(locale);
+    data: { locale, path },
+  } = event;
+  void loadData(locale, path);
 }
 
-async function loadData(locale: string) {
+async function loadData(locale: string, path?: string) {
-  if (locale !== 'custom') {
+  let importCount: number | undefined;
-    await importEmojiData(locale);
+  if (locale === 'custom') {
+    importCount = (await importCustomEmojiData())?.length;
+  } else if (path) {
+    importCount = (await importEmojiData(locale, path))?.length;
   } else {
-    await importCustomEmojiData();
+    throw new Error('Path is required for loading locale emoji data');
+  }
+  if (importCount) {
+    self.postMessage(`loaded ${importCount} emojis into ${locale}`);
   }
-  self.postMessage(`loaded ${locale}`);
 }

app/javascript/mastodon/main.tsx

@@ -9,7 +9,6 @@ import { me, reduceMotion } from 'mastodon/initial_state';
 import ready from 'mastodon/ready';
 import { store } from 'mastodon/store';
 
-import { initializeEmoji } from './features/emoji';
 import { isProduction, isDevelopment } from './utils/environment';
 
 function main() {
@@ -30,6 +29,7 @@ function main() {
       });
     }
 
+    const { initializeEmoji } = await import('./features/emoji/index');
     initializeEmoji();
 
     const root = createRoot(mountNode);