From 3d0a6ba83128641ddd8d66ec539df9d52a7679e3 Mon Sep 17 00:00:00 2001 From: Echo Date: Fri, 13 Feb 2026 16:23:22 +0100 Subject: [PATCH] Emoji: Cache data by path instead of just Etag (#37858) --- .../mastodon/actions/importer/emoji.ts | 4 +- .../mastodon/features/emoji/database.test.ts | 35 ------ .../mastodon/features/emoji/database.ts | 46 +++----- .../mastodon/features/emoji/db-schema.ts | 4 +- .../mastodon/features/emoji/loader.ts | 104 +++++++++++------- .../mastodon/features/emoji/locale.ts | 4 +- .../mastodon/features/emoji/types.ts | 2 +- 7 files changed, 85 insertions(+), 114 deletions(-) diff --git a/app/javascript/mastodon/actions/importer/emoji.ts b/app/javascript/mastodon/actions/importer/emoji.ts index c4baa57d56c..9e06c88f66e 100644 --- a/app/javascript/mastodon/actions/importer/emoji.ts +++ b/app/javascript/mastodon/actions/importer/emoji.ts @@ -7,7 +7,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) { } // First, check if we already have them all. - const { searchCustomEmojisByShortcodes, clearEtag } = + const { searchCustomEmojisByShortcodes, clearCache } = await import('@/mastodon/features/emoji/database'); const existingEmojis = await searchCustomEmojisByShortcodes( @@ -16,7 +16,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) { // If there's a mismatch, re-import all custom emojis. if (existingEmojis.length < emojis.length) { - await clearEtag('custom'); + await clearCache('custom'); await loadCustomEmoji(); } } diff --git a/app/javascript/mastodon/features/emoji/database.test.ts b/app/javascript/mastodon/features/emoji/database.test.ts index b7f667e8ab1..e272ec2dea5 100644 --- a/app/javascript/mastodon/features/emoji/database.test.ts +++ b/app/javascript/mastodon/features/emoji/database.test.ts @@ -3,7 +3,6 @@ import { IDBFactory } from 'fake-indexeddb'; import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories'; -import { EMOJI_DB_SHORTCODE_TEST } from './constants'; import { putEmojiData, loadEmojiByHexcode, @@ -12,8 +11,6 @@ import { putCustomEmojiData, putLegacyShortcodes, loadLegacyShortcodesByShortcode, - loadLatestEtag, - putLatestEtag, } from './database'; function rawEmojiFactory(data: Partial = {}): CompactEmoji { @@ -120,36 +117,4 @@ describe('emoji database', () => { ).resolves.toEqual(data); }); }); - - describe('loadLatestEtag', () => { - beforeEach(async () => { - await putLatestEtag('etag', 'en'); - await putEmojiData([unicodeEmojiFactory()], 'en'); - await putLatestEtag('fr-etag', 'fr'); - }); - - test('retrieves the etag for loaded locale', async () => { - await putEmojiData( - [unicodeEmojiFactory({ hexcode: EMOJI_DB_SHORTCODE_TEST })], - 'en', - ); - const etag = await loadLatestEtag('en'); - expect(etag).toBe('etag'); - }); - - test('returns null if locale has no shortcodes', async () => { - const etag = await loadLatestEtag('en'); - expect(etag).toBeNull(); - }); - - test('returns null if locale not loaded', async () => { - const etag = await loadLatestEtag('de'); - expect(etag).toBeNull(); - }); - - test('returns null if locale has no data', async () => { - const etag = await loadLatestEtag('fr'); - expect(etag).toBeNull(); - }); - }); }); diff --git a/app/javascript/mastodon/features/emoji/database.ts b/app/javascript/mastodon/features/emoji/database.ts index f64f3fb80d6..8dbd22c71ba 100644 --- a/app/javascript/mastodon/features/emoji/database.ts +++ b/app/javascript/mastodon/features/emoji/database.ts @@ -3,21 +3,16 @@ import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase'; import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; -import { EMOJI_DB_SHORTCODE_TEST } from './constants'; import { openEmojiDB } from './db-schema'; import type { Database } from './db-schema'; -import { - localeToSegmenter, - toSupportedLocale, - toSupportedLocaleOrCustom, -} from './locale'; +import { localeToSegmenter, toSupportedLocale } from './locale'; import { extractTokens, skinHexcodeToEmoji, transformCustomEmojiData, transformEmojiData, } from './normalize'; -import type { AnyEmojiData, EtagTypes } from './types'; +import type { AnyEmojiData, CacheKey } from './types'; import { emojiLogger } from './utils'; const loadedLocales = new Set(); @@ -214,16 +209,21 @@ export async function putLegacyShortcodes(shortcodes: ShortcodesDataset) { await trx.done; } -export async function putLatestEtag(etag: string, name: EtagTypes) { +export async function loadCacheValue(key: CacheKey) { + const db = await loadDB(); + const value = await db.get('etags', key); + return value; +} + +export async function putCacheValue(key: CacheKey, value: string) { const db = await loadDB(); - await db.put('etags', etag, name); + await db.put('etags', value, key); } -export async function clearEtag(localeString: string) { - const locale = toSupportedLocaleOrCustom(localeString); +export async function clearCache(key: CacheKey) { const db = await loadDB(); - await db.delete('etags', locale); - log('Cleared etag for %s', locale); + await db.delete('etags', key); + log('Cleared cache for %s', key); } export async function loadEmojiByHexcode( @@ -276,26 +276,6 @@ export async function loadLegacyShortcodesByShortcode(shortcode: string) { ); } -export async function loadLatestEtag(localeString: string) { - const locale = toSupportedLocaleOrCustom(localeString); - const db = await loadDB(); - const rowCount = await db.count(locale); - if (!rowCount) { - return null; // No data for this locale, return null even if there is an etag. - } - - // Check if shortcodes exist for the given Unicode locale. - if (locale !== 'custom') { - const result = await db.get(locale, EMOJI_DB_SHORTCODE_TEST); - if (!result?.shortcodes) { - return null; - } - } - - const etag = await db.get('etags', locale); - return etag ?? null; -} - // Private functions async function syncLocales(db: Database) { diff --git a/app/javascript/mastodon/features/emoji/db-schema.ts b/app/javascript/mastodon/features/emoji/db-schema.ts index f5582cf0c33..7f4d09fd0b0 100644 --- a/app/javascript/mastodon/features/emoji/db-schema.ts +++ b/app/javascript/mastodon/features/emoji/db-schema.ts @@ -10,7 +10,7 @@ import type { StoreNames, } from 'idb'; -import type { CustomEmojiData, EtagTypes, UnicodeEmojiData } from './types'; +import type { CustomEmojiData, CacheKey, UnicodeEmojiData } from './types'; import { emojiLogger } from './utils'; const log = emojiLogger('database'); @@ -35,7 +35,7 @@ interface EmojiDB extends LocaleTables, DBSchema { }; }; etags: { - key: EtagTypes; + key: CacheKey; value: string; }; } diff --git a/app/javascript/mastodon/features/emoji/loader.ts b/app/javascript/mastodon/features/emoji/loader.ts index 7a94d604a98..0774fd4063f 100644 --- a/app/javascript/mastodon/features/emoji/loader.ts +++ b/app/javascript/mastodon/features/emoji/loader.ts @@ -4,11 +4,11 @@ import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase'; import { putEmojiData, putCustomEmojiData, - loadLatestEtag, - putLatestEtag, + putCacheValue, putLegacyShortcodes, + loadCacheValue, } from './database'; -import { toSupportedLocale, toValidEtagName } from './locale'; +import { toSupportedLocale, toValidCacheKey } from './locale'; import type { CustomEmojiData } from './types'; import { emojiLogger } from './utils'; @@ -23,8 +23,8 @@ export async function importEmojiData(localeString: string, shortcodes = true) { shortcodes ? ' and shortcodes' : '', ); - let emojis = await fetchAndCheckEtag({ - etagString: locale, + let emojis = await fetchIfNotLoaded({ + key: locale, path: localeToEmojiPath(locale), }); if (!emojis) { @@ -33,8 +33,8 @@ export async function importEmojiData(localeString: string, shortcodes = true) { const shortcodesData: ShortcodesDataset[] = []; if (shortcodes) { - const shortcodesResponse = await fetchAndCheckEtag({ - etagString: `${locale}-shortcodes`, + const shortcodesResponse = await fetchIfNotLoaded({ + key: `${locale}-shortcodes`, path: localeToShortcodesPath(locale), }); if (shortcodesResponse) { @@ -51,13 +51,24 @@ export async function importEmojiData(localeString: string, shortcodes = true) { } export async function importCustomEmojiData() { - const emojis = await fetchAndCheckEtag({ - etagString: 'custom', + const response = await fetchAndCheckEtag({ + oldEtag: await loadCacheValue('custom'), path: '/api/v1/custom_emojis', }); - if (!emojis) { + + if (!response) { return; } + + const etag = response.headers.get('ETag'); + if (etag) { + log('Custom emoji data fetched successfully, storing etag %s', etag); + await putCacheValue('custom', etag); + } else { + log('No etag found in response for custom emoji data'); + } + + const emojis = (await response.json()) as CustomEmojiData[]; await putCustomEmojiData({ emojis, clear: true }); return emojis; } @@ -72,9 +83,8 @@ export async function importLegacyShortcodes() { if (!path) { throw new Error('IAMCAL shortcodes path not found'); } - const shortcodesData = await fetchAndCheckEtag({ - checkEtag: true, - etagString: 'shortcodes', + const shortcodesData = await fetchIfNotLoaded({ + key: 'shortcodes', path, }); if (!shortcodesData) { @@ -118,48 +128,64 @@ function localeToShortcodesPath(locale: Locale) { return path; } -async function fetchAndCheckEtag({ - etagString, +async function fetchIfNotLoaded({ + key: rawKey, path, - checkEtag = false, }: { - etagString: string; + key: string; path: string; - checkEtag?: boolean; }): Promise { - const etagName = toValidEtagName(etagString); - const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null; + const key = toValidCacheKey(rawKey); + + const value = await loadCacheValue(key); + + if (value === path) { + log('data for %s already loaded, skipping fetch', key); + return null; + } + + const response = await fetchAndCheckEtag({ path }); + if (!response) { + return null; + } + + log('data for %s fetched successfully, storing etag', key); + await putCacheValue(key, path); + + return (await response.json()) as ResultType; +} + +async function fetchAndCheckEtag({ + oldEtag, + path, +}: { + oldEtag?: string; + path: string; +}) { + const headers = new Headers({ + 'Content-Type': 'application/json', + }); + if (oldEtag) { + headers.set('If-None-Match', oldEtag); + } // Use location.origin as this script may be loaded from a CDN domain. const url = new URL(path, location.origin); - const response = await fetch(url, { - headers: { - 'Content-Type': 'application/json', - 'If-None-Match': oldEtag ?? '', // Send the old ETag to check for modifications - }, + headers, }); + // If not modified, return null if (response.status === 304) { - log('etag not modified for %s', etagName); + log('etag not modified for %s', path); return null; } + if (!response.ok) { throw new Error( - `Failed to fetch emoji data for ${etagName}: ${response.statusText}`, + `Failed to fetch emoji data for ${path}: ${response.statusText}`, ); } - const data = (await response.json()) as ResultType; - - // Store the ETag for future requests - const etag = response.headers.get('ETag'); - if (etag && checkEtag) { - log(`storing new etag for ${etagName}: ${etag}`); - await putLatestEtag(etag, etagName); - } else if (!etag) { - log(`no etag found in response for ${etagName}`); - } - - return data; + return response; } diff --git a/app/javascript/mastodon/features/emoji/locale.ts b/app/javascript/mastodon/features/emoji/locale.ts index e8f2df340f1..8f4e1d4adc2 100644 --- a/app/javascript/mastodon/features/emoji/locale.ts +++ b/app/javascript/mastodon/features/emoji/locale.ts @@ -2,7 +2,7 @@ import type { Locale } from 'emojibase'; import { SUPPORTED_LOCALES } from 'emojibase'; import { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants'; -import type { EtagTypes, LocaleOrCustom, LocaleWithShortcodes } from './types'; +import type { CacheKey, LocaleOrCustom, LocaleWithShortcodes } from './types'; export function toSupportedLocale(localeBase: string): Locale { const locale = localeBase.toLowerCase(); @@ -19,7 +19,7 @@ export function toSupportedLocaleOrCustom(locale: string): LocaleOrCustom { return toSupportedLocale(locale); } -export function toValidEtagName(input: string): EtagTypes { +export function toValidCacheKey(input: string): CacheKey { const lower = input.toLowerCase(); if (lower === EMOJI_TYPE_CUSTOM || lower === EMOJI_DB_NAME_SHORTCODES) { return lower; diff --git a/app/javascript/mastodon/features/emoji/types.ts b/app/javascript/mastodon/features/emoji/types.ts index 8ab756972a8..2eb0c0f2327 100644 --- a/app/javascript/mastodon/features/emoji/types.ts +++ b/app/javascript/mastodon/features/emoji/types.ts @@ -22,7 +22,7 @@ export type EmojiMode = export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM; export type LocaleWithShortcodes = `${Locale}-shortcodes`; -export type EtagTypes = +export type CacheKey = | LocaleOrCustom | typeof EMOJI_DB_NAME_SHORTCODES | LocaleWithShortcodes;