aiden
/
mastodon
mirror of https://github.com/mastodon/mastodon
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Emoji: Cache data by path instead of just Etag (#37858)

Browse Source
pull/37860/head
Echo 2 weeks ago committed by GitHub
parent c1a5bd52f2
commit 3d0a6ba831
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 85 additions and 114 deletions
Show Stats Download Patch File Download Diff File

4
app/javascript/mastodon/actions/importer/emoji.ts
Unescape Escape View File

@ -7,7 +7,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
}
// First, check if we already have them all.
const { searchCustomEmojisByShortcodes, clearEtag } =
const { searchCustomEmojisByShortcodes, clearCache } =
await import('@/mastodon/features/emoji/database');
const existingEmojis = await searchCustomEmojisByShortcodes(
@ -16,7 +16,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
// If there's a mismatch, re-import all custom emojis.
if (existingEmojis.length < emojis.length) {
await clearEtag('custom');
await clearCache('custom');
await loadCustomEmoji();
}
}

35
app/javascript/mastodon/features/emoji/database.test.ts
Unescape Escape View File

@ -3,7 +3,6 @@ import { IDBFactory } from 'fake-indexeddb';
import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
import { EMOJI_DB_SHORTCODE_TEST } from './constants';
import {
putEmojiData,
loadEmojiByHexcode,
@ -12,8 +11,6 @@ import {
putCustomEmojiData,
putLegacyShortcodes,
loadLegacyShortcodesByShortcode,
loadLatestEtag,
putLatestEtag,
} from './database';
function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
@ -120,36 +117,4 @@ describe('emoji database', () => {
).resolves.toEqual(data);
});
});
describe('loadLatestEtag', () => {
beforeEach(async () => {
await putLatestEtag('etag', 'en');
await putEmojiData([unicodeEmojiFactory()], 'en');
await putLatestEtag('fr-etag', 'fr');
});
test('retrieves the etag for loaded locale', async () => {
await putEmojiData(
[unicodeEmojiFactory({ hexcode: EMOJI_DB_SHORTCODE_TEST })],
'en',
);
const etag = await loadLatestEtag('en');
expect(etag).toBe('etag');
});
test('returns null if locale has no shortcodes', async () => {
const etag = await loadLatestEtag('en');
expect(etag).toBeNull();
});
test('returns null if locale not loaded', async () => {
const etag = await loadLatestEtag('de');
expect(etag).toBeNull();
});
test('returns null if locale has no data', async () => {
const etag = await loadLatestEtag('fr');
expect(etag).toBeNull();
});
});
});

46
app/javascript/mastodon/features/emoji/database.ts
Unescape Escape View File

@ -3,21 +3,16 @@ import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
import { EMOJI_DB_SHORTCODE_TEST } from './constants';
import { openEmojiDB } from './db-schema';
import type { Database } from './db-schema';
import {
localeToSegmenter,
toSupportedLocale,
toSupportedLocaleOrCustom,
} from './locale';
import { localeToSegmenter, toSupportedLocale } from './locale';
import {
extractTokens,
skinHexcodeToEmoji,
transformCustomEmojiData,
transformEmojiData,
} from './normalize';
import type { AnyEmojiData, EtagTypes } from './types';
import type { AnyEmojiData, CacheKey } from './types';
import { emojiLogger } from './utils';
const loadedLocales = new Set<Locale>();
@ -214,16 +209,21 @@ export async function putLegacyShortcodes(shortcodes: ShortcodesDataset) {
await trx.done;
}
export async function putLatestEtag(etag: string, name: EtagTypes) {
export async function loadCacheValue(key: CacheKey) {
const db = await loadDB();
const value = await db.get('etags', key);
return value;
}
export async function putCacheValue(key: CacheKey, value: string) {
const db = await loadDB();
await db.put('etags', etag, name);
await db.put('etags', value, key);
}
export async function clearEtag(localeString: string) {
const locale = toSupportedLocaleOrCustom(localeString);
export async function clearCache(key: CacheKey) {
const db = await loadDB();
await db.delete('etags', locale);
log('Cleared etag for %s', locale);
await db.delete('etags', key);
log('Cleared cache for %s', key);
}
export async function loadEmojiByHexcode(
@ -276,26 +276,6 @@ export async function loadLegacyShortcodesByShortcode(shortcode: string) {
);
}
export async function loadLatestEtag(localeString: string) {
const locale = toSupportedLocaleOrCustom(localeString);
const db = await loadDB();
const rowCount = await db.count(locale);
if (!rowCount) {
return null; // No data for this locale, return null even if there is an etag.
}
// Check if shortcodes exist for the given Unicode locale.
if (locale !== 'custom') {
const result = await db.get(locale, EMOJI_DB_SHORTCODE_TEST);
if (!result?.shortcodes) {
return null;
}
}
const etag = await db.get('etags', locale);
return etag ?? null;
}
// Private functions
async function syncLocales(db: Database) {

4
app/javascript/mastodon/features/emoji/db-schema.ts
Unescape Escape View File

@ -10,7 +10,7 @@ import type {
StoreNames,
} from 'idb';
import type { CustomEmojiData, EtagTypes, UnicodeEmojiData } from './types';
import type { CustomEmojiData, CacheKey, UnicodeEmojiData } from './types';
import { emojiLogger } from './utils';
const log = emojiLogger('database');
@ -35,7 +35,7 @@ interface EmojiDB extends LocaleTables, DBSchema {
};
};
etags: {
key: EtagTypes;
key: CacheKey;
value: string;
};
}

104
app/javascript/mastodon/features/emoji/loader.ts
Unescape Escape View File

@ -4,11 +4,11 @@ import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
import {
putEmojiData,
putCustomEmojiData,
loadLatestEtag,
putLatestEtag,
putCacheValue,
putLegacyShortcodes,
loadCacheValue,
} from './database';
import { toSupportedLocale, toValidEtagName } from './locale';
import { toSupportedLocale, toValidCacheKey } from './locale';
import type { CustomEmojiData } from './types';
import { emojiLogger } from './utils';
@ -23,8 +23,8 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
shortcodes ? ' and shortcodes' : '',
);
let emojis = await fetchAndCheckEtag<CompactEmoji[]>({
etagString: locale,
let emojis = await fetchIfNotLoaded<CompactEmoji[]>({
key: locale,
path: localeToEmojiPath(locale),
});
if (!emojis) {
@ -33,8 +33,8 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
const shortcodesData: ShortcodesDataset[] = [];
if (shortcodes) {
const shortcodesResponse = await fetchAndCheckEtag<ShortcodesDataset>({
etagString: `${locale}-shortcodes`,
const shortcodesResponse = await fetchIfNotLoaded<ShortcodesDataset>({
key: `${locale}-shortcodes`,
path: localeToShortcodesPath(locale),
});
if (shortcodesResponse) {
@ -51,13 +51,24 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
}
export async function importCustomEmojiData() {
const emojis = await fetchAndCheckEtag<CustomEmojiData[]>({
etagString: 'custom',
const response = await fetchAndCheckEtag({
oldEtag: await loadCacheValue('custom'),
path: '/api/v1/custom_emojis',
});
if (!emojis) {
if (!response) {
return;
}
const etag = response.headers.get('ETag');
if (etag) {
log('Custom emoji data fetched successfully, storing etag %s', etag);
await putCacheValue('custom', etag);
} else {
log('No etag found in response for custom emoji data');
}
const emojis = (await response.json()) as CustomEmojiData[];
await putCustomEmojiData({ emojis, clear: true });
return emojis;
}
@ -72,9 +83,8 @@ export async function importLegacyShortcodes() {
if (!path) {
throw new Error('IAMCAL shortcodes path not found');
}
const shortcodesData = await fetchAndCheckEtag<ShortcodesDataset>({
checkEtag: true,
etagString: 'shortcodes',
const shortcodesData = await fetchIfNotLoaded<ShortcodesDataset>({
key: 'shortcodes',
path,
});
if (!shortcodesData) {
@ -118,48 +128,64 @@ function localeToShortcodesPath(locale: Locale) {
return path;
}
async function fetchAndCheckEtag<ResultType extends object[] | object>({
etagString,
async function fetchIfNotLoaded<ResultType extends object[] | object>({
key: rawKey,
path,
checkEtag = false,
}: {
etagString: string;
key: string;
path: string;
checkEtag?: boolean;
}): Promise<ResultType | null> {
const etagName = toValidEtagName(etagString);
const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null;
const key = toValidCacheKey(rawKey);
const value = await loadCacheValue(key);
if (value === path) {
log('data for %s already loaded, skipping fetch', key);
return null;
}
const response = await fetchAndCheckEtag({ path });
if (!response) {
return null;
}
log('data for %s fetched successfully, storing etag', key);
await putCacheValue(key, path);
return (await response.json()) as ResultType;
}
async function fetchAndCheckEtag({
oldEtag,
path,
}: {
oldEtag?: string;
path: string;
}) {
const headers = new Headers({
'Content-Type': 'application/json',
});
if (oldEtag) {
headers.set('If-None-Match', oldEtag);
}
// Use location.origin as this script may be loaded from a CDN domain.
const url = new URL(path, location.origin);
const response = await fetch(url, {
headers: {
'Content-Type': 'application/json',
'If-None-Match': oldEtag ?? '', // Send the old ETag to check for modifications
},
headers,
});
// If not modified, return null
if (response.status === 304) {
log('etag not modified for %s', etagName);
log('etag not modified for %s', path);
return null;
}
if (!response.ok) {
throw new Error(
`Failed to fetch emoji data for ${etagName}: ${response.statusText}`,
`Failed to fetch emoji data for ${path}: ${response.statusText}`,
);
}
const data = (await response.json()) as ResultType;
// Store the ETag for future requests
const etag = response.headers.get('ETag');
if (etag && checkEtag) {
log(`storing new etag for ${etagName}: ${etag}`);
await putLatestEtag(etag, etagName);
} else if (!etag) {
log(`no etag found in response for ${etagName}`);
}
return data;
return response;
}

4
app/javascript/mastodon/features/emoji/locale.ts
Unescape Escape View File

@ -2,7 +2,7 @@ import type { Locale } from 'emojibase';
import { SUPPORTED_LOCALES } from 'emojibase';
import { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants';
import type { EtagTypes, LocaleOrCustom, LocaleWithShortcodes } from './types';
import type { CacheKey, LocaleOrCustom, LocaleWithShortcodes } from './types';
export function toSupportedLocale(localeBase: string): Locale {
const locale = localeBase.toLowerCase();
@ -19,7 +19,7 @@ export function toSupportedLocaleOrCustom(locale: string): LocaleOrCustom {
return toSupportedLocale(locale);
}
export function toValidEtagName(input: string): EtagTypes {
export function toValidCacheKey(input: string): CacheKey {
const lower = input.toLowerCase();
if (lower === EMOJI_TYPE_CUSTOM || lower === EMOJI_DB_NAME_SHORTCODES) {
return lower;

2
app/javascript/mastodon/features/emoji/types.ts
Unescape Escape View File

@ -22,7 +22,7 @@ export type EmojiMode =
export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM;
export type LocaleWithShortcodes = `${Locale}-shortcodes`;
export type EtagTypes =
export type CacheKey =
| LocaleOrCustom
| typeof EMOJI_DB_NAME_SHORTCODES
| LocaleWithShortcodes;

Write Preview
Loading…
Cancel
Save