From ae44e758d6ca820661a02a36893689cef1b248eb Mon Sep 17 00:00:00 2001 From: WilsonLe Date: Fri, 8 Nov 2024 19:36:07 +0700 Subject: [PATCH] explain why using grapheme clustering is important --- lib/pangea/widgets/chat/overlay_message_text.dart | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pangea/widgets/chat/overlay_message_text.dart b/lib/pangea/widgets/chat/overlay_message_text.dart index f23cbba5c..894be09e9 100644 --- a/lib/pangea/widgets/chat/overlay_message_text.dart +++ b/lib/pangea/widgets/chat/overlay_message_text.dart @@ -68,11 +68,16 @@ class OverlayMessageTextState extends State { ); } -// Convert the entire message into a list of characters + // Convert the entire message into a list of characters final Characters messageCharacters = widget.pangeaMessageEvent.event.body.characters; // When building token positions, use grapheme cluster indices + // We use grapheme cluster indices to avoid splitting emojis and other + // complex characters that requires multiple code units. + // For instance, the emoji 🇺🇸 is represented by two code units: + // - \u{1F1FA} + // - \u{1F1F8} final List tokenPositions = []; int globalIndex = 0;