From 191fc6962898c14257edf1920c6356a36d93dda7 Mon Sep 17 00:00:00 2001 From: William Jordan-Cooley Date: Tue, 7 May 2024 16:10:53 -0400 Subject: [PATCH] now displaying score for pronunciation --- assets/l10n/intl_en.arb | 6 +- lib/pangea/models/speech_to_text_models.dart | 61 +++- .../chat/message_speech_to_text_card.dart | 136 +++++++-- .../widgets/chat/speech_to_text_score.dart | 29 -- .../widgets/common/icon_number_widget.dart | 47 +++ needed-translations.txt | 288 +++++++++++++++--- 6 files changed, 451 insertions(+), 116 deletions(-) delete mode 100644 lib/pangea/widgets/chat/speech_to_text_score.dart create mode 100644 lib/pangea/widgets/common/icon_number_widget.dart diff --git a/assets/l10n/intl_en.arb b/assets/l10n/intl_en.arb index 47f866b3b..4941b0dfc 100644 --- a/assets/l10n/intl_en.arb +++ b/assets/l10n/intl_en.arb @@ -3935,5 +3935,9 @@ "unread": {} } }, - "messageAnalytics": "Message Analytics" + "messageAnalytics": "Message Analytics", + "words": "Words", + "score": "Score", + "accuracy": "Accuracy", + "points": "Points" } \ No newline at end of file diff --git a/lib/pangea/models/speech_to_text_models.dart b/lib/pangea/models/speech_to_text_models.dart index 18cf95825..05ab6ccbe 100644 --- a/lib/pangea/models/speech_to_text_models.dart +++ b/lib/pangea/models/speech_to_text_models.dart @@ -1,6 +1,5 @@ import 'dart:convert'; -import 'package:fluffychat/config/app_config.dart'; import 'package:fluffychat/pangea/enum/audio_encoding_enum.dart'; import 'package:fluffychat/pangea/models/pangea_token_model.dart'; import 'package:flutter/foundation.dart'; @@ -88,10 +87,14 @@ class STTToken { int get length => token.text.length; Color color(BuildContext context) { - if (confidence == null || confidence! > 80) { - return Theme.of(context).brightness == Brightness.dark - ? AppConfig.primaryColorLight - : AppConfig.primaryColor; + if (confidence == null) { + return Theme.of(context).textTheme.bodyMedium?.color ?? + (Theme.of(context).brightness == Brightness.dark + ? Colors.white + : Colors.black); + } + if (confidence! > 80) { + return const Color.fromARGB(255, 0, 152, 0); } if (confidence! > 50) { return const Color.fromARGB(255, 184, 142, 43); @@ -99,16 +102,19 @@ class STTToken { return Colors.red; } - factory STTToken.fromJson(Map json) => STTToken( - token: PangeaToken.fromJson(json['token']), - startTime: json['start_time'] != null - ? Duration(milliseconds: json['start_time']) - : null, - endTime: json['end_time'] != null - ? Duration(milliseconds: json['end_time']) - : null, - confidence: json['confidence'], - ); + factory STTToken.fromJson(Map json) { + // debugPrint('STTToken.fromJson: $json'); + return STTToken( + token: PangeaToken.fromJson(json['token']), + startTime: json['start_time'] != null + ? Duration(milliseconds: json['start_time'] * 1000.toInt()) + : null, + endTime: json['end_time'] != null + ? Duration(milliseconds: json['end_time'] * 1000.toInt()) + : null, + confidence: json['confidence'], + ); + } Map toJson() => { "token": token, @@ -116,6 +122,27 @@ class STTToken { "end_time": endTime?.inMilliseconds, "confidence": confidence, }; + + @override + bool operator ==(Object other) { + if (identical(this, other)) return true; + if (other is! STTToken) return false; + + return token == other.token && + startTime == other.startTime && + endTime == other.endTime && + confidence == other.confidence; + } + + @override + int get hashCode { + return Object.hashAll([ + token.hashCode, + startTime.hashCode, + endTime.hashCode, + confidence.hashCode, + ]); + } } class Transcript { @@ -133,7 +160,9 @@ class Transcript { factory Transcript.fromJson(Map json) => Transcript( text: json['transcript'], - confidence: json['confidence'].toDouble(), + confidence: json['confidence'] <= 100 + ? json['confidence'] + : json['confidence'] / 100, sttTokens: (json['stt_tokens'] as List) .map((e) => STTToken.fromJson(e)) .toList(), diff --git a/lib/pangea/widgets/chat/message_speech_to_text_card.dart b/lib/pangea/widgets/chat/message_speech_to_text_card.dart index 5118eba96..2ec1fa19d 100644 --- a/lib/pangea/widgets/chat/message_speech_to_text_card.dart +++ b/lib/pangea/widgets/chat/message_speech_to_text_card.dart @@ -1,12 +1,14 @@ import 'package:fluffychat/pangea/matrix_event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/models/speech_to_text_models.dart'; -import 'package:fluffychat/pangea/utils/error_handler.dart'; -import 'package:fluffychat/pangea/widgets/chat/speech_to_text_score.dart'; -import 'package:fluffychat/pangea/widgets/chat/speech_to_text_text.dart'; import 'package:fluffychat/pangea/widgets/chat/toolbar_content_loading_indicator.dart'; +import 'package:fluffychat/pangea/widgets/common/icon_number_widget.dart'; import 'package:fluffychat/pangea/widgets/igc/card_error_widget.dart'; import 'package:fluffychat/widgets/matrix.dart'; +import 'package:flutter/gestures.dart'; import 'package:flutter/material.dart'; +import 'package:flutter_gen/gen_l10n/l10n.dart'; + +import '../../utils/bot_style.dart'; class MessageSpeechToTextCard extends StatefulWidget { final PangeaMessageEvent messageEvent; @@ -24,6 +26,7 @@ class MessageSpeechToTextCardState extends State { SpeechToTextModel? speechToTextResponse; bool _fetchingTranscription = true; Object? error; + STTToken? selectedToken; String? get l1Code => MatrixState.pangeaController.languageController.activeL1Code( @@ -37,26 +40,90 @@ class MessageSpeechToTextCardState extends State { // look for transcription in message event // if not found, call API to transcribe audio Future getSpeechToText() async { - try { - if (l1Code == null || l2Code == null) { - throw Exception('Language selection not found'); + // try { + if (l1Code == null || l2Code == null) { + throw Exception('Language selection not found'); + } + speechToTextResponse ??= + await widget.messageEvent.getSpeechToText(l1Code!, l2Code!); + + debugPrint( + 'Speech to text transcript: ${speechToTextResponse?.transcript.text}', + ); + // } catch (e, s) { + // debugger(when: kDebugMode); + // error = e; + // ErrorHandler.logError( + // e: e, + // s: s, + // data: widget.messageEvent.event.content, + // ); + // } finally { + setState(() => _fetchingTranscription = false); + // } + } + + TextSpan _buildTranscriptText(BuildContext context) { + final Transcript transcript = speechToTextResponse!.transcript; + final List spans = []; + final String fullText = transcript.text; + int lastEnd = 0; + + for (final token in transcript.sttTokens) { + // debugPrint('Token confidence: ${token.confidence}'); + // debugPrint('color: ${token.color(context)}'); + if (token.offset > lastEnd) { + // Add any plain text before the token + spans.add( + TextSpan( + text: fullText.substring(lastEnd, token.offset), + ), + ); + // debugPrint('Pre: ${fullText.substring(lastEnd, token.offset)}'); } - speechToTextResponse ??= - await widget.messageEvent.getSpeechToText(l1Code!, l2Code!); - debugPrint( - 'Speech to text transcript: ${speechToTextResponse?.transcript.text}', + spans.add( + TextSpan( + text: fullText.substring(token.offset, token.offset + token.length), + style: BotStyle.text( + context, + existingStyle: TextStyle(color: token.color(context)), + setColor: false, + ), + // gesturRecognizer that sets selectedToken on click + recognizer: TapGestureRecognizer() + ..onTap = () { + debugPrint('Token tapped'); + debugPrint(token.toJson().toString()); + setState(() { + if (selectedToken == token) { + selectedToken = null; + } else { + selectedToken = token; + } + }); + }, + ), ); - } catch (e, s) { - error = e; - ErrorHandler.logError( - e: e, - s: s, - data: widget.messageEvent.event.content, + + // debugPrint( + // 'Main: ${fullText.substring(token.offset, token.offset + token.length)}', + // ); + + lastEnd = token.offset + token.length; + } + + if (lastEnd < fullText.length) { + // Add any remaining text after the last token + spans.add( + TextSpan( + text: fullText.substring(lastEnd), + ), ); - } finally { - setState(() => _fetchingTranscription = false); + // debugPrint('Post: ${fullText.substring(lastEnd)}'); } + + return TextSpan(children: spans); } @override @@ -76,12 +143,37 @@ class MessageSpeechToTextCardState extends State { return CardErrorWidget(error: error); } + final int words = speechToTextResponse!.transcript.sttTokens.length; + final int accuracy = speechToTextResponse!.transcript.confidence; + final int total = words * accuracy; + + //TODO: find better icons return Column( children: [ - SpeechToTextText(transcript: speechToTextResponse!.transcript), - const Divider(), - SpeechToTextScoreWidget( - score: speechToTextResponse!.transcript.confidence, + RichText( + text: _buildTranscriptText(context), + ), + const SizedBox(height: 16), + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + IconNumberWidget( + icon: Icons.abc, + number: (selectedToken == null ? words : 1).toString(), + toolTip: L10n.of(context)!.words, + ), + IconNumberWidget( + icon: Icons.approval, + number: + "${selectedToken?.confidence ?? speechToTextResponse!.transcript.confidence}%", + toolTip: L10n.of(context)!.accuracy, + ), + IconNumberWidget( + icon: Icons.score, + number: (selectedToken?.confidence ?? total).toString(), + toolTip: L10n.of(context)!.points, + ), + ], ), ], ); diff --git a/lib/pangea/widgets/chat/speech_to_text_score.dart b/lib/pangea/widgets/chat/speech_to_text_score.dart deleted file mode 100644 index 1e45f73b2..000000000 --- a/lib/pangea/widgets/chat/speech_to_text_score.dart +++ /dev/null @@ -1,29 +0,0 @@ -import 'package:flutter/material.dart'; - -class SpeechToTextScoreWidget extends StatefulWidget { - final int score; - const SpeechToTextScoreWidget({super.key, required this.score}); - - @override - SpeechToTextScoreWidgetState createState() => SpeechToTextScoreWidgetState(); -} - -class SpeechToTextScoreWidgetState extends State { - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.symmetric(vertical: 8.0), - child: AnimatedSwitcher( - duration: const Duration(milliseconds: 500), - transitionBuilder: (Widget child, Animation animation) { - return FadeTransition(opacity: animation, child: child); - }, - child: Text( - 'Score: ${widget.score}', - key: ValueKey(widget.score), - style: const TextStyle(fontSize: 24, fontWeight: FontWeight.bold), - ), - ), - ); - } -} diff --git a/lib/pangea/widgets/common/icon_number_widget.dart b/lib/pangea/widgets/common/icon_number_widget.dart new file mode 100644 index 000000000..b42777f91 --- /dev/null +++ b/lib/pangea/widgets/common/icon_number_widget.dart @@ -0,0 +1,47 @@ +import 'package:flutter/material.dart'; + +class IconNumberWidget extends StatelessWidget { + final IconData icon; + final String number; + final Color? iconColor; + final double? iconSize; + final String? toolTip; + + const IconNumberWidget({ + super.key, + required this.icon, + required this.number, + this.toolTip, + this.iconColor, + this.iconSize, + }); + + Widget _content(BuildContext context) { + return Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + icon, + color: iconColor ?? Theme.of(context).iconTheme.color, + size: iconSize ?? Theme.of(context).iconTheme.size, + ), + const SizedBox(width: 8), + Text( + number.toString(), + style: TextStyle( + fontSize: + iconSize ?? Theme.of(context).textTheme.bodyMedium?.fontSize, + color: iconColor ?? Theme.of(context).textTheme.bodyMedium?.color, + ), + ), + ], + ); + } + + @override + Widget build(BuildContext context) { + return toolTip != null + ? Tooltip(message: toolTip!, child: _content(context)) + : _content(context); + } +} diff --git a/needed-translations.txt b/needed-translations.txt index 5d462da84..2aabbf886 100644 --- a/needed-translations.txt +++ b/needed-translations.txt @@ -814,7 +814,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "be": [ @@ -2227,7 +2231,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "bn": [ @@ -3102,7 +3110,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "bo": [ @@ -3977,7 +3989,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ca": [ @@ -4852,7 +4868,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "cs": [ @@ -5727,7 +5747,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "de": [ @@ -6549,7 +6573,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "el": [ @@ -7424,7 +7452,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "eo": [ @@ -8299,7 +8331,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "es": [ @@ -8322,7 +8358,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "et": [ @@ -9140,7 +9180,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "eu": [ @@ -9958,7 +10002,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "fa": [ @@ -10833,7 +10881,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "fi": [ @@ -11708,7 +11760,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "fr": [ @@ -12583,7 +12639,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ga": [ @@ -13458,7 +13518,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "gl": [ @@ -14276,7 +14340,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "he": [ @@ -15151,7 +15219,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "hi": [ @@ -16026,7 +16098,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "hr": [ @@ -16888,7 +16964,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "hu": [ @@ -17763,7 +17843,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ia": [ @@ -19162,7 +19246,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "id": [ @@ -20037,7 +20125,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ie": [ @@ -20912,7 +21004,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "it": [ @@ -21772,7 +21868,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ja": [ @@ -22647,7 +22747,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ko": [ @@ -23522,7 +23626,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "lt": [ @@ -24397,7 +24505,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "lv": [ @@ -25272,7 +25384,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "nb": [ @@ -26147,7 +26263,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "nl": [ @@ -27022,7 +27142,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "pl": [ @@ -27897,7 +28021,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "pt": [ @@ -28772,7 +28900,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "pt_BR": [ @@ -29616,7 +29748,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "pt_PT": [ @@ -30491,7 +30627,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ro": [ @@ -31366,7 +31506,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ru": [ @@ -32184,7 +32328,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "sk": [ @@ -33059,7 +33207,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "sl": [ @@ -33934,7 +34086,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "sr": [ @@ -34809,7 +34965,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "sv": [ @@ -35649,7 +35809,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "ta": [ @@ -36524,7 +36688,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "th": [ @@ -37399,7 +37567,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "tr": [ @@ -38259,7 +38431,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "uk": [ @@ -39077,7 +39253,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "vi": [ @@ -39952,7 +40132,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "zh": [ @@ -40770,7 +40954,11 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ], "zh_Hant": [ @@ -41645,6 +41833,10 @@ "commandHint_ignore", "commandHint_unignore", "unreadChatsInApp", - "messageAnalytics" + "messageAnalytics", + "words", + "score", + "accuracy", + "points" ] }