SpeechToText #808

Author: VladislavAntonyuk

samples/CommunityToolkit.Maui.Sample/AppShell.xaml.cs

@@ -84,6 +84,7 @@ public partial class AppShell : Shell
 // Add Essentials View Models
 CreateViewModelMapping<FileSaverPage, FileSaverViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 CreateViewModelMapping<FolderPickerPage, FolderPickerViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
+CreateViewModelMapping<SpeechToTextPage, SpeechToTextViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 
 // Add Extensions View Models
 CreateViewModelMapping<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel, ExtensionsGalleryPage, ExtensionsGalleryViewModel>(),

samples/CommunityToolkit.Maui.Sample/MauiProgram.cs

@@ -19,6 +19,7 @@
 using CommunityToolkit.Maui.Sample.ViewModels.Layouts;
 using CommunityToolkit.Maui.Sample.ViewModels.Views;
 using CommunityToolkit.Maui.Sample.ViewModels.Views.AvatarView;
+using CommunityToolkit.Maui.SpeechToText;
 using CommunityToolkit.Maui.Storage;
 using Microsoft.Extensions.Logging;
 using Polly;
@@ -153,6 +154,7 @@ static void RegisterViewsAndViewModels(in IServiceCollection services)
 // Add Essentials Pages + ViewModels
 services.AddTransientWithShellRoute<FileSaverPage, FileSaverViewModel>();
 services.AddTransientWithShellRoute<FolderPickerPage, FolderPickerViewModel>();
+services.AddTransientWithShellRoute<SpeechToTextPage, SpeechToTextViewModel>();
 
 // Add Extensions Pages + ViewModels
 services.AddTransientWithShellRoute<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel>();
@@ -187,6 +189,8 @@ static void RegisterEssentials(in IServiceCollection services)
 services.AddSingleton<IDeviceDisplay>(DeviceDisplay.Current);
 services.AddSingleton<IFileSaver>(FileSaver.Default);
 services.AddSingleton<IFolderPicker>(FolderPicker.Default);
+services.AddSingleton<ITextToSpeech>(TextToSpeech.Default);
+services.AddSingleton<ISpeechToText>(SpeechToText.SpeechToText.Default);
 }
 
 static IServiceCollection AddTransientWithShellRoute<TPage, TViewModel>(this IServiceCollection services) where TPage : BasePage<TViewModel>

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/SpeechToTextPage.xaml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<pages:BasePage xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
+ xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
+ xmlns:pages="clr-namespace:CommunityToolkit.Maui.Sample.Pages"
+ x:Class="CommunityToolkit.Maui.Sample.Pages.Essentials.SpeechToTextPage"
+ xmlns:vm="clr-namespace:CommunityToolkit.Maui.Sample.ViewModels.Essentials"
+ xmlns:essentials="clr-namespace:CommunityToolkit.Maui.Sample.Pages.Essentials"
+ x:TypeArguments="vm:SpeechToTextViewModel"
+ x:DataType="vm:SpeechToTextViewModel"
+ Title="SpeechToText">
+
+ <ContentPage.Resources>
+ <essentials:PickerDisplayConverter x:Key="PickerDisplayConverter" />
+ </ContentPage.Resources>
+
+ <ScrollView>
+ <VerticalStackLayout
+ Spacing="25"
+ Padding="30,0"
+ VerticalOptions="Center">
+
+ <Picker ItemsSource="{Binding Locales}"
+ SelectedItem="{Binding Locale}"
+ ItemDisplayBinding="{Binding ., Converter={StaticResource PickerDisplayConverter}}">
+ </Picker>
+
+ <Editor
+ Text="{Binding Text}"
+ FontSize="18"
+ HorizontalOptions="Center" />
+
+ <Button
+ Text="Play"
+ Command="{Binding PlayCommand}"
+ HorizontalOptions="Center" />
+
+ <Label
+ Text="{Binding RecognitionText}"
+ FontSize="18"
+ HorizontalOptions="Center" />
+
+ <Button
+ Text="Listen"
+ Command="{Binding ListenCommand}"
+ HorizontalOptions="Center" />
+
+ <Button
+ Text="Listen Cancel"
+ Command="{Binding ListenCancelCommand}"
+ HorizontalOptions="Center" />
+
+ </VerticalStackLayout>
+ </ScrollView>
+ 
+</pages:BasePage>

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/SpeechToTextPage.xaml.cs

@@ -0,0 +1,30 @@
+using System.Globalization;
+using CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+namespace CommunityToolkit.Maui.Sample.Pages.Essentials;
+
+public partial class SpeechToTextPage : BasePage<SpeechToTextViewModel>
+{
+public SpeechToTextPage(SpeechToTextViewModel viewModel) : base(viewModel)
+{
+InitializeComponent();
+}
+}
+
+public class PickerDisplayConverter : IValueConverter
+{
+public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
+{
+if (value is Locale locale)
+{
+return $"{locale.Language} {locale.Name}";
+}
+
+return null;
+}
+
+public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
+{
+throw new NotImplementedException();
+}
+}

samples/CommunityToolkit.Maui.Sample/Platforms/Android/AndroidManifest.xml

@@ -4,4 +4,6 @@
 <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
 <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
 <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+
+ <uses-permission android:name="android.permission.RECORD_AUDIO" />
 </manifest>

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/EssentialsGalleryViewModel.cs

@@ -8,7 +8,8 @@ public EssentialsGalleryViewModel()
 : base(new[]
 {
 SectionModel.Create<FileSaverViewModel>("FileSaver", "Allows the user to save files to the filesystem"),
-SectionModel.Create<FolderPickerViewModel>("FolderPicker", "Allows picking folders from the file system")
+SectionModel.Create<FolderPickerViewModel>("FolderPicker", "Allows picking folders from the file system"),
+SectionModel.Create<SpeechToTextViewModel>("SpeechToText", "Allows converts speech to text"),
 })
 {
 }

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/SpeechToTextViewModel.cs

@@ -0,0 +1,67 @@
+using System.Globalization;
+using CommunityToolkit.Maui.Alerts;
+using CommunityToolkit.Maui.SpeechToText;
+using CommunityToolkit.Mvvm.ComponentModel;
+using CommunityToolkit.Mvvm.Input;
+
+namespace CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+public partial class SpeechToTextViewModel : BaseViewModel
+{
+readonly ITextToSpeech textToSpeech;
+readonly ISpeechToText speechToText;
+[ObservableProperty]
+List<Locale>? locales;
+
+[ObservableProperty]
+Locale? locale;
+
+[ObservableProperty]
+string text;
+
+[ObservableProperty]
+string? recognitionText;
+
+public SpeechToTextViewModel(ITextToSpeech textToSpeech, ISpeechToText speechToText)
+{
+this.textToSpeech = textToSpeech;
+this.speechToText = speechToText;
+Locales = new();
+text = @"Welcome to .NET MAUI Community Toolkit!";
+SetLocalesCommand.Execute(null);
+}
+
+[RelayCommand]
+async Task SetLocales()
+{
+Locales = (await textToSpeech.GetLocalesAsync()).ToList();
+Locale = Locales.FirstOrDefault();
+}
+
+[RelayCommand]
+async Task Play(CancellationToken cancellationToken)
+{
+await textToSpeech.SpeakAsync(Text, new SpeechOptions()
+{
+Locale = Locale,
+Pitch = 2,
+Volume = 1
+}, cancellationToken);
+}
+
+[RelayCommand(IncludeCancelCommand = true)]
+async Task Listen(CancellationToken cancellationToken)
+{
+try
+{
+RecognitionText = await speechToText.Listen(CultureInfo.GetCultureInfo(Locale?.Language ?? "en-us"), new Progress<string>(partialText =>
+{
+RecognitionText += partialText + " ";
+}), cancellationToken);
+}
+catch (Exception ex)
+{
+await Toast.Make(ex.Message).Show(CancellationToken.None);
+}
+}
+}

src/CommunityToolkit.Maui.Core/CommunityToolkit.Maui.Core.csproj

@@ -55,5 +55,7 @@
  <PrivateAssets>all</PrivateAssets>
  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
  </PackageReference>
+
+ <PackageReference Include="System.Speech" Version="7.0.0" Condition="'$(TargetFramework)' == 'net7.0-windows10.0.19041.0'" />
  </ItemGroup>
 </Project>

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/ISpeechToText.shared.cs

@@ -0,0 +1,18 @@
+using System.Globalization;
+
+namespace CommunityToolkit.Maui.SpeechToText;
+
+/// <summary>
+/// Allows the user to convert speech to text in real time.
+/// </summary>
+public interface ISpeechToText : IAsyncDisposable
+{
+/// <summary>
+/// Converts speech to text in real time.
+/// </summary>
+/// <param name="culture">Speak language</param>
+/// <param name="recognitionResult">Intermediate convertion result.</param>
+/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
+/// <returns>Final convertion result</returns>
+Task<string> Listen(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken);
+}

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/SpeechToText.shared.cs

@@ -0,0 +1,21 @@
+using System.Globalization;
+
+namespace CommunityToolkit.Maui.SpeechToText;
+
+/// <inheritdoc cref="ISpeechToText"/>
+public static class SpeechToText
+{
+static Lazy<ISpeechToText> defaultImplementation = new(new SpeechToTextImplementation());
+
+/// <summary>
+/// Default implementation of <see cref="ISpeechToText"/>
+/// </summary>
+public static ISpeechToText Default => defaultImplementation.Value;
+
+/// <inheritdoc cref="ISpeechToText.Listen"/>
+public static Task<string> Listen(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken) =>
+Default.Listen(culture, recognitionResult, cancellationToken);
+
+internal static void SetDefault(ISpeechToText implementation) =>
+defaultImplementation = new(implementation);
+}