Implement TranscodingStream, a streaming equivalent of Encoding.Convert

src/libraries/Common/tests/TestUtilities/System/IO/StreamExtensions.cs

@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace System.IO
+{
+ public static class StreamExtensions
+ {
+ public static async Task<int> ReadByteAsync(this Stream stream, CancellationToken cancellationToken = default)
+ {
+ byte[] buffer = new byte[1];
+
+ int numBytesRead = await stream.ReadAsync(buffer, 0, 1, cancellationToken);
+ if (numBytesRead == 0)
+ {
+ return -1; // EOF
+ }
+
+ return buffer[0];
+ }
+ }
+}

src/libraries/Common/tests/TestUtilities/TestUtilities.csproj

@@ -11,6 +11,7 @@
  <ItemGroup>
  <Compile Include="System\AdminHelpers.cs" />
  <Compile Include="System\AssertExtensions.cs" />
+ <Compile Include="System\IO\StreamExtensions.cs" />
  <Compile Include="System\RetryHelper.cs" />
  <Compile Include="System\Buffers\BoundedMemory.cs" />
  <Compile Include="System\Buffers\BoundedMemory.Creation.cs" />

src/libraries/System.Net.Http.Json/src/System.Net.Http.Json.csproj

@@ -9,8 +9,6 @@
  <Compile Include="System\Net\Http\Json\HttpClientJsonExtensions.Put.cs" />
  <Compile Include="System\Net\Http\Json\HttpContentJsonExtensions.cs" />
  <Compile Include="System\Net\Http\Json\JsonContent.cs" />
- <Compile Include="System\Net\Http\Json\TranscodingReadStream.cs" />
- <Compile Include="System\Net\Http\Json\TranscodingWriteStream.cs" />
  </ItemGroup>
  <ItemGroup Condition="'$(TargetFramework)' == '$(NetCoreAppCurrent)'">
  <Compile Include="System\Net\Http\Json\JsonContent.netcoreapp.cs" />
@@ -21,6 +19,8 @@
  </ItemGroup>
  <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
  <Compile Include="System\ArraySegmentExtensions.netstandard.cs" />
+ <Compile Include="System\Net\Http\Json\TranscodingReadStream.cs" />
+ <Compile Include="System\Net\Http\Json\TranscodingWriteStream.cs" />
  <Reference Include="System.Buffers" />
  </ItemGroup>
  <ItemGroup>

src/libraries/System.Net.Http.Json/src/System/Net/Http/Json/HttpContentJsonExtensions.cs

@@ -38,7 +38,11 @@ public static Task<T> ReadFromJsonAsync<T>(this HttpContent content, JsonSeriali
  // Wrap content stream into a transcoding stream that buffers the data transcoded from the sourceEncoding to utf-8.
  if (sourceEncoding != null && sourceEncoding != Encoding.UTF8)
  {
+#if NETCOREAPP
+ contentStream = Encoding.CreateTranscodingStream(contentStream, innerStreamEncoding: sourceEncoding, outerStreamEncoding: Encoding.UTF8);
+#else
  contentStream = new TranscodingReadStream(contentStream, sourceEncoding);
+#endif
  }
 
  using (contentStream)
@@ -54,7 +58,11 @@ private static async Task<T> ReadFromJsonAsyncCore<T>(HttpContent content, Encod
  // Wrap content stream into a transcoding stream that buffers the data transcoded from the sourceEncoding to utf-8.
  if (sourceEncoding != null && sourceEncoding != Encoding.UTF8)
  {
+#if NETCOREAPP
+ contentStream = Encoding.CreateTranscodingStream(contentStream, innerStreamEncoding: sourceEncoding, outerStreamEncoding: Encoding.UTF8);
+#else
  contentStream = new TranscodingReadStream(contentStream, sourceEncoding);
+#endif
  }
 
  using (contentStream)

src/libraries/System.Net.Http.Json/src/System/Net/Http/Json/JsonContent.cs

@@ -67,6 +67,19 @@ private async Task SerializeToStreamAsyncCore(Stream targetStream, CancellationT
  // Wrap provided stream into a transcoding stream that buffers the data transcoded from utf-8 to the targetEncoding.
  if (targetEncoding != null && targetEncoding != Encoding.UTF8)
  {
+#if NETCOREAPP
+ Stream transcodingStream = Encoding.CreateTranscodingStream(targetStream, targetEncoding, Encoding.UTF8, leaveOpen: true);
+ try
+ {
+ await JsonSerializer.SerializeAsync(transcodingStream, Value, ObjectType, _jsonSerializerOptions, cancellationToken).ConfigureAwait(false);
+ }
+ finally
+ {
+ // DisposeAsync will flush any partial write buffers. In practice our partial write
+ // buffers should be empty as we expect JsonSerializer to emit only well-formed UTF-8 data.
+ await transcodingStream.DisposeAsync().ConfigureAwait(false);
+ }
+#else
  using (TranscodingWriteStream transcodingStream = new TranscodingWriteStream(targetStream, targetEncoding))
  {
  await JsonSerializer.SerializeAsync(transcodingStream, Value, ObjectType, _jsonSerializerOptions, cancellationToken).ConfigureAwait(false);
@@ -75,6 +88,7 @@ private async Task SerializeToStreamAsyncCore(Stream targetStream, CancellationT
  // acceptable to Flush a Stream (multiple times) prior to completion.
  await transcodingStream.FinalWriteAsync(cancellationToken).ConfigureAwait(false);
  }
+#endif
  }
  else
  {

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -870,6 +870,7 @@
  <Compile Include="$(MSBuildThisFileDirectory)System\Text\StringBuilder.cs" />
  <Compile Include="$(MSBuildThisFileDirectory)System\Text\StringBuilder.Debug.cs" Condition="'$(Configuration)' == 'Debug'" />
  <Compile Include="$(MSBuildThisFileDirectory)System\Text\StringRuneEnumerator.cs" />
+ <Compile Include="$(MSBuildThisFileDirectory)System\Text\TranscodingStream.cs" />
  <Compile Include="$(MSBuildThisFileDirectory)System\Text\TrimType.cs" />
  <Compile Include="$(MSBuildThisFileDirectory)System\Text\Unicode\GraphemeClusterBreakType.cs" />
  <Compile Include="$(MSBuildThisFileDirectory)System\Text\Unicode\TextSegmentationUtility.cs" />

src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs

@@ -4,6 +4,7 @@
 
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Runtime.InteropServices;
 using System.Runtime.Serialization;
 
@@ -1040,6 +1041,50 @@ value is Encoding that &&
  public override int GetHashCode() =>
  _codePage + this.EncoderFallback.GetHashCode() + this.DecoderFallback.GetHashCode();
 
+ /// <summary>
+ /// Creates a <see cref="Stream"/> which serves to transcode data between an inner <see cref="Encoding"/>
+ /// and an outer <see cref="Encoding"/>, similar to <see cref="Convert"/>.
+ /// </summary>
+ /// <param name="innerStream">The <see cref="Stream"/> to wrap.</param>
+ /// <param name="innerStreamEncoding">The <see cref="Encoding"/> associated with <paramref name="innerStream"/>.</param>
+ /// <param name="outerStreamEncoding">The <see cref="Encoding"/> associated with the <see cref="Stream"/> returned
+ /// by this method.</param>
+ /// <param name="leaveOpen"><see langword="true"/> if disposing the <see cref="Stream"/> returned by this method
+ /// should <em>not</em> dispose <paramref name="innerStream"/>.</param>
+ /// <returns>A <see cref="Stream"/> which transcodes the contents of <paramref name="innerStream"/>
+ /// as <paramref name="outerStreamEncoding"/>.</returns>
+ /// <remarks>
+ /// The returned <see cref="Stream"/>'s <see cref="Stream.CanRead"/> and <see cref="Stream.CanWrite"/> properties
+ /// will reflect whether <paramref name="innerStream"/> is readable or writable. If <paramref name="innerStream"/>
+ /// is full-duplex, the returned <see cref="Stream"/> will be as well. However, the returned <see cref="Stream"/>
+ /// is not seekable, even if <paramref name="innerStream"/>'s <see cref="Stream.CanSeek"/> property returns <see langword="true"/>.
+ /// </remarks>
+ public static Stream CreateTranscodingStream(Stream innerStream, Encoding innerStreamEncoding, Encoding outerStreamEncoding, bool leaveOpen = false)
+ {
+ if (innerStream is null)
+ {
+ throw new ArgumentNullException(nameof(innerStream));
+ }
+
+ if (innerStreamEncoding is null)
+ {
+ throw new ArgumentNullException(nameof(innerStreamEncoding));
+ }
+
+ if (outerStreamEncoding is null)
+ {
+ throw new ArgumentNullException(nameof(outerStreamEncoding));
+ }
+
+ // We can't entirely optimize away the case where innerStreamEncoding == outerStreamEncoding. For example,
+ // the Encoding might perform a lossy conversion when it sees invalid data, so we still need to call it
+ // to perform basic validation. It's also possible that somebody subclassed one of the built-in types
+ // like ASCIIEncoding or UTF8Encoding and is running some non-standard logic. If this becomes a bottleneck
+ // we can consider targeted optimizations in a future release.
+
+ return new TranscodingStream(innerStream, innerStreamEncoding, outerStreamEncoding, leaveOpen);
+ }
+
  internal virtual char[] GetBestFitUnicodeToBytesData() =>
  // Normally we don't have any best fit data.
  Array.Empty<char>();