SocketsHttpHandler: Add a switch to allow non-ascii headers

src/Common/tests/System/Net/Http/GenericLoopbackServer.cs

@@ -83,17 +83,21 @@ public class GenericLoopbackOptions
 
     public struct HttpHeaderData
     {
+        public static readonly Encoding Latin1Encoding = Encoding.GetEncoding("ISO-8859-1");
+
         public string Name { get; }
         public string Value { get; }
         public bool HuffmanEncoded { get; }
         public byte[] Raw { get; }
+        public bool Latin1 { get; }
 
-        public HttpHeaderData(string name, string value, bool huffmanEncoded = false, byte[] raw = null)
+        public HttpHeaderData(string name, string value, bool huffmanEncoded = false, byte[] raw = null, bool latin1 = false)
         {
             Name = name;
             Value = value;
             HuffmanEncoded = huffmanEncoded;
             Raw = raw;
+            Latin1 = latin1;
         }
 
         public override string ToString() => Name == null ? "<empty>" : (Name + ": " + (Value ?? string.Empty));

src/Common/tests/System/Net/Http/Http2LoopbackConnection.cs

@@ -381,9 +381,9 @@ private static (int bytesConsumed, string value) DecodeString(ReadOnlySpan<byte>
             }
         }
 
-        private static int EncodeString(string value, Span<byte> headerBlock, bool huffmanEncode)
+        private static int EncodeString(string value, Span<byte> headerBlock, bool huffmanEncode, bool latin1 = false)
         {
-            byte[] data = Encoding.ASCII.GetBytes(value);
+            byte[] data = (latin1 ? HttpHeaderData.Latin1Encoding : Encoding.ASCII).GetBytes(value);
             byte prefix;
 
             if (!huffmanEncode)
@@ -536,12 +536,12 @@ private static (int bytesConsumed, HttpHeaderData headerData) DecodeHeader(ReadO
             }
         }
 
-        public static int EncodeHeader(HttpHeaderData headerData, Span<byte> headerBlock)
+        public static int EncodeHeader(HttpHeaderData headerData, Span<byte> headerBlock, bool latin1 = false)
         {
             // Always encode as literal, no indexing.
             int bytesGenerated = EncodeInteger(0, 0, 0b11110000, headerBlock);
             bytesGenerated += EncodeString(headerData.Name, headerBlock.Slice(bytesGenerated), headerData.HuffmanEncoded);
-            bytesGenerated += EncodeString(headerData.Value, headerBlock.Slice(bytesGenerated), headerData.HuffmanEncoded);
+            bytesGenerated += EncodeString(headerData.Value, headerBlock.Slice(bytesGenerated), headerData.HuffmanEncoded, latin1);
             return bytesGenerated;
         }
 
@@ -680,7 +680,7 @@ public async Task SendResponseHeadersAsync(int streamId, bool endStream = true,
             {
                 foreach (HttpHeaderData headerData in headers)
                 {
-                    bytesGenerated += EncodeHeader(headerData, headerBlock.AsSpan().Slice(bytesGenerated));
+                    bytesGenerated += EncodeHeader(headerData, headerBlock.AsSpan().Slice(bytesGenerated), headerData.Latin1);
                 }
             }
 

src/Common/tests/System/Net/Http/LoopbackServer.cs

@@ -16,6 +16,9 @@ namespace System.Net.Test.Common
 {
     public sealed partial class LoopbackServer : GenericLoopbackServer, IDisposable
     {
+        private static readonly byte[] s_newLineBytes = new byte[] { (byte)'\r', (byte)'\n' };
+        private static readonly byte[] s_colonSpaceBytes = new byte[] { (byte)':', (byte)' ' };
+
         private Socket _listenSocket;
         private Options _options;
         private Uri _uri;
@@ -514,6 +517,16 @@ public string ReadLine()
             }
 
             public async Task<string> ReadLineAsync()
+            {
+                byte[] lineBytes = await ReadLineBytesAsync().ConfigureAwait(false);
+
+                if (lineBytes is null)
+                    return null;
+
+                return Encoding.ASCII.GetString(lineBytes);
+            }
+
+            private async Task<byte[]> ReadLineBytesAsync()
             {
                 int index = 0;
                 int startSearch = _readStart;
@@ -560,14 +573,40 @@ public async Task<string> ReadLineAsync()
                     if (_readBuffer[_readStart + stringLength] == '\n') { stringLength--; }
                     if (_readBuffer[_readStart + stringLength] == '\r') { stringLength--; }
 
-                    string line = System.Text.Encoding.ASCII.GetString(_readBuffer, _readStart, stringLength + 1);
+                    byte[] line = _readBuffer.AsSpan(_readStart, stringLength + 1).ToArray();
                     _readStart = index + 1;
                     return line;
                 }
 
                 return null;
             }
 
+            private async Task<List<byte[]>> ReadRequestHeaderBytesAsync()
+            {
+                var lines = new List<byte[]>();
+
+                byte[] line;
+
+                while (true)
+                {
+                    line = await ReadLineBytesAsync().ConfigureAwait(false);
+
+                    if (line is null || line.Length == 0)
+                    {
+                        break;
+                    }
+
+                    lines.Add(line);
+                }
+
+                if (line == null)
+                {
+                    throw new IOException("Unexpected EOF trying to read request header");
+                }
+
+                return lines;
+            }
+
             public override void Dispose()
             {
                 try
@@ -640,24 +679,24 @@ public async Task<List<string>> ReadRequestHeaderAndSendResponseAsync(HttpStatus
 
             public override async Task<HttpRequestData> ReadRequestDataAsync(bool readBody = true)
             {
-                List<string> headerLines = null;
                 HttpRequestData requestData = new HttpRequestData();
 
-                headerLines = await ReadRequestHeaderAsync().ConfigureAwait(false);
+                List<byte[]> headerLines = await ReadRequestHeaderBytesAsync().ConfigureAwait(false);
 
                 // Parse method and path
-                string[] splits = headerLines[0].Split(' ');
+                string[] splits = Encoding.ASCII.GetString(headerLines[0]).Split(' ');
                 requestData.Method = splits[0];
                 requestData.Path = splits[1];
 
                 // Convert header lines to key/value pairs
                 // Skip first line since it's the status line
-                foreach (var line in headerLines.Skip(1))
+                foreach (byte[] lineBytes in headerLines.Skip(1))
                 {
+                    string line = Encoding.ASCII.GetString(lineBytes);
                     int offset = line.IndexOf(':');
                     string name = line.Substring(0, offset);
                     string value = line.Substring(offset + 1).TrimStart();
-                    requestData.Headers.Add(new HttpHeaderData(name, value));
+                    requestData.Headers.Add(new HttpHeaderData(name, value, raw: lineBytes));
                 }
 
                 if (requestData.Method != "GET")
@@ -737,7 +776,7 @@ public override async Task<Byte[]> ReadRequestBodyAsync()
 
             public override async Task SendResponseAsync(HttpStatusCode? statusCode = HttpStatusCode.OK, IList<HttpHeaderData> headers = null, string content = null, bool isFinal = true, int requestId = 0)
             {
-                string headerString = null;
+                MemoryStream headerBytes = new MemoryStream();
                 int contentLength = -1;
                 bool isChunked = false;
                 bool hasContentLength  = false;
@@ -762,22 +801,36 @@ public override async Task SendResponseAsync(HttpStatusCode? statusCode = HttpSt
                             isChunked = true;
                         }
 
-                        headerString = headerString + $"{headerData.Name}: {headerData.Value}\r\n";
+                        byte[] nameBytes = Encoding.ASCII.GetBytes(headerData.Name);
+                        headerBytes.Write(nameBytes, 0, nameBytes.Length);
+                        headerBytes.Write(s_colonSpaceBytes, 0, s_colonSpaceBytes.Length);
+
+                        byte[] valueBytes = (headerData.Latin1 ? HttpHeaderData.Latin1Encoding : Encoding.ASCII).GetBytes(headerData.Value);
+                        headerBytes.Write(valueBytes, 0, valueBytes.Length);
+                        headerBytes.Write(s_newLineBytes, 0, s_newLineBytes.Length);
                     }
                 }
 
                 bool endHeaders = content != null || isFinal;
                 if (statusCode != null)
                 {
-                    // If we need to send status line, prepped it to headers and possibly add missing headers to the end.
-                    headerString =
+                    byte[] temp = headerBytes.ToArray();
+                    headerBytes.SetLength(0);
+                    byte[] headerStartBytes = Encoding.ASCII.GetBytes(
                         $"HTTP/1.1 {(int)statusCode} {GetStatusDescription((HttpStatusCode)statusCode)}\r\n" +
-                        (!hasContentLength && !isChunked && content != null ? $"Content-length: {content.Length}\r\n" : "") +
-                        headerString +
-                        (endHeaders ? "\r\n" : "");
+                        (!hasContentLength && !isChunked && content != null ? $"Content-length: {content.Length}\r\n" : ""));
+                    headerBytes.Write(headerStartBytes, 0, headerStartBytes.Length);
+                    headerBytes.Write(temp, 0, temp.Length);
+
+                    if (endHeaders)
+                    {
+                        headerBytes.Write(s_newLineBytes, 0, s_newLineBytes.Length);
+                    }
                 }
 
-                await SendResponseAsync(headerString).ConfigureAwait(false);
+                headerBytes.Position = 0;
+                await headerBytes.CopyToAsync(_stream).ConfigureAwait(false);
+
                 if (content != null)
                 {
                     await SendResponseBodyAsync(content, isFinal: isFinal, requestId: requestId).ConfigureAwait(false);

src/System.Net.Http/src/System.Net.Http.csproj

@@ -93,6 +93,7 @@
     <Compile Include="System\Net\Http\SocketsHttpHandler\HPack\HPackEncoder.cs" />
     <Compile Include="System\Net\Http\SocketsHttpHandler\HPack\IntegerEncoder.cs" />
     <Compile Include="System\Net\Http\SocketsHttpHandler\HPack\StaticTable.cs" />
+    <Compile Include="System\Net\Http\SocketsHttpHandler\StaticHttpSettings.cs" />
     <Compile Include="System\Net\Http\SocketsHttpHandler\SystemProxyInfo.cs" />
     <Compile Include="$(CommonPath)\CoreLib\System\IO\StreamHelpers.CopyValidation.cs">
       <Link>Common\CoreLib\System\IO\StreamHelpers.CopyValidation.cs</Link>

src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HPack/HPackEncoder.cs

@@ -217,10 +217,11 @@ private static bool EncodeStringLiteralValue(string value, Span<byte> destinatio
         {
             if (value.Length <= destination.Length)
             {
+                int mask = StaticHttpSettings.EncodingValidationMask;
                 for (int i = 0; i < value.Length; i++)
                 {
-                    char c = value[i];
-                    if ((c & 0xFF80) != 0)
+                    int c = value[i];
+                    if ((c & mask) != 0)
                     {
                         throw new HttpRequestException(SR.net_http_request_invalid_char_encoding);
                     }

src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs

@@ -1147,14 +1147,16 @@ private Task WriteStringAsync(string s)
             if (s.Length <= _writeBuffer.Length - offset)
             {
                 byte[] writeBuffer = _writeBuffer;
-                foreach (char c in s)
+                int mask = StaticHttpSettings.EncodingValidationMask;
+                foreach (int c in s)
                 {
-                    if ((c & 0xFF80) != 0)
+                    if ((c & mask) != 0)
                     {
                         throw new HttpRequestException(SR.net_http_request_invalid_char_encoding);
                     }
                     writeBuffer[offset++] = (byte)c;
                 }
+
                 _writeOffset = offset;
                 return Task.CompletedTask;
             }
@@ -1186,13 +1188,17 @@ private Task WriteAsciiStringAsync(string s)
 
         private async Task WriteStringAsyncSlow(string s)
         {
+            int mask = StaticHttpSettings.EncodingValidationMask;
+
             for (int i = 0; i < s.Length; i++)
             {
-                char c = s[i];
-                if ((c & 0xFF80) != 0)
+                int c = s[i];
+
+                if ((c & mask) != 0)
                 {
                     throw new HttpRequestException(SR.net_http_request_invalid_char_encoding);
                 }
+
                 await WriteByteAsync((byte)c).ConfigureAwait(false);
             }
         }

src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/StaticHttpSettings.cs

@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Net.Http
+{
+    internal static class StaticHttpSettings
+    {
+        private const string AllowLatin1CharactersEnvironmentVariableSettingName = "DOTNET_SYSTEM_NET_HTTP_SOCKETSHTTPHANDLER_ALLOWLATIN1HEADERS";
+        private const string AllowLatin1CharactersAppCtxSettingName = "System.Net.Http.SocketsHttpHandler.AllowLatin1Headers";
+
+        // Disables a validation that checks whether Http headers contain a non-ASCII character.
+        // This is a workaround that has been introduced as a patch specific to the 3.1 branch.
+        // Unlike options in HttpConnectionSettings, this one has a global scope.
+        // Lazy initialization is being used to make sure clients can also use AppContext.SetSwitch() or Environment.SetEnvironmentVariable()
+        // before the first calls to HttpClient API-s.
+        internal static bool AllowLatin1Headers { get; } = GetAllowLatin1HeadersSetting();
+
+        internal static int EncodingValidationMask => AllowLatin1Headers ? 0xFF00 : 0xFF80;
+
+        private static bool GetAllowLatin1HeadersSetting()
+        {
+            // First check for the AppContext switch, giving it priority over the environment variable.
+            if (AppContext.TryGetSwitch(AllowLatin1CharactersAppCtxSettingName, out bool value))
+            {
+                return value;
+            }
+
+            // AppContext switch wasn't used. Check the environment variable.
+            string envVar = Environment.GetEnvironmentVariable(AllowLatin1CharactersEnvironmentVariableSettingName);
+            if (envVar != null && (envVar.Equals("true", StringComparison.OrdinalIgnoreCase) || envVar.Equals("1")))
+            {
+                return true;
+            }
+
+            return false;
+        }
+    }
+}