Fix SocketsHttpHandler proxy auth for 'Negotiate' scheme (#39933) (#39981)

Issue #39887 reported that proxy authentication with 'Negotiate' scheme broke between
.NET Core 3.0 Preview 6 and Preview 7. The base64 blob was no longer using SPNEGO protocol
but instead was always using NTLM. While 'Negotiate' scheme can use either SPNEGO or NTLM,
it should always use SPNEGO if possible. And many enterprises have a setting which requires
it and rejects NTLM protocol.

This issue was caused by PR #38465 which fixed some other SPN issues with Kerberos
authentication. That PR regressed the SPN calculation for the proxy authentication by
using the wrong host name in the SPN. A mismatch of the SPN will cause NTLM to be used
instead of SPNEGO.

The fix is to check if proxy authentication is being used instead of server authentication.
If so, it ignores any 'Host' header and always will use the uri, which in this case is the
uri of the proxy server.

This was tested manually. It is impossible right now to test Kerberos and proxy scenarios in
CI because they require machine configuration to register SPNs in a Windows Active Directory
environment.

This PR will be ported for release/3.0 for ASK mode consideration since it affects a mainline
enterprise scenario.

Fixes #39887

Author: davidsh

src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/AuthenticationHelper.NtAuth.cs

@@ -83,8 +83,10 @@ private static async Task<HttpResponseMessage> SendWithNtAuthAsync(HttpRequestMe
 
                         // Calculate SPN (Service Principal Name) using the host name of the request.
                         // Use the request's 'Host' header if available. Otherwise, use the request uri.
+                        // Ignore the 'Host' header if this is proxy authentication since we need to use
+                        // the host name of the proxy itself for SPN calculation.
                         string hostName;
-                        if (request.HasHeaders && request.Headers.Host != null)
+                        if (!isProxyAuth && request.HasHeaders && request.Headers.Host != null)
                         {
                             // Use the host name without any normalization.
                             hostName = request.Headers.Host;

src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Authentication.cs

@@ -525,6 +525,55 @@ public static IEnumerable<object[]> ServerUsesWindowsAuthentication_MemberData()
                     Configuration.Security.ActiveDirectoryUserPassword,
                     Configuration.Security.ActiveDirectoryName);
 
+        public static IEnumerable<object[]> EchoServersData()
+        {
+            foreach (Uri serverUri in Configuration.Http.EchoServerList)
+            {
+                yield return new object[] { serverUri };
+            }
+        }
+
+        [MemberData(nameof(EchoServersData))]
+        [ConditionalTheory(nameof(IsDomainJoinedServerAvailable))]
+        public async Task Proxy_DomainJoinedProxyServerUsesKerberos_Success(Uri server)
+        {
+            // We skip the test unless it is running on a Windows client machine. That is because only Windows
+            // automatically registers an SPN for HTTP/<hostname> of the machine. This will enable Kerberos to properly
+            // work with the loopback proxy server.
+            if (!PlatformDetection.IsWindows || !PlatformDetection.IsNotWindowsNanoServer)
+            {
+                throw new SkipTestException("Test can only run on domain joined Windows client machine");
+            }
+
+            var options = new LoopbackProxyServer.Options { AuthenticationSchemes = AuthenticationSchemes.Negotiate };
+            using (LoopbackProxyServer proxyServer = LoopbackProxyServer.Create(options))
+            {
+                using (HttpClientHandler handler = CreateHttpClientHandler())
+                using (HttpClient client = CreateHttpClient(handler))
+                {
+                    // Use 'localhost' DNS name for loopback proxy server (instead of IP address) so that the SPN will
+                    // get calculated properly to use Kerberos.
+                    _output.WriteLine(proxyServer.Uri.AbsoluteUri.ToString());
+                    handler.Proxy = new WebProxy("localhost", proxyServer.Uri.Port) { Credentials = DomainCredential };
+
+                    using (HttpResponseMessage response = await client.GetAsync(server))
+                    {
+                        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+
+                        int requestCount = proxyServer.Requests.Count;
+                        
+                        // We expect 2 requests to the proxy server. One without the 'Proxy-Authorization' header and
+                        // one with the header.
+                        Assert.Equal(2, requestCount);
+                        Assert.Equal("Negotiate", proxyServer.Requests[requestCount - 1].AuthorizationHeaderValueScheme);
+                        
+                        // Base64 tokens that use SPNEGO protocol start with 'Y'. NTLM tokens start with 'T'.
+                        Assert.Equal('Y', proxyServer.Requests[requestCount - 1].AuthorizationHeaderValueToken[0]);
+                    }
+                }
+            }
+        }
+
         [ConditionalFact(nameof(IsDomainJoinedServerAvailable))]
         public async Task Credentials_DomainJoinedServerUsesKerberos_Success()
         {