using System.Net; using System.Text; namespace XFESystem.Browser; /// /// Explicit HttpClient adapter for a runtime with working sockets, TLS, clock /// and trust roots. It is never selected as an implicit bare-metal fallback. /// public sealed class HttpWebTransport : IWebRequestTransport, IWebTransportCapabilities, IWebCookieStore, IBrowserRandomSource, IDisposable { public bool IsNative => false; public bool SupportsHttp => true; public bool SupportsHttps => true; public void FillRandom(Span bytes) => System.Security.Cryptography.RandomNumberGenerator.Fill(bytes); private readonly BrowserCookieStore cookies = new(); public string ReadDocumentCookies(Uri address) => cookies.ReadDocumentCookies(address); public void WriteDocumentCookie(Uri address, string value) => cookies.WriteDocumentCookie(address, value); private readonly HttpClient client = new(new SocketsHttpHandler { AllowAutoRedirect = false, UseCookies = false, AutomaticDecompression = DecompressionMethods.All }) { Timeout = Timeout.InfiniteTimeSpan }; public WebResponse Get(Uri address, BrowserLimits limits) => GetCore(address, limits, null); public WebResponse Get(Uri address, BrowserLimits limits, IReadOnlyDictionary headers) => GetCore(address, limits, BrowserRequestHeaders.Copy(headers)); private WebResponse GetCore(Uri address, BrowserLimits limits, IReadOnlyDictionary? headers) { ArgumentNullException.ThrowIfNull(address); ArgumentNullException.ThrowIfNull(limits); limits.Validate(); BrowserService.ValidateWebAddress(address); using var cancellation = new CancellationTokenSource(limits.NetworkTimeout); var target = address; for (var redirects = 0; ; redirects++) { using var request = new HttpRequestMessage(HttpMethod.Get, target); request.Headers.UserAgent.ParseAdd(BrowserIdentity.UserAgent); if (headers?.ContainsKey("accept") != true) request.Headers.Accept.ParseAdd("text/html,application/xhtml+xml,text/css,application/javascript,image/png,image/jpeg,image/svg+xml,*/*;q=0.5"); if (headers?.ContainsKey("accept-language") != true) request.Headers.AcceptLanguage.ParseAdd("zh-CN,zh;q=0.9,en;q=0.8"); foreach (var field in headers ?? new Dictionary()) { if (!request.Headers.TryAddWithoutValidation(field.Key, field.Value)) { request.Content ??= new ByteArrayContent([]); request.Content.Headers.TryAddWithoutValidation(field.Key, field.Value); } } if (cookies.RequestHeader(target) is { Length: > 0 } cookie) request.Headers.TryAddWithoutValidation("Cookie", cookie); using var response = client.Send(request, HttpCompletionOption.ResponseHeadersRead, cancellation.Token); var status = (int)response.StatusCode; bool noSniff = response.Headers.TryGetValues("X-Content-Type-Options", out var sniffHeaders) && sniffHeaders.FirstOrDefault()?.Split(',')[0].Trim().Equals("nosniff", StringComparison.OrdinalIgnoreCase) == true; if (response.Headers.TryGetValues("Set-Cookie", out var setCookies)) foreach (string header in setCookies) cookies.Receive(target, header); if (status is 301 or 302 or 303 or 307 or 308) { if (redirects >= limits.MaximumRedirects) throw new InvalidOperationException("The redirect limit was reached."); var location = response.Headers.Location ?? throw new InvalidOperationException("The redirect has no Location header."); var next = location.IsAbsoluteUri ? location : new Uri(target, location); BrowserService.ValidateWebAddress(next); if (headers is not null && next.GetLeftPart(UriPartial.Authority) != address.GetLeftPart(UriPartial.Authority)) throw new NotSupportedException("Cross-origin script redirects require CORS support."); if (target.Scheme == "https" && next.Scheme != "https") throw new InvalidOperationException("HTTPS cannot redirect to HTTP."); target = next; continue; } if (response.Content.Headers.ContentLength > (long)limits.MaximumHtmlCharacters * 4) throw new BrowserLimitException("The HTTP body exceeds its byte limit."); var charset = response.Content.Headers.ContentType?.CharSet?.Trim('"') ?? "utf-8"; Encoding encoding; try { encoding = Encoding.GetEncoding(charset); } catch (ArgumentException) { throw new NotSupportedException("The response charset is unsupported: " + charset); } using var stream = response.Content.ReadAsStream(cancellation.Token); if (response.Content.Headers.ContentType?.MediaType?.StartsWith("image/", StringComparison.OrdinalIgnoreCase) == true) { using var bytes = new MemoryStream(); var chunk = new byte[8192]; while (true) { int size = stream.ReadAsync(chunk, cancellation.Token).AsTask().GetAwaiter().GetResult(); if (size == 0) break; if (size > (long)limits.MaximumHtmlCharacters * 4 - bytes.Length) throw new BrowserLimitException("The HTTP image exceeds its byte limit."); bytes.Write(chunk, 0, size); } return new(target, status, response.Content.Headers.ContentType.ToString(), "", target.Scheme == "https", BinaryBody: bytes.ToArray(), NoSniff: noSniff); } using var reader = new StreamReader(stream, encoding, true, 4_096); var text = new BoundedText(limits.MaximumHtmlCharacters); var buffer = new char[4_096]; while (true) { var read = reader.ReadAsync(buffer.AsMemory(), cancellation.Token).AsTask().GetAwaiter().GetResult(); if (read == 0) break; text.Append(new string(buffer, 0, read)); } // HttpClient's default TLS handler validates trust chain, hostname and validity. // No custom certificate callback is installed by this adapter. return new(target, status, response.Content.Headers.ContentType?.ToString() ?? string.Empty, text.ToString(), target.Scheme == "https", response.Headers.TryGetValues("Content-Security-Policy", out var policies) ? string.Join(", ", policies) : null, NoSniff: noSniff); } } public void Dispose() => client.Dispose(); }