using System.IO.Compression; using System.Text.Json; using XFEEDToolBox.Core.Models; namespace XFEEDToolBox.Core.Data; /// Reads observed market response bodies only. Never replays captured requests or loads credentials. public static class CaptureImporter { public static async Task ImportAsync(string path, MarketRepository repository, IProgress? progress = null, CancellationToken token = default) { using var zip = ZipFile.OpenRead(path); var stations = new HashSet(); var quotes = 0; foreach (var entry in zip.Entries.Where(e => e.FullName.EndsWith("/events.jsonl", StringComparison.Ordinal))) { if (entry.Length > 100 * 1024 * 1024) continue; var folder = entry.FullName[..^"events.jsonl".Length]; using var reader = new StreamReader(entry.Open()); while (await reader.ReadLineAsync(token) is { } line) { token.ThrowIfCancellationRequested(); if (line.Length > 2 * 1024 * 1024) continue; using var doc = JsonDocument.Parse(line); var e = doc.RootElement; // market/trade may return only a delta. Only market/list proves a whole snapshot. if (e.Text("type") != "http-response" || e.Text("path") != "/2.0/elite/market/list" || e.Long("status") != 200) continue; var id = GetMarketId(e.Text("query")); if (id <= 0 && zip.GetEntry(folder + $"decoded/http-response-{e.Long("id"):D6}.query.json") is { Length: < 65536 } queryEntry) { await using var queryStream = queryEntry.Open(); using var query = await JsonDocument.ParseAsync(queryStream, cancellationToken: token); if (query.RootElement.ValueKind == JsonValueKind.Object) id = query.RootElement.Long("marketId"); else foreach (var pair in query.RootElement.Items()) if (pair.ValueKind == JsonValueKind.Array && pair.GetArrayLength() == 2 && pair[0].ValueKind == JsonValueKind.String && pair[0].GetString()!.Equals("marketId", StringComparison.OrdinalIgnoreCase)) { if (pair[1].ValueKind == JsonValueKind.Number) pair[1].TryGetInt64(out id); else if (pair[1].ValueKind == JsonValueKind.String) long.TryParse(pair[1].GetString(), out id); } } var time = e.Time("utc"); if (id <= 0 || time == null) continue; var body = zip.GetEntry(folder + $"decoded/http-response-{e.Long("id"):D6}.json"); if (body == null || body.Length > 8 * 1024 * 1024) continue; await using var bodyStream = body.Open(); using var decoded = await JsonDocument.ParseAsync(bodyStream, cancellationToken: token); var root = decoded.RootElement; if (root.Get("commodities").ValueKind is not (JsonValueKind.Array or JsonValueKind.Object)) continue; var station = repository.GetStation(id) ?? new Station { MarketId = id, Name = $"Market #{id}", SystemName = "未识别星系", UpdatedAt = time.Value }; var goods = root.Get("commodities").Items().Select(x => MarketParser.ParseQuote(x, id, time.Value, DataSource.Capture)).ToArray(); if (await repository.SaveSnapshotAsync(new(station, goods, time.Value, DataSource.Capture), token)) { stations.Add(id); quotes += goods.Length; } progress?.Report(new(0, stations.Count, quotes, "只读导入历史行情;保留原始采集时间")); } } return new(0, stations.Count, quotes, "抓包行情导入完成"); } private static long GetMarketId(string query) { foreach (var part in query.TrimStart('?').Split('&')) { var pair = part.Split('=', 2); if (pair.Length == 2 && Uri.UnescapeDataString(pair[0]).Equals("marketId", StringComparison.OrdinalIgnoreCase) && long.TryParse(Uri.UnescapeDataString(pair[1]), out var id)) return id; } return 0; } }