using System.IO.Compression;
using System.Text.Json;
using XFEEDToolBox.Core.Models;
namespace XFEEDToolBox.Core.Data;
/// Reads observed market response bodies only. Never replays captured requests or loads credentials.
public static class CaptureImporter
{
public static async Task ImportAsync(string path, MarketRepository repository, IProgress? progress = null, CancellationToken token = default)
{
using var zip = ZipFile.OpenRead(path);
var stations = new HashSet(); var quotes = 0;
foreach (var entry in zip.Entries.Where(e => e.FullName.EndsWith("/events.jsonl", StringComparison.Ordinal)))
{
if (entry.Length > 100 * 1024 * 1024) continue;
var folder = entry.FullName[..^"events.jsonl".Length];
using var reader = new StreamReader(entry.Open());
while (await reader.ReadLineAsync(token) is { } line)
{
token.ThrowIfCancellationRequested();
if (line.Length > 2 * 1024 * 1024) continue;
using var doc = JsonDocument.Parse(line);
var e = doc.RootElement;
// market/trade may return only a delta. Only market/list proves a whole snapshot.
if (e.Text("type") != "http-response" || e.Text("path") != "/2.0/elite/market/list" || e.Long("status") != 200) continue;
var id = GetMarketId(e.Text("query"));
if (id <= 0 && zip.GetEntry(folder + $"decoded/http-response-{e.Long("id"):D6}.query.json") is { Length: < 65536 } queryEntry)
{
await using var queryStream = queryEntry.Open();
using var query = await JsonDocument.ParseAsync(queryStream, cancellationToken: token);
if (query.RootElement.ValueKind == JsonValueKind.Object) id = query.RootElement.Long("marketId");
else foreach (var pair in query.RootElement.Items())
if (pair.ValueKind == JsonValueKind.Array && pair.GetArrayLength() == 2 && pair[0].ValueKind == JsonValueKind.String && pair[0].GetString()!.Equals("marketId", StringComparison.OrdinalIgnoreCase))
{ if (pair[1].ValueKind == JsonValueKind.Number) pair[1].TryGetInt64(out id); else if (pair[1].ValueKind == JsonValueKind.String) long.TryParse(pair[1].GetString(), out id); }
}
var time = e.Time("utc");
if (id <= 0 || time == null) continue;
var body = zip.GetEntry(folder + $"decoded/http-response-{e.Long("id"):D6}.json");
if (body == null || body.Length > 8 * 1024 * 1024) continue;
await using var bodyStream = body.Open();
using var decoded = await JsonDocument.ParseAsync(bodyStream, cancellationToken: token);
var root = decoded.RootElement;
if (root.Get("commodities").ValueKind is not (JsonValueKind.Array or JsonValueKind.Object)) continue;
var station = repository.GetStation(id) ?? new Station { MarketId = id, Name = $"Market #{id}", SystemName = "未识别星系", UpdatedAt = time.Value };
var goods = root.Get("commodities").Items().Select(x => MarketParser.ParseQuote(x, id, time.Value, DataSource.Capture)).ToArray();
if (await repository.SaveSnapshotAsync(new(station, goods, time.Value, DataSource.Capture), token)) { stations.Add(id); quotes += goods.Length; }
progress?.Report(new(0, stations.Count, quotes, "只读导入历史行情;保留原始采集时间"));
}
}
return new(0, stations.Count, quotes, "抓包行情导入完成");
}
private static long GetMarketId(string query)
{
foreach (var part in query.TrimStart('?').Split('&'))
{
var pair = part.Split('=', 2);
if (pair.Length == 2 && Uri.UnescapeDataString(pair[0]).Equals("marketId", StringComparison.OrdinalIgnoreCase) && long.TryParse(Uri.UnescapeDataString(pair[1]), out var id)) return id;
}
return 0;
}
}