using System.IO.Compression;
using System.Net.Http.Json;
using System.Text.Json;
using XFEEDToolBox.Core.Models;
namespace XFEEDToolBox.Core.Data;
public sealed record ImportProgress(int Systems, int Stations, int Quotes, string Message);
public sealed class SpanshImporter(MarketRepository repository, HttpClient http)
{
private readonly SemaphoreSlim requestGate = new(1, 1);
private DateTimeOffset lastRequest;
private async Task<JsonDocument> GetAsync(string url, CancellationToken token)
{
await requestGate.WaitAsync(token);
try
{
var delay = TimeSpan.FromSeconds(1.1) - (DateTimeOffset.UtcNow - lastRequest);
if (delay > TimeSpan.Zero) await Task.Delay(delay, token);
lastRequest = DateTimeOffset.UtcNow;
using var requestTimeout = CancellationTokenSource.CreateLinkedTokenSource(token);
requestTimeout.CancelAfter(TimeSpan.FromSeconds(25));
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.UserAgent.ParseAdd("XFEEDToolBox/0.1");
using var response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, requestTimeout.Token);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(requestTimeout.Token);
return await JsonDocument.ParseAsync(stream, cancellationToken: requestTimeout.Token);
}
finally { requestGate.Release(); }
}
public async Task<ImportProgress> ImportSystemAsync(string systemName, IProgress<ImportProgress>? progress = null, CancellationToken token = default)
{
if (string.IsNullOrWhiteSpace(systemName)) throw new ArgumentException("请输入星系名。");
using var lookup = await GetAsync("https://spansh.co.uk/api/systems/field_values/system_names?q=" + Uri.EscapeDataString(systemName.Trim()), token);
var found = lookup.RootElement.Get("min_max").Items().FirstOrDefault(x => x.Text("name").Equals(systemName.Trim(), StringComparison.OrdinalIgnoreCase));
var id64 = found.Long("id64");
if (id64 <= 0) throw new InvalidDataException("Spansh 未找到该星系,请使用完整星系名。");
return await ImportSystemIdAsync(id64, progress, token);
}
private async Task<ImportProgress> ImportSystemIdAsync(long id64, IProgress<ImportProgress>? progress, CancellationToken token)
{
using var data = await GetAsync($"https://spansh.co.uk/api/dump/{id64}", token);
var system = data.RootElement.Get("system");
if (system.ValueKind != JsonValueKind.Object) throw new InvalidDataException("Spansh 返回缺少 system。");
var result = await SaveSystemAsync(system, token);
progress?.Report(result);
return result;
}
public async Task<ImportProgress> ImportNearbyAsync(string origin, int maxSystems, IProgress<ImportProgress>? progress = null, CancellationToken token = default)
{
var total = await ImportSystemAsync(origin, progress, token);
// EDSM's documented sphere endpoint caps its radius at 100 ly; use a modest 50 ly warm start.
using var sphere = await GetAsync("https://www.edsm.net/api-v1/sphere-systems?systemName=" + Uri.EscapeDataString(origin) + "&radius=50&showId=1&showCoordinates=1", token);
var candidates = sphere.RootElement.Items().Where(x => x.Long("id64") > 0 && !x.Text("name").Equals(origin, StringComparison.OrdinalIgnoreCase))
.OrderBy(x => x.Number("distance") ?? double.MaxValue).Take(Math.Clamp(maxSystems, 1, 40) - 1).ToArray();
foreach (var system in candidates)
{
token.ThrowIfCancellationRequested();
try
{
var imported = await ImportSystemIdAsync(system.Long("id64"), null, token);
total = new(total.Systems + 1, total.Stations + imported.Stations, total.Quotes + imported.Quotes, system.Text("name"));
progress?.Report(total);
}
catch (HttpRequestException) { progress?.Report(total with { Message = $"已跳过暂时无法访问的星系:{system.Text("name")}" }); }
}
return total;
}
public async Task<ImportProgress> ImportFileAsync(string path, string? originSystem = null, double radius = double.MaxValue, IProgress<ImportProgress>? progress = null, CancellationToken token = default)
{
await using var file = File.OpenRead(path);
await using Stream stream = path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase) ? new GZipStream(file, CompressionMode.Decompress, leaveOpen: true) : file;
Coordinates? origin = null;
if (!string.IsNullOrWhiteSpace(originSystem)) origin = repository.GetStations(originSystem).FirstOrDefault(s => s.SystemName.Equals(originSystem, StringComparison.OrdinalIgnoreCase))?.Position;
if (radius < double.MaxValue && origin is null) throw new ArgumentException("局部导入需要已知的中心星系坐标。");
var total = new ImportProgress(0, 0, 0, "读取星系数据");
await foreach (var element in JsonSerializer.DeserializeAsyncEnumerable<JsonElement>(stream, cancellationToken: token))
{
token.ThrowIfCancellationRequested();
var position = element.Position("coords");
if (origin != null && (position == null || position.DistanceTo(origin) > radius)) continue;
var count = await SaveSystemAsync(element, token);
total = new(total.Systems + 1, total.Stations + count.Stations, total.Quotes + count.Quotes, element.Text("name"));
if (total.Systems % 25 == 0) progress?.Report(total);
}
progress?.Report(total);
return total;
}
public async Task<ImportProgress> SaveSystemAsync(JsonElement system, CancellationToken token = default)
{
var count = 0; var quotes = 0;
var systemName = system.Text("name");
var position = system.Position("coords");
var security = JsonFields.Security(system.Text("security"));
var stations = system.Get("stations").Items().Concat(system.Get("bodies").Items().SelectMany(b => b.Get("stations").Items())).DistinctBy(x => x.Long("id"));
foreach (var item in stations)
{
token.ThrowIfCancellationRequested();
var market = item.Get("market");
var time = market.Time("updateTime") ?? item.Time("updateTime") ?? system.Time("date");
var type = item.Text("type");
var pads = item.Get("landingPads");
var station = new Station
{
MarketId = item.Long("id"), Name = item.Text("name"), SystemName = systemName, Position = position,
DistanceToArrivalLs = item.Number("distanceToArrival"), Type = type, Security = security,
MaxLandingPad = pads.Long("large") > 0 ? LandingPad.Large : pads.Long("medium") > 0 ? LandingPad.Medium : pads.Long("small") > 0 ? LandingPad.Small : LandingPad.Unknown,
IsPlanetary = type.Contains("Planetary", StringComparison.OrdinalIgnoreCase) || type.Contains("Settlement", StringComparison.OrdinalIgnoreCase),
IsFleetCarrier = type.Contains("Carrier", StringComparison.OrdinalIgnoreCase), RequiresPermit = system.Flag("permitRequired"),
IsRestricted = item.Text("dockingAccess") is "friends" or "squadron" or "none",
UpdatedAt = time ?? DateTimeOffset.MinValue
};
if (station.MarketId <= 0) continue;
await repository.SaveStationAsync(station, token);
count++;
if (time == null || market.Get("commodities").ValueKind != JsonValueKind.Array) continue;
var prohibited = item.Get("prohibitedCommodities").Items().Concat(market.Get("prohibitedCommodities").Items())
.Where(x => x.ValueKind == JsonValueKind.String).Select(x => CommodityCatalog.Key(x.GetString()!)).ToHashSet();
var goods = market.Get("commodities").Items().Select(x => MarketParser.ParseQuote(x, station.MarketId, time.Value, DataSource.Spansh) with
{ Category = x.Text("category") }).Select(x => x with { Illegal = x.Illegal || prohibited.Contains(x.Commodity) }).ToArray();
if (await repository.SaveSnapshotAsync(new(station, goods, time.Value, DataSource.Spansh), token)) quotes += goods.Length;
}
await repository.UpdateSystemAsync(systemName, position, security, token);
return new(1, count, quotes, systemName);
}
}
using System.IO.Compression;
using System.Net.Http.Json;
using System.Text.Json;
using XFEEDToolBox.Core.Models;
namespace XFEEDToolBox.Core.Data;
public sealed record ImportProgress(int Systems, int Stations, int Quotes, string Message);
public sealed class SpanshImporter(MarketRepository repository, HttpClient http)
{
private readonly SemaphoreSlim requestGate = new(1, 1);
private DateTimeOffset lastRequest;
private async Task<JsonDocument> GetAsync(string url, CancellationToken token)
{
await requestGate.WaitAsync(token);
try
{
var delay = TimeSpan.FromSeconds(1.1) - (DateTimeOffset.UtcNow - lastRequest);
if (delay > TimeSpan.Zero) await Task.Delay(delay, token);
lastRequest = DateTimeOffset.UtcNow;
using var requestTimeout = CancellationTokenSource.CreateLinkedTokenSource(token);
requestTimeout.CancelAfter(TimeSpan.FromSeconds(25));
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.UserAgent.ParseAdd("XFEEDToolBox/0.1");
using var response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, requestTimeout.Token);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(requestTimeout.Token);
return await JsonDocument.ParseAsync(stream, cancellationToken: requestTimeout.Token);
}
finally { requestGate.Release(); }
}
public async Task<ImportProgress> ImportSystemAsync(string systemName, IProgress<ImportProgress>? progress = null, CancellationToken token = default)
{
if (string.IsNullOrWhiteSpace(systemName)) throw new ArgumentException("请输入星系名。");
using var lookup = await GetAsync("https://spansh.co.uk/api/systems/field_values/system_names?q=" + Uri.EscapeDataString(systemName.Trim()), token);
var found = lookup.RootElement.Get("min_max").Items().FirstOrDefault(x => x.Text("name").Equals(systemName.Trim(), StringComparison.OrdinalIgnoreCase));
var id64 = found.Long("id64");
if (id64 <= 0) throw new InvalidDataException("Spansh 未找到该星系,请使用完整星系名。");
return await ImportSystemIdAsync(id64, progress, token);
}
private async Task<ImportProgress> ImportSystemIdAsync(long id64, IProgress<ImportProgress>? progress, CancellationToken token)
{
using var data = await GetAsync($"https://spansh.co.uk/api/dump/{id64}", token);
var system = data.RootElement.Get("system");
if (system.ValueKind != JsonValueKind.Object) throw new InvalidDataException("Spansh 返回缺少 system。");
var result = await SaveSystemAsync(system, token);
progress?.Report(result);
return result;
}
public async Task<ImportProgress> ImportNearbyAsync(string origin, int maxSystems, IProgress<ImportProgress>? progress = null, CancellationToken token = default)
{
var total = await ImportSystemAsync(origin, progress, token);
// EDSM's documented sphere endpoint caps its radius at 100 ly; use a modest 50 ly warm start.
using var sphere = await GetAsync("https://www.edsm.net/api-v1/sphere-systems?systemName=" + Uri.EscapeDataString(origin) + "&radius=50&showId=1&showCoordinates=1", token);
var candidates = sphere.RootElement.Items().Where(x => x.Long("id64") > 0 && !x.Text("name").Equals(origin, StringComparison.OrdinalIgnoreCase))
.OrderBy(x => x.Number("distance") ?? double.MaxValue).Take(Math.Clamp(maxSystems, 1, 40) - 1).ToArray();
foreach (var system in candidates)
{
token.ThrowIfCancellationRequested();
try
{
var imported = await ImportSystemIdAsync(system.Long("id64"), null, token);
total = new(total.Systems + 1, total.Stations + imported.Stations, total.Quotes + imported.Quotes, system.Text("name"));
progress?.Report(total);
}
catch (HttpRequestException) { progress?.Report(total with { Message = $"已跳过暂时无法访问的星系:{system.Text("name")}" }); }
}
return total;
}
public async Task<ImportProgress> ImportFileAsync(string path, string? originSystem = null, double radius = double.MaxValue, IProgress<ImportProgress>? progress = null, CancellationToken token = default)
{
await using var file = File.OpenRead(path);
await using Stream stream = path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase) ? new GZipStream(file, CompressionMode.Decompress, leaveOpen: true) : file;
Coordinates? origin = null;
if (!string.IsNullOrWhiteSpace(originSystem)) origin = repository.GetStations(originSystem).FirstOrDefault(s => s.SystemName.Equals(originSystem, StringComparison.OrdinalIgnoreCase))?.Position;
if (radius < double.MaxValue && origin is null) throw new ArgumentException("局部导入需要已知的中心星系坐标。");
var total = new ImportProgress(0, 0, 0, "读取星系数据");
await foreach (var element in JsonSerializer.DeserializeAsyncEnumerable<JsonElement>(stream, cancellationToken: token))
{
token.ThrowIfCancellationRequested();
var position = element.Position("coords");
if (origin != null && (position == null || position.DistanceTo(origin) > radius)) continue;
var count = await SaveSystemAsync(element, token);
total = new(total.Systems + 1, total.Stations + count.Stations, total.Quotes + count.Quotes, element.Text("name"));
if (total.Systems % 25 == 0) progress?.Report(total);
}
progress?.Report(total);
return total;
}
public async Task<ImportProgress> SaveSystemAsync(JsonElement system, CancellationToken token = default)
{
var count = 0; var quotes = 0;
var systemName = system.Text("name");
var position = system.Position("coords");
var security = JsonFields.Security(system.Text("security"));
var stations = system.Get("stations").Items().Concat(system.Get("bodies").Items().SelectMany(b => b.Get("stations").Items())).DistinctBy(x => x.Long("id"));
foreach (var item in stations)
{
token.ThrowIfCancellationRequested();
var market = item.Get("market");
var time = market.Time("updateTime") ?? item.Time("updateTime") ?? system.Time("date");
var type = item.Text("type");
var pads = item.Get("landingPads");
var station = new Station
{
MarketId = item.Long("id"), Name = item.Text("name"), SystemName = systemName, Position = position,
DistanceToArrivalLs = item.Number("distanceToArrival"), Type = type, Security = security,
MaxLandingPad = pads.Long("large") > 0 ? LandingPad.Large : pads.Long("medium") > 0 ? LandingPad.Medium : pads.Long("small") > 0 ? LandingPad.Small : LandingPad.Unknown,
IsPlanetary = type.Contains("Planetary", StringComparison.OrdinalIgnoreCase) || type.Contains("Settlement", StringComparison.OrdinalIgnoreCase),
IsFleetCarrier = type.Contains("Carrier", StringComparison.OrdinalIgnoreCase), RequiresPermit = system.Flag("permitRequired"),
IsRestricted = item.Text("dockingAccess") is "friends" or "squadron" or "none",
UpdatedAt = time ?? DateTimeOffset.MinValue
};
if (station.MarketId <= 0) continue;
await repository.SaveStationAsync(station, token);
count++;
if (time == null || market.Get("commodities").ValueKind != JsonValueKind.Array) continue;
var prohibited = item.Get("prohibitedCommodities").Items().Concat(market.Get("prohibitedCommodities").Items())
.Where(x => x.ValueKind == JsonValueKind.String).Select(x => CommodityCatalog.Key(x.GetString()!)).ToHashSet();
var goods = market.Get("commodities").Items().Select(x => MarketParser.ParseQuote(x, station.MarketId, time.Value, DataSource.Spansh) with
{ Category = x.Text("category") }).Select(x => x with { Illegal = x.Illegal || prohibited.Contains(x.Commodity) }).ToArray();
if (await repository.SaveSnapshotAsync(new(station, goods, time.Value, DataSource.Spansh), token)) quotes += goods.Length;
}
await repository.UpdateSystemAsync(systemName, position, security, token);
return new(1, count, quotes, systemName);
}
}