using System.Diagnostics; using System.Runtime.InteropServices; using System.Text; using System.Text.Json; using Microsoft.Data.Sqlite; using XFEEDToolBox.Core.Data; using XFEEDToolBox.Core.Models; if(args.Length<2)throw new ArgumentException("Usage: MarketDatabaseProbe database report"); var results=new List(); var path=Path.GetFullPath(args[0]); var report=Path.GetFullPath(args[1]); var repository=new MarketRepository(path); SqliteConnection Open(string file,SqliteOpenMode mode){var connection=new SqliteConnection(new SqliteConnectionStringBuilder{DataSource=file,Mode=mode,Pooling=false,DefaultTimeout=30}.ToString());connection.Open();return connection;} void Run(string name,Func action){var timer=Stopwatch.StartNew();try{var value=action();results.Add(new{name,ok=true,ms=timer.Elapsed.TotalMilliseconds,value});}catch(Exception error){results.Add(new{name,ok=false,ms=timer.Elapsed.TotalMilliseconds,error=error.ToString(),code=(error as SqliteException)?.SqliteErrorCode,extendedCode=(error as SqliteException)?.SqliteExtendedErrorCode});}File.WriteAllText(report,JsonSerializer.Serialize(new{path,pid=Environment.ProcessId,localAppData=Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),results},new JsonSerializerOptions{WriteIndented=true}));Console.WriteLine(name);} Directory.CreateDirectory(Path.GetDirectoryName(report)!); Run("file",()=>{using var file=File.OpenHandle(path,FileMode.Open,FileAccess.Read,FileShare.ReadWrite|FileShare.Delete);var final=new StringBuilder(4096);Native.GetFinalPathNameByHandle(file.DangerousGetHandle(),final,4096,0);return new{requested=path,final=final.ToString(),size=new FileInfo(path).Length};}); Run("sqlite",()=>{using var c=new SqliteConnection(new SqliteConnectionStringBuilder{DataSource=path,Mode=SqliteOpenMode.ReadOnly,Pooling=false}.ToString());c.Open();using var q=c.CreateCommand();q.CommandText="SELECT sqlite_version()";return q.ExecuteScalar();}); if(args.Contains("--events")) { foreach(var mode in new[]{SqliteOpenMode.ReadOnly,SqliteOpenMode.ReadWrite}) foreach(var sql in new[]{"PRAGMA quick_check(5)","SELECT count(*) FROM game_events","SELECT count(*) FROM game_events WHERE event='LoadGame'","SELECT id,time_ticks,event FROM game_events ORDER BY time_ticks DESC,id DESC LIMIT 5"}) Run(mode+" "+sql,()=>{using var c=Open(path,mode);using var q=c.CreateCommand();q.CommandText=sql;using var reader=q.ExecuteReader();var rows=new List();while(reader.Read()){var row=new object[reader.FieldCount];reader.GetValues(row);rows.Add(row);}return rows;}); SqliteConnection.ClearAllPools();return; } foreach(var mode in new[]{SqliteOpenMode.ReadOnly,SqliteOpenMode.ReadWrite,SqliteOpenMode.ReadWriteCreate})Run("mode "+mode,()=>{using var c=Open(path,mode);using var q=c.CreateCommand();q.CommandText="PRAGMA quick_check(1)";var health=q.ExecuteScalar();q.CommandText="SELECT count(*) FROM quotes";object? count=null;string? error=null;try{count=q.ExecuteScalar();}catch(Exception e){error=e.Message;}return new{health,count,error};}); Run("sqlite-file-handles",()=>{using var c=new SqliteConnection(new SqliteConnectionStringBuilder{DataSource=path,Mode=SqliteOpenMode.ReadOnly,Pooling=false}.ToString());c.Open();var process=Process.Start(new ProcessStartInfo("powershell.exe"){UseShellExecute=false,CreateNoWindow=true,RedirectStandardOutput=true,ArgumentList={"-NoProfile","-File",Path.Combine(Environment.CurrentDirectory,"research","Inspect-MarketHandles.ps1"),"-TargetProcessId",Environment.ProcessId.ToString()}})!;var output=process.StandardOutput.ReadToEnd();process.WaitForExit();return output;}); if(Array.IndexOf(args,"--backup") is var backupIndex && backupIndex>=0)Run("backup",()=>{using var source=Open(path,SqliteOpenMode.ReadOnly);using var destination=Open(Path.GetFullPath(args[backupIndex+1]),SqliteOpenMode.ReadWriteCreate);source.BackupDatabase(destination);return new{target=args[backupIndex+1]};}); if(Array.IndexOf(args,"--restore") is var restoreIndex && restoreIndex>=0)Run("restore",()=>{using var source=Open(Path.GetFullPath(args[restoreIndex+1]),SqliteOpenMode.ReadOnly);using var check=source.CreateCommand();check.CommandText="PRAGMA integrity_check";if((string?)check.ExecuteScalar()!="ok")throw new InvalidDataException("Restore source is not valid");using var destination=Open(path,SqliteOpenMode.ReadWrite);source.BackupDatabase(destination);using var checkpoint=destination.CreateCommand();checkpoint.CommandText="PRAGMA wal_checkpoint(TRUNCATE)";using var result=checkpoint.ExecuteReader();result.Read();return new{source=args[restoreIndex+1],checkpointBusy=result.GetInt32(0),walFrames=result.GetInt32(1),checkpointed=result.GetInt32(2)};}); if(Array.IndexOf(args,"--install-verified") is var installIndex && installIndex>=0)Run("InstallVerifiedBackup",()=> { using var source=Open(Path.GetFullPath(args[installIndex+1]),SqliteOpenMode.ReadWrite);using var check=source.CreateCommand();check.CommandText="PRAGMA integrity_check(1)"; if((string?)check.ExecuteScalar()!="ok")throw new InvalidDataException("Candidate failed integrity checking"); using var destination=Open(repository.VerifiedBackupPath,SqliteOpenMode.ReadWriteCreate);source.BackupDatabase(destination); using var verify=destination.CreateCommand();verify.CommandText="PRAGMA integrity_check(1)"; if((string?)verify.ExecuteScalar()!="ok")throw new InvalidDataException("Installed backup failed integrity checking"); verify.CommandText="SELECT count(*) FROM quotes";return new{installed=repository.VerifiedBackupPath,quotes=verify.ExecuteScalar()}; }); if(args.Contains("--initialize"))Run("Initialize",()=>{repository.Initialize();return repository.LastRecovery;}); if(args.Contains("--save-verified"))Run("SaveVerifiedBackup",()=>{repository.SaveVerifiedBackupAsync().GetAwaiter().GetResult();return repository.VerifiedBackupPath;}); if(Array.IndexOf(args,"--stress-seconds") is var stressIndex && stressIndex>=0)Run("stress",()=> { if(!path.StartsWith(Path.GetFullPath("artifacts"),StringComparison.OrdinalIgnoreCase))throw new InvalidOperationException("Stress writes require an isolated workspace artifact database"); repository.Initialize(); using var cancel=new CancellationTokenSource(TimeSpan.FromSeconds(int.Parse(args[stressIndex+1]))); var sourceRows=repository.Search("",null,limit:1000).GroupBy(x=>x.Station.MarketId).Where(x=>x.Count()>1).Take(24).ToArray(); if(sourceRows.Length<2)throw new InvalidDataException("Need real market data for workload"); long reads=0,writes=0;var errors=new System.Collections.Concurrent.ConcurrentQueue(); var workers=Enumerable.Range(0,3).Select(worker=>Task.Run(async()=> { var counter=0; try{while(!cancel.IsCancellationRequested) { if(worker==0){var station=sourceRows[counter++%sourceRows.Length];await repository.SaveSnapshotAsync(new(station.First().Station,station.Select(x=>x.Quote).ToArray(),DateTimeOffset.UtcNow,DataSource.Eddn));Interlocked.Increment(ref writes);await Task.Delay(30);} else{repository.Statistics();repository.GetStations();repository.Search("",worker==1?sourceRows[counter++%sourceRows.Length].Key:null,sort:(MarketSort)(counter%6));Interlocked.Increment(ref reads);await Task.Delay(25);} }}catch(Exception error){errors.Enqueue(error.ToString());} })).ToArray(); Task.WaitAll(workers);repository.SaveVerifiedBackupAsync().GetAwaiter().GetResult();SqliteConnection.ClearAllPools(); var restart=new MarketRepository(path);restart.Initialize();var stats=restart.Statistics(); if(errors.Count>0||writes<10||reads<10)throw new InvalidDataException(JsonSerializer.Serialize(new{writes,reads,errors})); return new{writes,reads,errors,stats.Stations,stats.Quotes}; }); Run("Statistics",()=>{var value=repository.Statistics();return new{value.Stations,value.Quotes,value.Latest};}); Run("Search",()=>repository.Search("",null).Count); Run("GetStations",()=>repository.GetStations().Count); Run("GetStations full galaxy",()=>repository.GetStations(limit:10000).Count); foreach(var sort in Enum.GetValues()) Run("Search "+sort,()=>repository.Search("",null,sort:sort).Count); Run("quick_check",()=>{using var c=new SqliteConnection(new SqliteConnectionStringBuilder{DataSource=path,Mode=SqliteOpenMode.ReadOnly,Pooling=false}.ToString());c.Open();using var q=c.CreateCommand();q.CommandText="PRAGMA quick_check(5)";using var r=q.ExecuteReader();var checks=new List();while(r.Read())checks.Add(r.GetString(0));return checks;}); SqliteConnection.ClearAllPools(); static class Native{[DllImport("kernel32.dll",CharSet=CharSet.Unicode)]internal static extern uint GetFinalPathNameByHandle(IntPtr handle,StringBuilder path,uint size,uint flags);}