using System.Runtime.InteropServices; namespace XFESystem.NativeKernel; // This ABI contains integers only. It reports discovery, never an initialized // graphics/compute device. Unknown BAR sizes are zero, not guessed from a model. [StructLayout(LayoutKind.Sequential, Pack = 8)] public struct NativeGpuBarDiscovery { public ulong Address, Size; public uint Index, Flags, SizeSource, Reserved; } [StructLayout(LayoutKind.Sequential, Pack = 8)] public struct NativeGpuDeviceDiscovery { public uint AbiVersion, RecordBytes, Segment, Bus, Device, Function; public uint VendorId, DeviceId, ClassCode, Revision, HeaderType, Command, Status; public uint PcieCapabilityOffset, CapabilityStatus, ExtendedStatus, RebarStatus, Flags; public ulong EcamPhysical; public NativeGpuBarDiscovery Bar0, Bar1, Bar2, Bar3, Bar4, Bar5; } /// Bounded, read-only PCI discovery on the independent BSP kernel. public static unsafe class GpuHardwareBridge { public const uint AbiVersion = 1, RecordLimit = 8; public const uint Memory = 1, Io = 2, Width64 = 4, Prefetch = 8, UpperHalf = 16; public const uint Unassigned = 32, Invalid = 64, SizeKnown = 128; public const uint SourceUnknown = 0, SourceRebar = 1, SourceFirmware = 2; public const uint Sm120Candidate = 1; // CapabilityStatus: 0 absent, 1 PCIe found, 2 malformed. // ExtendedStatus: 0 unavailable, 1 valid, 2 malformed, 3 mapping/identity failed. // RebarStatus: 0 unavailable, 1 absent, 2 validated current size, 3 malformed. // McfgStatus: 0 absent, 1 validated table, 2 rejected ACPI/MCFG. const ulong PhysicalLimit = 1UL << 52; static NativeGpuDeviceDiscovery* _records; static uint _state, _count, _matched, _mcfgStatus, _mcfgEntries, _unmapFailures; static ulong _configurationReads, _ecamMaps; static byte* _mcfg; // Reuse the network bridge's IRQ-serialized configuration mechanism. Its // CF8 address-port write is not a PCI configuration-data write. This probe // neither imports the PCI-write ABI nor enables bus mastering/memory decode. [DllImport("xfe_native", EntryPoint = "xfe_network_pci_read")] static extern uint ReadPci(uint bus, uint device, uint function, uint offset); static uint Read(uint bus, uint device, uint function, uint offset) { _configurationReads++; return ReadPci(bus, device, function, offset); } public static void Initialize(NativeBootInfo* boot) { if (_state != 0) return; _state = 1; NativeGpuFirmware.Initialize(boot); Serial.Line("XFE_GPU_DISCOVERY_BEGIN|Version|1|ReadOnly|1|BarAccess|0|GpuAcceleration|0|Cuda|0"); ulong physical = PhysicalMemory.Allocate(); if (physical == 0 || sizeof(NativeGpuDeviceDiscovery) * RecordLimit > 4096) { _state = 3; Serial.Line("XFE_GPU_DISCOVERY_UNAVAILABLE|Reason|RecordStorage"); PrintSummary(); return; } _records = (NativeGpuDeviceDiscovery*)PhysicalMemory.ToVirtual(physical); byte* mcfg = null; uint mcfgEntries = 0; _mcfgStatus = LocateMcfg(boot, &mcfg, &mcfgEntries); _mcfg = mcfg; _mcfgEntries = mcfgEntries; Serial.Write("XFE_GPU_DISCOVERY_PLATFORM|McfgStatus|"); Serial.Number(_mcfgStatus); Serial.Write("|McfgEntries|"); Serial.Number(_mcfgEntries); Serial.Line("|Segment|0|RecordLimit|8"); uint* configuration = stackalloc uint[1024]; for (uint bus = 0; bus < 256; bus++) for (uint device = 0; device < 32; device++) { uint identity = Read(bus, device, 0, 0); if ((identity & 65535) == 65535 || (identity & 65535) == 0) continue; uint header = Read(bus, device, 0, 12); uint functions = (header & 0x800000) != 0 ? 8u : 1u; for (uint function = 0; function < functions; function++) { uint id = function == 0 ? identity : Read(bus, device, function, 0); if ((id & 65535) != 0x10de) continue; uint classCode = Read(bus, device, function, 8); if ((classCode >> 24) != 3) continue; _matched++; if (_count == RecordLimit) continue; for (uint word = 0; word < 64; word++) configuration[word] = Read(bus, device, function, word * 4); NativeGpuDeviceDiscovery* record = _records + _count; if (!DecodeConfiguration(configuration, 64, record)) continue; record->Bus = bus; record->Device = device; record->Function = function; if (record->CapabilityStatus == 1 && _mcfgStatus == 1 && TryEcamAddress(boot, _mcfg, _mcfgEntries, bus, device, function, &physical)) { ulong flags = Native.IrqSave(); ulong mapping = VirtualMemory.MapDeviceReadOnly(physical, 4096); Native.IrqRestore(flags); if (mapping != 0) { _ecamMaps++; uint ecamIdentity = Native.MmioRead32(mapping), ecamClass = Native.MmioRead32(mapping + 8); uint ecamHeader = Native.MmioRead32(mapping + 12), ecamCommand = Native.MmioRead32(mapping + 4); _configurationReads += 4; bool identityMatches = ecamIdentity == configuration[0] && ecamClass == configuration[2] && ((ecamHeader ^ configuration[3]) & 0xff0000) == 0 && ((ecamCommand ^ configuration[1]) & 7) == 0; // Two identical GPUs may have the same vendor/class. // Cross-check their assigned BAR configuration too; // these are PCI config reads, not BAR-aperture reads. for (uint bar = 0; bar < 6; bar++) { uint assigned = Native.MmioRead32(mapping + 16 + bar * 4); _configurationReads++; if (assigned != configuration[4 + bar]) identityMatches = false; } if (identityMatches) { for (uint word = 64; word < 1024; word++) configuration[word] = Native.MmioRead32(mapping + word * 4); _configurationReads += 960; DecodeConfiguration(configuration, 1024, record); record->Bus = bus; record->Device = device; record->Function = function; record->EcamPhysical = physical; } else record->ExtendedStatus = 3; flags = Native.IrqSave(); bool unmapped = VirtualMemory.UnmapDevice(mapping, 4096); Native.IrqRestore(flags); if (!unmapped) _unmapFailures++; } else record->ExtendedStatus = 3; } NativeGpuFirmware.Apply(boot, record); PrintRecord(_count, record); _count++; } } _state = 2; PrintSummary(); } // A fixed integer ABI: 0 version, 1 record bytes, 2 saved count, 3 matched // count, 4 state (0 not run/1 running/2 complete/3 unavailable), 5 MCFG // status, 6 configuration reads, 7 GPU acceleration=0, 8 CUDA=0, // 9 capacity, 10 omitted, 11 ECAM page maps, 12 unmap failures, 13 BAR bytes, // 14..19 additive firmware diagnostic version/stage/count/address/match/BAR0. [UnmanagedCallersOnly(EntryPoint = "xfe_gpu_discovery_info")] public static ulong Info(uint field) { if (field == 0) return AbiVersion; if (field == 1) return (ulong)sizeof(NativeGpuDeviceDiscovery); if (field == 2) return _count; if (field == 3) return _matched; if (field == 4) return _state; if (field == 5) return _mcfgStatus; if (field == 6) return _configurationReads; if (field == 9) return RecordLimit; if (field == 10) return _matched - _count; if (field == 11) return _ecamMaps; if (field == 12) return _unmapFailures; if (field == 13) return (ulong)sizeof(NativeGpuBarDiscovery); if (field >= 14 && field <= 19) return NativeGpuFirmware.Diagnostic(field - 14); return 0; } [UnmanagedCallersOnly(EntryPoint = "xfe_gpu_discovery_record")] public static int CopyRecord(uint index, NativeGpuDeviceDiscovery* destination, uint bytes) { if (_state != 2 || destination == null || index >= _count || bytes < sizeof(NativeGpuDeviceDiscovery)) return 0; *destination = _records[index]; return 1; } // Pure snapshot parsing, shared verbatim with the host safety checks. // Constants follow PCI/PCIe definitions, including the read-only ReBAR // capability/control layout in edk2 MdePkg/IndustryStandard/PciExpress21.h. public static bool DecodeConfiguration(uint* config, uint words, NativeGpuDeviceDiscovery* result) { if (result == null) return false; *result = default; if (config == null || (words != 64 && words != 1024) || (config[0] & 65535) != 0x10de || (config[2] >> 24) != 3) return false; result->AbiVersion = AbiVersion; result->RecordBytes = (uint)sizeof(NativeGpuDeviceDiscovery); result->VendorId = config[0] & 65535; result->DeviceId = config[0] >> 16; result->ClassCode = config[2] >> 8; result->Revision = config[2] & 255; result->HeaderType = (config[3] >> 16) & 255; result->Command = config[1] & 65535; result->Status = config[1] >> 16; if ((result->HeaderType & 127) != 0) { result->CapabilityStatus = 2; return true; } NativeGpuBarDiscovery* bars = &result->Bar0; for (uint i = 0; i < 6; i++) { bars[i].Index = i; uint low = config[4 + i]; if (low == 0) continue; if (low == 0xffffffff) { bars[i].Flags = Invalid; continue; } if ((low & 1) != 0) { bars[i].Address = low & ~3u; bars[i].Flags = Io; continue; } uint type = low & 6; bars[i].Flags = Memory | ((low & 8) != 0 ? Prefetch : 0); bars[i].Address = low & ~15u; if (type == 4 && i < 5) { bars[i].Address |= (ulong)config[5 + i] << 32; bars[i].Flags |= Width64; bars[i + 1].Index = i + 1; bars[i + 1].Flags = UpperHalf; } else if (type != 0) bars[i].Flags |= Invalid; if (bars[i].Address == 0) bars[i].Flags |= Unassigned; if (bars[i].Address >= PhysicalLimit) bars[i].Flags |= Invalid; if (type == 4 && i < 5) i++; } if ((result->Status & 16) != 0) { uint offset = config[13] & 255; ulong visited = 0; while (offset != 0) { if (offset < 64 || offset > 252 || (offset & 3) != 0 || (visited & (1UL << (int)(offset / 4))) != 0) { result->CapabilityStatus = 2; break; } visited |= 1UL << (int)(offset / 4); uint capability = config[offset / 4]; if ((capability & 255) == 0x10) { if (result->PcieCapabilityOffset != 0 || ((capability >> 16) & 15) == 0) { result->CapabilityStatus = 2; break; } result->PcieCapabilityOffset = offset; result->CapabilityStatus = 1; } offset = (capability >> 8) & 255; } } if (result->CapabilityStatus == 1 && result->DeviceId == 0x2b85) result->Flags = Sm120Candidate; if (words == 1024 && result->CapabilityStatus == 1) DecodeExtended(config, result); return true; } static void DecodeExtended(uint* config, NativeGpuDeviceDiscovery* result) { result->ExtendedStatus = 1; result->RebarStatus = 1; uint* visited = stackalloc uint[32]; for (uint i = 0; i < 32; i++) visited[i] = 0; uint offset = 256, rebarOffset = 0; while (offset != 0) { if (offset < 256 || offset > 4092 || (offset & 3) != 0) { RejectExtended(result); return; } uint word = offset / 4, bit = 1u << (int)(word & 31); if ((visited[word / 32] & bit) != 0) { RejectExtended(result); return; } visited[word / 32] |= bit; uint header = config[word]; if (offset == 256 && (header == 0 || header == 0xffffffff)) return; if ((header & 65535) == 0 || (header & 65535) == 65535 || ((header >> 16) & 15) == 0) { RejectExtended(result); return; } uint next = header >> 20; if ((header & 65535) == 0x15) { if (rebarOffset != 0 || ((header >> 16) & 15) != 1 || offset > 4084) { RejectExtended(result); return; } rebarOffset = offset; uint entries = (config[(offset + 8) / 4] >> 5) & 7; if (entries == 0 || entries > 6 || 4 + entries * 8 > 4096 - offset || (next > offset && next < offset + 4 + entries * 8)) { RejectExtended(result); return; } uint seenBars = 0; NativeGpuBarDiscovery* bars = &result->Bar0; for (uint i = 0; i < entries; i++) { uint capability = config[(offset + 4 + i * 8) / 4], control = config[(offset + 8 + i * 8) / 4]; uint index = control & 7, sizeCode = (control >> 8) & 63; // PCIe's newer definition extends supported-size bits into // Control[31:16]. Never accept a size absent from that mask. ulong supported = (capability >> 4) | ((ulong)(control >> 16) << 28); if (index >= 6 || (seenBars & (1u << (int)index)) != 0 || (capability & 15) != 0 || (control & 0xc018) != 0 || sizeCode > 43 || (supported & (1UL << (int)sizeCode)) == 0 || !ApplyResourceSize(bars + index, bars[index].Address, 1UL << (int)(20 + sizeCode), SourceRebar)) { RejectExtended(result); return; } seenBars |= 1u << (int)index; } result->RebarStatus = 2; } offset = next; } } static void RejectExtended(NativeGpuDeviceDiscovery* result) { result->ExtendedStatus = 2; result->RebarStatus = 3; NativeGpuBarDiscovery* bars = &result->Bar0; for (uint i = 0; i < 6; i++) { bars[i].Flags &= ~SizeKnown; bars[i].Size = 0; bars[i].SizeSource = SourceUnknown; } } // Firmware/resource providers may add a size only for the exact assigned // memory BAR. A conflicting independent size invalidates both claims. public static bool ApplyResourceSize(NativeGpuBarDiscovery* bar, ulong address, ulong bytes, uint source) { if (bar == null || source == SourceUnknown || source > SourceFirmware || bar->Address != address || address == 0 || bytes < 16 || (bytes & (bytes - 1)) != 0 || (address & (bytes - 1)) != 0 || address >= PhysicalLimit || bytes > PhysicalLimit - address || (bar->Flags & (Memory | Io | UpperHalf | Unassigned | Invalid)) != Memory || ((bar->Flags & Width64) == 0 && (address >= 0x100000000UL || bytes > 0x100000000UL - address))) return false; if ((bar->Flags & SizeKnown) != 0 && bar->Size != bytes) { bar->Flags = (bar->Flags & ~SizeKnown) | Invalid; bar->Size = 0; bar->SizeSource = SourceUnknown; return false; } bar->Size = bytes; bar->SizeSource = source; bar->Flags |= SizeKnown; return true; } static uint U32(byte* p) => (uint)p[0] | ((uint)p[1] << 8) | ((uint)p[2] << 16) | ((uint)p[3] << 24); static ulong U64(byte* p) => U32(p) | ((ulong)U32(p + 4) << 32); static bool Checksum(byte* p, uint bytes) { uint sum = 0; for (uint i = 0; i < bytes; i++) sum += p[i]; return (sum & 255) == 0; } internal static bool Covered(NativeBootInfo* boot, ulong address, ulong bytes, bool acpi) { if (boot == null || boot->MemoryEntries == null || boot->MemoryEntryCount > 4096 || address == 0 || bytes == 0 || address >= PhysicalLimit || bytes > PhysicalLimit - address) return false; bool covered = false; for (ulong i = 0; i < boot->MemoryEntryCount; i++) { MemoryMapEntry* entry = boot->MemoryEntries[i]; if (entry == null || entry->Base > ~0UL - entry->Length) return false; bool kind = acpi ? (entry->Kind == 2 || entry->Kind == 3 || entry->Kind == 8) : entry->Kind == 1; if (entry->Length != 0 && address < entry->Base + entry->Length && entry->Base < address + bytes && !kind) return false; if (kind && address >= entry->Base && address - entry->Base <= entry->Length && bytes <= entry->Length - (address - entry->Base)) covered = true; } return covered; } static byte* AcpiPointer(NativeBootInfo* boot, ulong physical, uint bytes) { if (!Covered(boot, physical, bytes, true) || boot->Hhdm == 0 || boot->Hhdm > ~0UL - physical || bytes - 1 > ~0UL - (boot->Hhdm + physical)) return null; return (byte*)(boot->Hhdm + physical); } static byte* AcpiTable(NativeBootInfo* boot, ulong physical, uint signature) { byte* header = AcpiPointer(boot, physical, 36); if (header == null || U32(header) != signature) return null; uint length = U32(header + 4); if (length < 36 || length > 65536) return null; byte* table = AcpiPointer(boot, physical, length); return table != null && Checksum(table, length) ? table : null; } internal static uint LocateMcfg(NativeBootInfo* boot, byte** result, uint* entries) { *result = null; *entries = 0; if (boot == null || boot->Rsdp == 0) return 0; // This kernel requests Limine base revision 6: RSDP is an HHDM pointer, // ACPI table links are physical. Do not assume a first-4GiB direct map. if (boot->Hhdm == 0 || boot->Rsdp < boot->Hhdm) return 2; ulong rsdpPhysical = boot->Rsdp - boot->Hhdm; byte* rsdp = AcpiPointer(boot, rsdpPhysical, 20); if (rsdp == null || U64(rsdp) != 0x2052545020445352UL || !Checksum(rsdp, 20)) return 2; ulong rootPhysical = U32(rsdp + 16); uint stride = 4, signature = 0x54445352; if (rsdp[15] >= 2) { rsdp = AcpiPointer(boot, rsdpPhysical, 36); if (rsdp == null) return 2; uint length = U32(rsdp + 20); if (length < 36 || length > 4096 || AcpiPointer(boot, rsdpPhysical, length) == null || !Checksum(rsdp, length)) return 2; if (U64(rsdp + 24) != 0) { rootPhysical = U64(rsdp + 24); stride = 8; signature = 0x54445358; } } byte* root = AcpiTable(boot, rootPhysical, signature); if (root == null) return 2; uint payload = U32(root + 4) - 36; if ((payload % stride) != 0 || payload / stride > 1024) return 2; byte* found = null; for (uint offset = 36; offset < payload + 36; offset += stride) { ulong address = stride == 8 ? U64(root + offset) : U32(root + offset); byte* header = AcpiPointer(boot, address, 36); if (header == null || U32(header) != 0x4746434d) continue; byte* table = AcpiTable(boot, address, 0x4746434d); if (found != null || table == null || !ValidateMcfg(table)) return 2; found = table; } if (found == null) return 0; *result = found; *entries = (U32(found + 4) - 44) / 16; return 1; } internal static bool ValidateMcfg(byte* table) { if (table == null) return false; uint length = U32(table + 4); if (length < 60 || length > 44 + 32 * 16 || (length - 44) % 16 != 0 || U64(table + 36) != 0) return false; for (uint offset = 44; offset < length; offset += 16) { byte* entry = table + offset; ulong baseAddress = U64(entry); uint start = entry[10], end = entry[11]; ulong ending = ((ulong)end + 1) << 20; if (baseAddress == 0 || (baseAddress & 0xfffff) != 0 || baseAddress >= PhysicalLimit || ending > PhysicalLimit - baseAddress || start > end || U32(entry + 12) != 0) return false; uint segment = (uint)entry[8] | ((uint)entry[9] << 8); for (uint prior = 44; prior < offset; prior += 16) { byte* old = table + prior; uint oldSegment = (uint)old[8] | ((uint)old[9] << 8); if (segment == oldSegment && start <= old[11] && old[10] <= end) return false; ulong thisStart = baseAddress + ((ulong)start << 20), oldStart = U64(old) + ((ulong)old[10] << 20); ulong thisEnd = baseAddress + ending, oldEnd = U64(old) + (((ulong)old[11] + 1) << 20); if (thisStart < oldEnd && oldStart < thisEnd) return false; } } return true; } internal static bool TryEcamAddress(NativeBootInfo* boot, byte* table, uint entries, uint bus, uint device, uint function, ulong* physical) { *physical = 0; if (table == null || !ValidateMcfg(table) || entries == 0 || entries != (U32(table + 4) - 44) / 16 || bus > 255 || device > 31 || function > 7) return false; for (uint i = 0; i < entries; i++) { byte* entry = table + 44 + i * 16; if (entry[8] != 0 || entry[9] != 0 || bus < entry[10] || bus > entry[11]) continue; ulong start = U64(entry) + ((ulong)entry[10] << 20), bytes = ((ulong)entry[11] + 1 - entry[10]) << 20; // A checksum-valid MCFG alone is not permission to read arbitrary // physical memory. Require the complete allocation to be explicitly // reserved (not RAM, ACPI data or the GOP framebuffer) by firmware. if (!Covered(boot, start, bytes, false)) return false; *physical = U64(entry) + ((ulong)bus << 20) + ((ulong)device << 15) + ((ulong)function << 12); return true; } return false; } static void PrintRecord(uint index, NativeGpuDeviceDiscovery* record) { Serial.Write("XFE_GPU_DEVICE|Index|"); Serial.Number(index); Serial.Write("|Bus|"); Serial.Number(record->Bus); Serial.Write("|Device|"); Serial.Number(record->Device); Serial.Write("|Function|"); Serial.Number(record->Function); Serial.Write("|Vendor|"); Serial.Hex(record->VendorId); Serial.Write("|DeviceId|"); Serial.Hex(record->DeviceId); Serial.Write("|Class|"); Serial.Hex(record->ClassCode); Serial.Write("|Sm120Candidate|"); Serial.Number(record->Flags & Sm120Candidate); Serial.Write("|PcieStatus|"); Serial.Number(record->CapabilityStatus); Serial.Write("|ExtendedStatus|"); Serial.Number(record->ExtendedStatus); Serial.Write("|RebarStatus|"); Serial.Number(record->RebarStatus); Serial.Write("|Ecam|"); Serial.Hex(record->EcamPhysical); Serial.Line(""); NativeGpuBarDiscovery* bars = &record->Bar0; for (uint i = 0; i < 6; i++) { Serial.Write("XFE_GPU_BAR|DeviceIndex|"); Serial.Number(index); Serial.Write("|Index|"); Serial.Number(i); Serial.Write("|Address|"); Serial.Hex(bars[i].Address); Serial.Write("|Bytes|"); Serial.Number(bars[i].Size); Serial.Write("|Flags|"); Serial.Number(bars[i].Flags); Serial.Write("|SizeSource|"); Serial.Number(bars[i].SizeSource); Serial.Write("|SizeKnown|"); Serial.Number((bars[i].Flags & SizeKnown) != 0 ? 1UL : 0UL); Serial.Line("|MappingDenied|1|Reason|DiscoveryOnly"); } } static void PrintSummary() { Serial.Write("XFE_GPU_DISCOVERY_END|State|"); Serial.Number(_state); Serial.Write("|Devices|"); Serial.Number(_count); Serial.Write("|MatchedDevices|"); Serial.Number(_matched); Serial.Write("|Omitted|"); Serial.Number(_matched - _count); Serial.Write("|ConfigReads|"); Serial.Number(_configurationReads); Serial.Write("|EcamPageMaps|"); Serial.Number(_ecamMaps); Serial.Write("|UnmapFailures|"); Serial.Number(_unmapFailures); Serial.Line("|BarMaps|0|BarReads|0|ConfigWrites|0|GpuAcceleration|0|Cuda|0"); } }