XFEEDToolBox

"""Extract original galaxy appearance floats from pinned local frozen data.

No game process, network request, or guessed palette is used. The configuration
root is linked independently by the original constructors and several named
vector references. Only this small appearance subset is written for review.
"""
import hashlib
import json
import math
import struct
from pathlib import Path

out=Path('artifacts/galaxy-query-research/palette-resources')
key='86ec03fba621526059fc6812faca6f10adcb63cb'
f=(out/(key+'.fres')).read_bytes();b=(out/(key+'.indexed1')).read_bytes()
pool_count=struct.unpack_from('<I',f,0x34)[0]
buffer_count=struct.unpack_from('<I',f,0x38)[0]-struct.unpack_from('<I',f,0x40)[0]
root_count=struct.unpack_from('<I',f,0x20)[0]
cursor=0;total=0
while total<pool_count:
    kind,count=struct.unpack_from('<HH',b,cursor);cursor+=4;total+=count
    if kind not in (2,3,4):raise ValueError('Unexpected original frozen-pool kind')
if total!=pool_count:raise ValueError('Original pool count changed')
pools=[dict(size=struct.unpack_from('<I',b,cursor+i*32+8)[0],offset=struct.unpack_from('<I',b,cursor+i*32+12)[0]) for i in range(pool_count)]
cursor+=pool_count*32
buffers=[struct.unpack_from('<IIHHIQQ',b,cursor+i*32) for i in range(buffer_count)]
cursor+=buffer_count*32+sum(row[3] for row in buffers)*8
roots=[struct.unpack_from('<IIII',b,cursor+i*16) for i in range(root_count)]
cursor+=root_count*16;references={}
while cursor+16<=len(b):
    source,source_offset,target,target_offset=struct.unpack_from('<IIII',b,cursor)
    if source>=pool_count or target>=pool_count or source_offset>=pools[source]['size'] or target_offset>=pools[target]['size']:break
    if (source,source_offset) in references:raise ValueError('Duplicate frozen reference')
    references[source,source_offset]=(target,target_offset);cursor+=16
# Original stream's 16-byte delimiter precedes the contiguous frozen pools.
if b[cursor:cursor+16]!=bytes.fromhex('0000000000000000ffbf7f3f08040201'):raise ValueError('Frozen stream delimiter changed')
data_start=cursor+16
def at(pointer):
    pool,offset=pointer
    if pool>=pool_count or offset<0 or offset>=pools[pool]['size']:raise ValueError('Frozen pointer out of bounds')
    return data_start+pools[pool]['offset']+offset
def string(pointer):
    start=at(pointer);end=b.find(b'\0',start,start+512)
    if end<start:raise ValueError('Unterminated original appearance label')
    return b[start:end].decode('utf-8')
def floats(pointer,count):
    values=struct.unpack_from('<'+'f'*count,b,at(pointer))
    if not all(math.isfinite(value) for value in values):raise ValueError('Non-finite original appearance value')
    return list(values)

# Federation row is (411,6168); its table starts (411,6144). The only
# reference to that table is (286,6296) = shared configuration +98.
config=(286,6144);base=at(config)
if references[(286,6144+0x98)]!=(411,6144):raise ValueError('Original allegiance vector changed')
tables={}
for field in (0x98,0xA8,0xB8,0xC8,0xD8,0xE8,0x1F0):
    target=references.get((config[0],config[1]+field));count=struct.unpack_from('<I',b,base+field+8)[0]
    if target is None or not 0<count<=64:raise ValueError('Original category vector changed')
    rows=[]
    for index in range(count):
        pointer=(target[0],target[1]+index*24)
        name=references.get(pointer)
        if name is None:raise ValueError(f'Original category name is missing at field {field:x}, index {index}, pointer {pointer}')
        rows.append(dict(index=index,key=string(name),rgb=floats((pointer[0],pointer[1]+8),3),pool=pointer[0],offset=pointer[1]))
    tables[hex(field)]=rows
integer_mappings={}
for field in (0x50,0x60):
    target=references[(config[0],config[1]+field)]
    count=struct.unpack_from('<I',b,base+field+8)[0]
    integer_mappings[hex(field)]=[list(struct.unpack_from('<II',b,at((target[0],target[1]+index*8)))) for index in range(count)]
commodity=[]
target=references[(config[0],config[1]+0x1E0)]
for index in range(struct.unpack_from('<I',b,base+0x1E8)[0]):
    pointer=(target[0],target[1]+index*32)
    commodity.append(dict(key=string(references[pointer]),index=struct.unpack_from('<I',b,at(pointer)+8)[0],
        rgba=floats((pointer[0],pointer[1]+12),4),lastInteger=struct.unpack_from('<I',b,at(pointer)+28)[0]))
palette_pointer=references[(config[0],config[1]+0x28)]
palette_count=struct.unpack_from('<I',b,base+0x30)[0]
common_palette=[floats((palette_pointer[0],palette_pointer[1]+index*12),3) for index in range(palette_count)]
result=dict(sourceResource=key,sourceSha256='29a90189d0bb238c88d09dcc0f15ff751ea29aa208b1a90894ef3b47f8942e5f',
    frozenSha256=hashlib.sha256(b).hexdigest(),gameImageSha256='6891f70a021fa9f3a0e00458c284f48d1226e68ae7fc02ad48081aed47857e93',
    gameProcessAccess=False,gameCalls=0,networkRequests=0,configuration=dict(pool=config[0],offset=config[1],fileOffset=base),
    poolDataOffset=data_start,rootCount=root_count,poolCount=pool_count,
    commonPalette=common_palette,tables=tables,commodity=commodity,
    integerMappingsUnconfirmedMeaning=integer_mappings,
    population=dict(allowedMaximum=floats((286,6144+0x14),1)[0],sizeMinimum=floats((286,6144+0x70),1)[0],sizeMaximum=floats((286,6144+0x74),1)[0],
        colourMinimum=floats((286,6144+0x7C),3),colourMaximum=floats((286,6144+0x88),3)),
    defaultSize=floats((286,6144+0x1C),1)[0],
    colonisation=[dict(index=index,rgb=floats((286,6144+field),3),field=hex(field)) for index,field in enumerate((0x14C,0x158,0x164,0x170,0x17C))],
    unknownFields={hex(field):floats((286,6144+field),1)[0] for field in (0x10,0x18,0x20,0x38,0x3C,0x40,0x44,0x48,0x4C,0x118,0x11C,0x120,0x128,0x12C,0x130,0x134,0x138,0x13C,0x140,0x144,0x148)})
(out/'original-appearance.json').write_text(json.dumps(result,indent=2)+'\n',encoding='utf-8')
print(json.dumps(result,indent=2))