"""Freeze original Public Export card text for offline MOD search; never access a game process."""
import hashlib
import json
import lzma
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
project_root = Path(__file__).resolve().parent.parent
endpoint = "https://content.warframe.com/PublicExport/"
result = {"schemaVersion": 2, "source": endpoint, "retrievedUtc": datetime.now(timezone.utc).isoformat(), "documents": {}, "cards": {}}
for language in ("en", "zh"):
with urllib.request.urlopen(endpoint + f"index_{language}.txt.lzma", timeout=25) as response:
compressed = response.read(2_000_001)
if not 13 < len(compressed) <= 2_000_000:
raise ValueError("Invalid Public Export index size")
# These streams contain an end marker as well as a declared size. Decode to the
# marker (liblzma rejects that combination in known-size ALONE mode), then verify
# the declared size instead of dropping the final bytes of the index.
decoder = lzma.LZMADecompressor(format=lzma.FORMAT_ALONE)
decoded = decoder.decompress(compressed[:5] + b"\xff" * 8 + compressed[13:], max_length=2_000_001)
if not decoder.eof or len(decoded) != int.from_bytes(compressed[5:13], "little"):
raise ValueError("Incomplete Public Export index")
index = decoded.decode("utf-8-sig")
def strings(value):
if isinstance(value, str):
yield value
elif isinstance(value, list):
for child in value:
yield from strings(child)
elif isinstance(value, dict):
for child in value.values():
yield from strings(child)
for document in ("ExportUpgrades", "ExportRelicArcane"):
filename = next(line.strip() for line in index.splitlines() if line.startswith(f"{document}_{language}.json!"))
with urllib.request.urlopen(endpoint + "Manifest/" + filename, timeout=25) as response:
content = response.read(20_000_001)
if len(content) > 20_000_000:
raise ValueError("Public Export search document is too large")
data = json.loads(content)
result["documents"][document + "_" + language] = {"url": endpoint + "Manifest/" + filename, "sha256": hashlib.sha256(content).hexdigest()}
for table in (document, "ExportModSet"):
for card in data.get(table, []):
if document == "ExportRelicArcane" and table == document and not card.get("levelStats"):
continue # Relic reward tables are not MOD/Arcane card effects.
text = "\n".join(dict.fromkeys(strings({key: card.get(key) for key in ("name", "description", "stats")})))
ranks = ["\n".join(dict.fromkeys(strings(level))) for level in card.get("levelStats", [])]
if text or ranks:
result["cards"].setdefault(card["uniqueName"], {})[language] = {"text": text, "ranks": ranks}
if len(result["cards"]) < 1000:
raise ValueError("Incomplete Public Export search document")
target = project_root / "Server/data/editor/mod-search-text.json"
target.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"cards": len(result["cards"]), "documents": list(result["documents"]), "path": str(target)}))
"""Freeze original Public Export card text for offline MOD search; never access a game process."""
import hashlib
import json
import lzma
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
project_root = Path(__file__).resolve().parent.parent
endpoint = "https://content.warframe.com/PublicExport/"
result = {"schemaVersion": 2, "source": endpoint, "retrievedUtc": datetime.now(timezone.utc).isoformat(), "documents": {}, "cards": {}}
for language in ("en", "zh"):
with urllib.request.urlopen(endpoint + f"index_{language}.txt.lzma", timeout=25) as response:
compressed = response.read(2_000_001)
if not 13 < len(compressed) <= 2_000_000:
raise ValueError("Invalid Public Export index size")
# These streams contain an end marker as well as a declared size. Decode to the
# marker (liblzma rejects that combination in known-size ALONE mode), then verify
# the declared size instead of dropping the final bytes of the index.
decoder = lzma.LZMADecompressor(format=lzma.FORMAT_ALONE)
decoded = decoder.decompress(compressed[:5] + b"\xff" * 8 + compressed[13:], max_length=2_000_001)
if not decoder.eof or len(decoded) != int.from_bytes(compressed[5:13], "little"):
raise ValueError("Incomplete Public Export index")
index = decoded.decode("utf-8-sig")
def strings(value):
if isinstance(value, str):
yield value
elif isinstance(value, list):
for child in value:
yield from strings(child)
elif isinstance(value, dict):
for child in value.values():
yield from strings(child)
for document in ("ExportUpgrades", "ExportRelicArcane"):
filename = next(line.strip() for line in index.splitlines() if line.startswith(f"{document}_{language}.json!"))
with urllib.request.urlopen(endpoint + "Manifest/" + filename, timeout=25) as response:
content = response.read(20_000_001)
if len(content) > 20_000_000:
raise ValueError("Public Export search document is too large")
data = json.loads(content)
result["documents"][document + "_" + language] = {"url": endpoint + "Manifest/" + filename, "sha256": hashlib.sha256(content).hexdigest()}
for table in (document, "ExportModSet"):
for card in data.get(table, []):
if document == "ExportRelicArcane" and table == document and not card.get("levelStats"):
continue # Relic reward tables are not MOD/Arcane card effects.
text = "\n".join(dict.fromkeys(strings({key: card.get(key) for key in ("name", "description", "stats")})))
ranks = ["\n".join(dict.fromkeys(strings(level))) for level in card.get("levelStats", [])]
if text or ranks:
result["cards"].setdefault(card["uniqueName"], {})[language] = {"text": text, "ranks": ranks}
if len(result["cards"]) < 1000:
raise ValueError("Incomplete Public Export search document")
target = project_root / "Server/data/editor/mod-search-text.json"
target.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"cards": len(result["cards"]), "documents": list(result["documents"]), "path": str(target)}))
