"""Offline negative controls for the Bing evidence reader; not guest acceptance."""
import copy
import hashlib
import json
from pathlib import Path
import subprocess
import tempfile
import time
import unittest
from PIL import Image, ImageDraw
from IndependentBingEvidence import (HOME, NIC_ARGUMENT, NAT_ARGUMENT, REMOTE, RESPONSE, json_blocks,
matching_presentation, native_network_profile, network_arguments, presentation_identity,
native_navigation_failure, remote_text_pixels, run_bing, stable_screenshot_presentation, validate_network_arguments,
validate_remote_page, validate_response_chain, validate_loaded_home, photograph_pixels, validate_search_results, validate_search_images, validate_wide_home)
from IndependentBingEvidence import validate_visible_images, validate_citation_layout, chevron_pixels, validate_scroll_coverage, validate_search_navigation
from IndependentBingEvidence import capture_matching_presentation
ROOT = Path(__file__).resolve().parents[2]
TEMP_ROOT = ROOT / 'build' / 'IndependentKernel' / 'ToolGuardTests'
def temporary_directory():
TEMP_ROOT.mkdir(parents=True, exist_ok=True)
return tempfile.TemporaryDirectory(dir=TEMP_ROOT)
def response(uri=HOME, status=200, location=None):
from urllib.parse import urlsplit
url = urlsplit(uri)
target = (url.path or '/') + ('?' + url.query if url.query else '')
headers = 'HTTP/1.1 ' + str(status) + ' OK\r\nContent-Type: text/html\r\n'
if location:
headers += 'Location: ' + location + '\r\n'
return {'Value': {'Kind': 'NativeResponse', 'RequestId': 1, 'RequestUri': uri, 'ResponseUri': uri,
'StatusCode': status, 'ContentType': 'text/html; charset=utf-8', 'CertificateValidated': True,
'NativeTransport': True, 'RequestHeaders': 'GET ' + target + ' HTTP/1.1\r\nHost: ' + url.hostname + '\r\n\r\n',
'ResponseHeaders': headers + '\r\n', 'BodyBytes': 1000, 'BodySha256': 'a' * 64,
'HtmlTitle': 'Bing', 'HasHtml': True, 'HasHead': True, 'Location': location}}
def remote():
return {'Address': HOME, 'Title': '必应', 'BodySha256': 'a' * 64, 'CertificateValidated': True,
'DocumentInitializationComplete': True, 'ScriptBudgetExceeded': False, 'PendingResources':0, 'NavigationLinks':[{'MenuId':7,'Painted':True,'ContainerBounds':{'X':0,'Y':5,'Width':200,'Height':20},'Bounds':{'X':i*25,'Y':5,'Width':20,'Height':15},'Text':'tab'} for i in range(5)], 'MissingVisibleImages': [], 'DocumentHeight': 100,
'SnapshotSequence': 12, 'ScrollY': 0, 'ViewportWidth': 200, 'ViewportHeight': 100,
'SearchInputs': [{'Name': 'q', 'Bounds': {'X': 30, 'Y': 20, 'Width': 120, 'Height': 20}}],
'TextRuns': [{'Text': text, 'Bounds': {'X': 10, 'Y': y, 'Width': 60, 'Height': 12}, 'Color': '#e0e0e0'} for text, y in (('Bing', 4), ('Search', 60))],
'LoadedStyles': [{'Address': 'https://www.bing.com/real.css', 'Characters': 1000}],
'DeclaredStyleLinks': ['https://www.bing.com/real.css'], 'InlineStyleCount': 2}
class IndependentBingEvidenceTests(unittest.TestCase):
def test_all_photograph_pixel_call_sites_bind_to_the_actual_helper(self):
import ast,inspect
source=(ROOT/'Tools/IndependentKernel/IndependentBingEvidence.py').read_text(encoding='utf-8')
signature=inspect.signature(photograph_pixels);calls=[]
for node in ast.walk(ast.parse(source)):
if isinstance(node,ast.Call) and isinstance(node.func,ast.Name) and node.func.id=='photograph_pixels':
signature.bind(*([None]*len(node.args)),**{item.arg:None for item in node.keywords});calls.append(node)
self.assertEqual(3,len(calls))
with self.assertRaises(TypeError):signature.bind(None,None,None,False)
def test_declared_navigation_requires_real_paint_after_clipping(self):
page=remote();self.assertEqual(5,len(validate_search_navigation(page)['Links']))
page['NavigationLinks'][2]['Painted']=False
with self.assertRaises(AssertionError):validate_search_navigation(page)
del page['NavigationLinks']
with self.assertRaises(AssertionError):validate_search_navigation(page)
def test_visible_images_and_citation_text_cannot_be_replaced_by_success_flags(self):
page=remote();validate_visible_images(page)
page['TextRuns']=[{'Text':'url','Ancestors':'strong|cite|div','ListItemId':4,'Bounds':{'X':10,'Y':20,'Width':80,'Height':20}},
{'Text':'title','Ancestors':'a|h2|li','ListItemId':4,'Bounds':{'X':10,'Y':44,'Width':100,'Height':24}}]
self.assertTrue(validate_citation_layout(page)['Passed'])
page['TextRuns'][0]['Bounds']['Height']=30
with self.assertRaises(AssertionError):validate_citation_layout(page)
page['MissingVisibleImages']=[{'Source':'not-decoded.png'}]
with self.assertRaises(AssertionError):validate_visible_images(page)
del page['MissingVisibleImages']
with self.assertRaises(AssertionError):validate_visible_images(page)
def test_full_scroll_coverage_requires_overlapping_actual_snapshots_through_the_bottom(self):
frames=[]
for sequence,y in enumerate([0,80,150],1):
page=remote();page.update(ScrollY=y,SnapshotSequence=sequence,DocumentHeight=250)
frames.append({'RemotePage':page})
self.assertTrue(validate_scroll_coverage(frames)['Passed'])
for index,field,value in [(1,'ScrollY',110),(2,'DocumentHeight',300),(2,'SnapshotSequence',1),(1,'BodySha256','b'*64)]:
changed=copy.deepcopy(frames);changed[index]['RemotePage'][field]=value
with self.subTest(field=field),self.assertRaises(AssertionError):validate_scroll_coverage(changed)
def test_chevron_requires_rotated_scaled_geometry_and_actual_thin_diagonal_pixels(self):
page=remote();frame=Image.new('RGB',(200,100),'#202020');draw=ImageDraw.Draw(frame)
page['LayoutSamples']=[{'Class':'nav-arrow','Bounds':{'X':x,'Y':60,'Width':14.142,'Height':14.142},'Transform':[.176777,.176777,-.176777,.176777,0,0]} for x in [20,60]]
draw.line([(28,62),(22,67),(28,72)],fill='#ffffff',width=1)
draw.line([(64,62),(70,67),(64,72)],fill='#999999',width=1)
self.assertTrue(chevron_pixels(frame,page,[0,0,200,100])['Passed'])
with self.assertRaises(AssertionError):chevron_pixels(Image.new('RGB',(200,100),'#808080'),page,[0,0,200,100])
page['LayoutSamples'][0]['Bounds']['Width']=40
with self.assertRaises(AssertionError):chevron_pixels(frame,page,[0,0,200,100])
def test_wide_home_requires_actual_navigation_and_six_distinct_hot_search_links(self):
page = remote(); page.update(ViewportWidth=1276, ViewportHeight=656)
page['Links'] = [{'Address': f'https://cn.bing.com/link/{i}', 'Bounds': {'X': i*40, 'Y': 20, 'Width': 30, 'Height': 20}} for i in range(4)]
with self.assertRaises(AssertionError): validate_wide_home(page)
page['Links'] += [{'Address': f'https://cn.bing.com/search?q=actual{i}', 'Bounds': {'X': i%3*200, 'Y': 280+i//3*30, 'Width': 180, 'Height': 20}} for i in range(6)]
self.assertTrue(validate_wide_home(page)['Passed'])
page['ViewportWidth'] = 796
with self.assertRaises(AssertionError): validate_wide_home(page)
def test_gray_home_queued_or_uniform_images_cannot_pass_visual_home_acceptance(self):
page = remote()
with self.assertRaises(AssertionError): validate_loaded_home(page)
page['PendingResources'] = 0; page['SearchInputs'][0]['Bounds']['Width'] = 160
photo = {'Source': 'https://images.example.test/actual.webp', 'Width': 800, 'Height': 600,
'Bounds': {'X': 0, 'Y': 0, 'Width': 200, 'Height': 100},
'Samples': [{'X': 10 + i * 10, 'Y': 75, 'Argb': f'ff{i * 8:02x}8070'} for i in range(16)]}
page['LoadedImages'] = [photo]
self.assertEqual([photo], validate_loaded_home(page))
frame = Image.new('RGB', (240, 160), '#808080'); viewport = [20, 30, 200, 100]
self.assertFalse(photograph_pixels(frame, [photo], viewport)['Passed'])
for sample in photo['Samples']: frame.putpixel((sample['X'] + 20, sample['Y'] + 30), tuple(bytes.fromhex(sample['Argb'][2:])))
self.assertTrue(photograph_pixels(frame, [photo], viewport)['Passed'])
for field, value in (('PendingResources', 1), ('Diagnostics', 'Event loop failed: btoa is not defined'), ('Diagnostics', 'JavaScript error: The operation has timed out.'), ('Diagnostics', 'Image failed: The underlying compression routine could not be loaded correctly.'), ('LoadedImages', [])):
changed = copy.deepcopy(page); changed[field] = value
with self.subTest(field=field), self.assertRaises(AssertionError): validate_loaded_home(changed)
def test_search_acceptance_requires_real_query_input_and_visible_result_link(self):
page = remote(); page.update(Address='https://cn.bing.com/search?q=Microsoft+.NET', PendingResources=0)
page['SearchInputs'][0]['Value'] = 'Microsoft .NET'
page['Links'] = [{'Address': 'https://dotnet.microsoft.com/download', 'Text': 'Download .NET',
'Bounds': {'X': 10, 'Y': 60, 'Width': 130, 'Height': 20}}]
self.assertEqual(page['Links'], validate_search_results(page, 'Microsoft .NET'))
for field, value in (('Address', HOME), ('Links', []), ('PendingResources', 1)):
changed = copy.deepcopy(page); changed[field] = value
with self.subTest(field=field), self.assertRaises(AssertionError): validate_search_results(changed, 'Microsoft .NET')
def test_search_images_require_visible_decoded_icon_instead_of_only_a_sprite(self):
page = remote()
with self.assertRaises(AssertionError): validate_search_images(page)
icon = {'Format': 'PNG', 'Width': 32, 'Height': 32, 'Bounds': {'X': 10, 'Y': 20, 'Width': 24, 'Height': 24},
'Samples': [{'Argb': 'ff6633cc'}, {'Argb': 'ffffffff'}]}
page['LoadedImages'] = [icon]; self.assertEqual([icon], validate_search_images(page))
for field, value in (('Width', 1030), ('Format', 'WebP'), ('Samples', [{'Argb': '006633cc'}])):
changed = copy.deepcopy(page); changed['LoadedImages'][0][field] = value
with self.subTest(field=field), self.assertRaises(AssertionError): validate_search_images(changed)
def test_native_validated_200_html_chain_is_required(self):
self.assertEqual(1, len(validate_response_chain([response()])))
for field, value in (('NativeTransport', False), ('CertificateValidated', False), ('StatusCode', 500),
('HasHtml', False), ('HasHead', False), ('HtmlTitle', 'Navigation failed'),
('BodySha256', 'missing'), ('BodyBytes', 0), ('ResponseHeaders', 'HTTP/1.1 404 Missing\r\n')):
item = response(); item['Value'][field] = value
with self.subTest(field=field), self.assertRaises(AssertionError):
validate_response_chain([item])
def test_redirects_require_matching_native_raw_headers_and_tls(self):
next_url = 'https://cn.bing.com/?ensearch=1'
chain = [response(status=302, location=next_url), response(next_url)]
self.assertEqual(2, len(validate_response_chain(chain)))
for change in ('downgrade', 'missing', 'loop', 'host', 'tls'):
invalid = copy.deepcopy(chain)
if change == 'downgrade':
invalid[0]['Value']['Location'] = 'http://cn.bing.com/'
elif change == 'missing':
invalid = invalid[:1]
elif change == 'loop':
invalid[0] = response(status=302, location=HOME); invalid.append(response(status=302, location=HOME))
elif change == 'host':
invalid[1]['Value']['RequestHeaders'] = 'GET /?ensearch=1 HTTP/1.1\r\nHost: attacker.test\r\n\r\n'
else:
invalid[1]['Value']['CertificateValidated'] = False
with self.subTest(change=change), self.assertRaises(AssertionError):
validate_response_chain(invalid)
def test_failure_page_wrong_html_and_missing_styles_are_not_remote_bing(self):
result = validate_remote_page(remote(), response()['Value'])
self.assertFalse(result['CssParityVerified'])
for change in ('failure', 'body', 'search', 'style', 'nonfinite', 'tls', 'snapshot'):
page = remote()
if change == 'failure': page['Title'] = 'Bing Navigation failed'
elif change == 'body': page['BodySha256'] = 'b' * 64
elif change == 'search': page['SearchInputs'][0]['Name'] = 'fake'
elif change == 'style': page['LoadedStyles'] = []
elif change == 'nonfinite': page['SearchInputs'][0]['Bounds']['X'] = float('nan')
elif change == 'tls': page['CertificateValidated'] = False
else: page['SnapshotSequence'] = 0
with self.subTest(change=change), self.assertRaises(AssertionError):
validate_remote_page(page, response()['Value'])
def test_non_bing_or_missing_packets_do_not_qualify(self):
for uri in ('https://bing.com.attacker.test/', 'https://example.com/', 'about:Start', 'http://www.bing.com/'):
with self.subTest(uri=uri), self.assertRaises(AssertionError):
validate_response_chain([response(uri)] if '://' in uri else [])
def test_stylesheet_redirects_bind_the_declared_request_and_actual_https_final(self):
page = remote()
style = page['LoadedStyles'][0]
requested = style['Address']; final = 'https://r.bing.com/styles/final.css?v=2'
style.update(RequestedAddress=requested, FinalAddress=final, Address=final)
evidence = validate_remote_page(page, response()['Value'])
self.assertEqual(final, evidence['LoadedStyles'][0]['FinalAddress'])
self.assertEqual(requested, evidence['LoadedStyles'][0]['RequestedAddress'])
self.assertFalse(evidence['CssParityVerified'])
for name in ('unrelated-request', 'wrong-final', 'http-final', 'http-request', 'empty-resource'):
changed = copy.deepcopy(page); receipt = changed['LoadedStyles'][0]
if name == 'unrelated-request': receipt['RequestedAddress'] = 'https://www.bing.com/unrelated.css'
elif name == 'wrong-final': receipt['FinalAddress'] = 'https://r.bing.com/different.css'
elif name == 'http-final': receipt['Address'] = receipt['FinalAddress'] = 'http://r.bing.com/styles.css'
elif name == 'http-request': receipt['RequestedAddress'] = 'http://www.bing.com/real.css'
else: receipt['Characters'] = 0
with self.subTest(case=name), self.assertRaises(AssertionError):
validate_remote_page(changed, response()['Value'])
def test_native_navigation_error_must_be_fresh_complete_current_window_and_bing(self):
failure = 'Navigation failed: TypeInitializerException: exact actual guest error'
def state(window=4, address=HOME, content=failure):
return f'XFE_BROWSER_STATE_BEGIN\n{window}|{address}\n{content}\nXFE_BROWSER_STATE_END\n'
actual = state()
self.assertEqual(failure, native_navigation_failure(actual, 0, 4)['Text'])
for serial, after in ((actual, len(actual)), (state(window=5), 0),
(state(address='https://example.com/'), 0), (state(address='about:Start'), 0),
(state(content='Actual Bing home'), 0), (actual[:-10], 0),
(actual + state(content='Actual Bing home'), 0)):
with self.subTest(serial=serial, after=after):
self.assertIsNone(native_navigation_failure(serial, after, 4))
prefixed = actual + 'NEW_INPUT\n' + actual
self.assertGreater(native_navigation_failure(prefixed, len(actual), 4)['SerialOffset'], len(actual))
def test_real_current_navigation_failure_exits_promptly_and_preserves_false_receipt(self):
serial = ('XFE_DESKTOP_ACTIVE|Browser\nXFE_WINDOW_STATE|4|Browser|216|132|800|500|0|0|1\nXFE_DESKTOP_STATE_END\n'
'XFE_BROWSER_STATE_BEGIN\n4|' + HOME + '\nNavigation failed: exact guest TLS init failure\nXFE_BROWSER_STATE_END\n')
with temporary_directory() as temporary:
started = time.monotonic()
with self.assertRaisesRegex(AssertionError, 'exact guest TLS init failure'):
run_bing(temporary, lambda: serial, lambda: serial, lambda *_: None,
lambda *_: self.fail('A failed navigation must not capture an acceptance screenshot'),
lambda *_: None, lambda *_: None, lambda *_: None,
timeout=180, open_browser=False, start_offset=0)
self.assertLess(time.monotonic() - started, 1)
result = json.loads((Path(temporary) / 'BingEvidence.json').read_text(encoding='utf-8'))
self.assertFalse(result['Passed']); self.assertFalse(result['NativeHttpsBingLoaded'])
self.assertEqual('Navigation failed: exact guest TLS init failure', result['NavigationFailure']['Text'])
self.assertEqual(1, result['RawGuestEvidence']['NavigationFailures'])
self.assertIn('exact guest TLS init failure', (Path(temporary) / 'BingRequestEvidence.log').read_text(encoding='utf-8'))
def test_parser_requires_complete_json_packet_and_retains_raw_evidence(self):
payload = json.dumps(response()['Value'])
packet = RESPONSE + '_BEGIN\n' + payload + '\n' + RESPONSE + '_END'
self.assertEqual(packet, json_blocks(packet, RESPONSE)[0]['Raw'])
self.assertEqual([], json_blocks(packet[:-4], RESPONSE))
self.assertEqual([], json_blocks(RESPONSE + '_BEGIN\n{} broken\n' + RESPONSE + '_END', RESPONSE))
self.assertEqual([], json_blocks(packet, RESPONSE, len(packet)))
def test_blank_uniform_wrong_color_and_wrong_geometry_do_not_count_as_glyphs(self):
page = remote(); viewport = [20, 30, 200, 100]
blank = Image.new('RGB', (240, 160), '#101010')
self.assertFalse(remote_text_pixels(blank, page, viewport)['Passed'])
self.assertFalse(remote_text_pixels(Image.new('RGB', blank.size, '#e0e0e0'), page, viewport)['Passed'])
# This generated image tests only the verifier's positive/negative
# controls; run_bing never uses it or calls a host image generator.
control = blank.copy(); draw = ImageDraw.Draw(control)
for run in page['TextRuns']:
x, y = run['Bounds']['X'] + viewport[0], run['Bounds']['Y'] + viewport[1]
for offset in range(0, 30, 5): draw.rectangle((x + offset, y + 2, x + offset + 1, y + 8), fill='#e0e0e0')
self.assertTrue(remote_text_pixels(control, page, viewport)['Passed'])
changed = copy.deepcopy(page)
for run in changed['TextRuns']: run['Color'] = '#ff0000'
self.assertFalse(remote_text_pixels(control, changed, viewport)['Passed'])
self.assertFalse(remote_text_pixels(control, page, [viewport[0] + 100, viewport[1], *viewport[2:]])['Passed'])
def test_real_run_without_https_always_keeps_false_timeout_and_raw_packets(self):
with temporary_directory() as temporary:
def no_op(*_): pass
with self.assertRaisesRegex(AssertionError, 'timed out'):
run_bing(temporary, lambda: '', lambda: '', no_op, no_op, no_op, no_op, no_op, timeout=0)
result = json.loads((Path(temporary) / 'BingEvidence.json').read_text(encoding='utf-8'))
self.assertFalse(result['Passed']); self.assertFalse(result['NativeHttpsBingLoaded'])
self.assertFalse(result['HostBrowserScreenshots']); self.assertFalse(result['AllUserRequirementsMet'])
self.assertEqual(0, result['RawGuestEvidence']['Packets'])
self.assertTrue((Path(temporary) / 'BingRequestEvidence.log').exists())
def test_native_qemu_profile_has_exact_e1000_nat_cpu_and_utc_arguments(self):
base = ['qemu-system-x86_64', '-cpu', 'max', '-rtc', 'base=utc']
arguments = base + network_arguments(True)
profile = validate_network_arguments(arguments, True)
self.assertEqual(native_network_profile(True), profile)
self.assertEqual('8086:100e', profile['PciVendorDevice'])
self.assertEqual('Static', profile['AddressConfiguration'])
self.assertFalse(profile['HostProxy']); self.assertFalse(profile['HostForwarding'])
self.assertEqual(['-netdev', NAT_ARGUMENT, '-device', NIC_ARGUMENT], network_arguments(True))
self.assertFalse(validate_network_arguments(base + network_arguments(False), False)['Enabled'])
def test_wrong_nic_proxy_forwarding_offline_duplicate_cpu_and_rtc_are_rejected(self):
valid = ['-cpu', 'max', '-rtc', 'base=utc', *network_arguments(True)]
invalid = [valid + ['-net', 'none'], valid + ['-nic', 'user'],
valid + ['-device', 'usb-net,netdev=NativeNet'],
valid + ['-netdev', NAT_ARGUMENT], valid + ['-cpu', 'max'],
[item.replace('e1000,', 'e1000e,') for item in valid],
[item.replace('dns=10.0.2.3', 'dns=127.0.0.1') for item in valid],
[item.replace('base=utc', 'base=localtime') for item in valid],
[item + ',hostfwd=tcp::8080-:80' if item == NAT_ARGUMENT else item for item in valid],
['-cpu', 'max', '-rtc', 'base=utc', '-net', 'none']]
for arguments in invalid:
with self.subTest(arguments=arguments), self.assertRaises(AssertionError):
validate_network_arguments(arguments, True)
def test_presentation_requires_latest_exact_snapshot_url_scroll_and_real_scanout(self):
def marker(snapshot=12, address=HOME, sequence=6, ticks=100, scroll=0, window=3):
return f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|{window}|Generation|2|Address|{address}|SnapshotSequence|{snapshot}|PresentationSequence|{sequence}|PresentationTicks|{ticks}|ScrollY|{scroll}\n'
valid = marker()
value = presentation_identity(matching_presentation(valid, remote(), 3))
self.assertEqual(12, value['SnapshotSequence']); self.assertEqual(100, value['PresentationTicks'])
for serial in ('', marker(snapshot=13), marker(address='https://example.com/'), marker(sequence=0),
marker(ticks=0), marker(scroll=5), marker(window=4), valid + marker(snapshot=13)):
with self.subTest(serial=serial), self.assertRaises(AssertionError):
matching_presentation(serial, remote(), 3)
with self.assertRaises(AssertionError): matching_presentation(valid, remote(), 3, len(valid))
def test_screenshot_requires_unchanged_snapshot_and_monotonic_presentation_counters(self):
initial = {'WindowId': 3, 'Generation': 2, 'Address': HOME, 'SnapshotSequence': 12,
'PresentationSequence': 6, 'PresentationTicks': 100, 'ScrollY': 0}
later = {**initial, 'PresentationSequence': 7, 'PresentationTicks': 110}
self.assertTrue(stable_screenshot_presentation(initial, later)['StableSnapshot'])
for field, value in (('WindowId', 4), ('Generation', 3), ('Address', 'about:Start'),
('SnapshotSequence', 13), ('PresentationSequence', 5), ('PresentationTicks', 99), ('ScrollY', 5)):
with self.subTest(field=field), self.assertRaises(AssertionError):
stable_screenshot_presentation(initial, {**later, field: value})
def test_capture_rejects_snapshot_advancing_during_the_actual_screenshot(self):
initial=f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|3|Generation|2|Address|{HOME}|SnapshotSequence|12|PresentationSequence|6|PresentationTicks|100|ScrollY|0\n'
changed=initial+initial.replace('SnapshotSequence|12','SnapshotSequence|13').replace('Sequence|6|','Sequence|7|').replace('Ticks|100','Ticks|110')
captures=[]
with self.assertRaisesRegex(AssertionError,'differs'):
capture_matching_presentation(initial,remote(),3,0,captures.append,lambda:changed,'attempt')
self.assertEqual(['attempt'],captures)
def test_capture_rejects_stale_dom_before_taking_any_screenshot(self):
latest=f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|3|Generation|2|Address|{HOME}|SnapshotSequence|13|PresentationSequence|7|PresentationTicks|110|ScrollY|0\n'
captures=[]
with self.assertRaisesRegex(AssertionError,'differs'):
capture_matching_presentation(latest,remote(),3,0,captures.append,lambda:latest,'attempt')
self.assertEqual([],captures)
def test_capture_accepts_only_a_stable_snapshot_on_both_sides(self):
initial=f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|3|Generation|2|Address|{HOME}|SnapshotSequence|12|PresentationSequence|6|PresentationTicks|100|ScrollY|0\n'
later=initial+initial.replace('Sequence|6|','Sequence|7|').replace('Ticks|100','Ticks|110')
captures=[]
proof=capture_matching_presentation(initial,remote(),3,0,captures.append,lambda:later,'attempt')
self.assertEqual(['attempt'],captures);self.assertTrue(proof['StableSnapshot'])
self.assertEqual(12,proof['After']['SnapshotSequence']);self.assertEqual(7,proof['After']['PresentationSequence'])
def test_already_open_browser_requires_actual_navigation_start_offset(self):
with temporary_directory() as temporary, self.assertRaisesRegex(AssertionError, 'start offset'):
run_bing(temporary, lambda: '', lambda: '', lambda *_: None,
lambda *_: None, lambda *_: None, lambda *_: None, lambda *_: None,
open_browser=False)
def test_powershell_launcher_checks_actual_receipt_hashes_and_snapshot_bindings(self):
"""Execute only Open.ps1's real read-only Bing verifier against small guard fixtures.
These fixtures are neither guest evidence nor boot proofs. No launcher,
image copy, host network request, display or VM is invoked by this test.
"""
source = (ROOT / 'Tools/IndependentKernel/Open.ps1').read_text(encoding='utf-8-sig')
helpers = source[source.index('function Get-IndependentHash'):source.index('$sourceImage=')]
guard = source[source.index('$bingBinding='):source.index('foreach($capture in $documentWorkflow.Screenshots.')]
with temporary_directory() as temporary:
root = Path(temporary)
cases = []
for name in ('valid', 'tampered-png', 'tampered-log', 'wrong-body', 'wrong-snapshot',
'unstable', 'missing-generation', 'escaped-png', 'no-glyphs', 'unchecked-tls', 'parity-overclaim',
'old-acceptance', 'no-photograph', 'no-search', 'wrong-query', 'no-png', 'tampered-search',
'no-scroll', 'late-timeout', 'missing-icons', 'unpainted-icons', 'no-home-content', 'unpainted-home-content', 'unfinished-scripts', 'no-repeated-search', 'wrong-repeated-query', 'tampered-repeated-search','no-full-content','citation-overlap','visible-image-missing','content-gap','no-chevrons','clipped-navigation','repeated-placeholders'):
directory = root / name; directory.mkdir()
image = directory / 'Browser-Bing-Native-1.png'
Image.new('RGB', (2, 2), '#101010').save(image)
raw = directory / 'BingRequestEvidence.log'; raw.write_text('Verifier unit-test fixture only', encoding='utf-8')
sha = lambda path: hashlib.sha256(path.read_bytes()).hexdigest()
presentation = {'WindowId': 3, 'Generation': 2, 'Address': HOME, 'SnapshotSequence': 12,
'PresentationSequence': 6, 'PresentationTicks': 100, 'ScrollY': 0}
capture = {'Path': image.name, 'Sha256': sha(image), 'BodySha256': 'a' * 64,
'PresentationCorrelation': {'Before': dict(presentation), 'After': dict(presentation), 'StableSnapshot': True},
'TextPixels': {'Passed': True}, 'PhotographPixels': {'Passed': True}, 'HomeContentPixels': {'Passed': True}}
home_page = remote(); home_page.update(PendingResources=0, DecodedImageFormats=['WebP', 'PNG'], ViewportWidth=1276)
for link in home_page['NavigationLinks']:link['ContainerBounds']['Width']=1276
search_page = copy.deepcopy(home_page)
search_page.update(Address='https://cn.bing.com/search?q=Microsoft+.NET', SnapshotSequence=13)
search_page['SearchInputs'][0]['Value'] = 'Microsoft .NET'
search_page['LoadedImages'] = [{'Format': 'PNG', 'Width': 32, 'Height': 32}]
scrolled_page = copy.deepcopy(search_page); scrolled_page.update(SnapshotSequence=14, ScrollY=80)
def search_capture(filename, page):
path = directory / filename; Image.new('RGB', (2, 2), '#202020').save(path)
identity = {**presentation, 'Address': page['Address'], 'SnapshotSequence': page['SnapshotSequence'], 'ScrollY': page['ScrollY']}
return {'Path': filename, 'Sha256': sha(path), 'BodySha256': page['BodySha256'], 'ScrollY': page['ScrollY'],
'PresentationCorrelation': {'Before': dict(identity), 'After': dict(identity), 'StableSnapshot': True}, 'TextPixels': {'Passed': True}, 'IconPixels': {'Passed': True}}
search = {'Query': 'Microsoft .NET', 'Responses': [response(search_page['Address'])['Value']], 'RemotePage': search_page,
'ScrolledRemotePage': scrolled_page,
'VisibleResultLinks': [{'Address': 'https://dotnet.microsoft.com/download', 'Text': 'Download .NET', 'Bounds': {'Width': 130, 'Height': 20}}],
'Screenshot': search_capture('Search.png', search_page), 'ScrolledScreenshot': search_capture('Scroll.png', scrolled_page)}
repeated_page=copy.deepcopy(search_page);repeated_page.update(Address='https://cn.bing.com/search?q=Microsoft+CSharp',SnapshotSequence=15,BodySha256='b'*64)
repeated_page['SearchInputs'][0]['Value']='Microsoft CSharp'
repeated_response=response(repeated_page['Address'])['Value'];repeated_response['BodySha256']='b'*64
repeated={'Query':'Microsoft CSharp','Responses':[repeated_response],'RemotePage':repeated_page,'VisibleResultLinks':[{'Address':'https://learn.microsoft.com/dotnet/csharp/','Text':'C# programming','Bounds':{'Width':130,'Height':20}}],
'Screenshot':search_capture('Repeated.png',repeated_page)}
for item in (search_page,repeated_page):
item['TextRuns']=[{'Text':'domain','Bounds':{'X':10,'Y':55,'Width':40,'Height':10},'Ancestors':'cite|div|li','ListItemId':10},
{'Text':'Title','Bounds':{'X':10,'Y':70,'Width':50,'Height':15},'Ancestors':'a|h2|li','ListItemId':10}]
search_page['DocumentHeight']=180;scrolled_page['DocumentHeight']=180
search['FullContent']={'Passed':True,'ReachedBottom':True,'Frames':[{'RemotePage':search_page,'Screenshot':search['Screenshot']},{'RemotePage':scrolled_page,'Screenshot':search['ScrolledScreenshot']}]}
repeated['FullContent']={'Passed':True,'ReachedBottom':True,'Frames':[{'RemotePage':repeated_page,'Screenshot':repeated['Screenshot']}]}
record = {'Kind': 'XfeIndependentNativeBingEvidence', 'Passed': True, 'NativeHttpsBingLoaded': True,
'AcceptanceVersion': 9,'NavigationVisible':True, 'VisibleContentLoaded':True,'CssFixesVerified':True,'HomeChevronPixels':{'Passed':True}, 'VisualHomeLoaded': True, 'RealSearchCompleted': True, 'RepeatedSearchCompleted':True,'RepeatedSearch':repeated,'SearchScrolled': True, 'DecodedPngVerified': True, 'SearchImagesLoaded': True,
'HomeContent': {'Passed': True, 'NavigationLinks': [{}]*4, 'HotSearchLinks': [{}]*6},
'AllUserRequirementsMet': False, 'FullCssParityVerified': False, 'CompleteJavaScriptCompatibility': False,
'HostHttpRequests': False, 'HostBrowserScreenshots': False, 'HostJavaScriptInjection': False,
'VmStartedByThisTool': False, 'RequestedUrl': HOME, 'Responses': [response()['Value']],
'RemotePage': home_page, 'Presentation': presentation, 'Screenshots': [capture], 'Search': search,
'RawGuestEvidence': {'Path': raw.name, 'Sha256': sha(raw), 'Packets': 2, 'PresentationMarkers': 1}}
if name == 'tampered-png': image.write_bytes(b'changed PNG')
elif name == 'tampered-log': raw.write_text('changed log', encoding='utf-8')
elif name == 'wrong-body': capture['BodySha256'] = 'b' * 64
elif name == 'wrong-snapshot': capture['PresentationCorrelation']['Before']['SnapshotSequence'] = 13
elif name == 'unstable': capture['PresentationCorrelation']['StableSnapshot'] = False
elif name == 'missing-generation':
del capture['PresentationCorrelation']['Before']['Generation']; del capture['PresentationCorrelation']['After']['Generation']
elif name == 'escaped-png': capture['Path'] = '../outside.png'
elif name == 'no-glyphs': capture['TextPixels']['Passed'] = False
elif name == 'unchecked-tls': record['Responses'][0]['CertificateValidated'] = False
elif name == 'parity-overclaim': record['AllUserRequirementsMet'] = True
elif name == 'old-acceptance': record['AcceptanceVersion'] = 2
elif name == 'no-photograph': capture['PhotographPixels']['Passed'] = False
elif name == 'no-search': del record['Search']
elif name == 'wrong-query': search_page['SearchInputs'][0]['Value'] = 'different query'
elif name == 'no-png':
home_page['DecodedImageFormats'] = ['WebP']; search_page['DecodedImageFormats'] = ['WebP']
elif name == 'tampered-search': (directory / 'Search.png').write_bytes(b'changed search PNG')
elif name == 'no-scroll': scrolled_page['ScrollY'] = 0
elif name == 'late-timeout': scrolled_page['Diagnostics'] = 'JavaScript error: The operation has timed out.'
elif name == 'missing-icons': search_page['LoadedImages'] = []
elif name == 'unpainted-icons': search['Screenshot']['IconPixels']['Passed'] = False
elif name == 'no-home-content': record['HomeContent']['HotSearchLinks'] = []
elif name == 'unpainted-home-content': capture['HomeContentPixels']['Passed'] = False
elif name == 'unfinished-scripts': search_page['DocumentInitializationComplete'] = False
elif name == 'no-repeated-search': del record['RepeatedSearch']
elif name == 'wrong-repeated-query': repeated_page['SearchInputs'][0]['Value']='old query'
elif name == 'tampered-repeated-search': (directory/'Repeated.png').write_bytes(b'changed repeated search PNG')
if name=='no-full-content':del search['FullContent']
elif name=='citation-overlap':search_page['TextRuns'][0]['Bounds']['Height']=30
elif name=='visible-image-missing':repeated_page['MissingVisibleImages']=[{'Source':'missing.png'}]
elif name=='content-gap':scrolled_page['ScrollY']=150
elif name=='no-chevrons':record['HomeChevronPixels']['Passed']=False
elif name=='clipped-navigation':search_page['NavigationLinks'][2]['Painted']=False
elif name=='repeated-placeholders':repeated_page['LoadedImages']=[]
receipt = directory / 'BingEvidence.json'; receipt.write_text(json.dumps(record), encoding='utf-8')
binding = {'Path': receipt.name, 'Sha256': sha(receipt), 'Passed': True, 'NativeHttpsBingLoaded': True}
cases.append({'Name': name, 'Root': str(directory), 'Binding': binding, 'Expected': name == 'valid'})
fixtures = root / 'Cases.json'; fixtures.write_text(json.dumps(cases), encoding='utf-8')
script = root / 'Check.ps1'
script.write_text("param([string]$Cases)\n$ErrorActionPreference='Stop'\n"
"Import-Module (Join-Path $PSHOME 'Modules/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1')\n" + helpers +
"\n$check={\n" + guard + "\n}\n$results=@(foreach($case in (Get-Content -Raw -Encoding UTF8 $Cases | ConvertFrom-Json)){\n"
"$desktopInput=[pscustomobject]@{BingEvidence=$case.Binding};$guestEvidenceRoot=$case.Root;$passed=$false;$reason=$null\n"
"try{& $check;$passed=$true}catch{$reason=$_.Exception.Message}\n"
"[pscustomobject]@{Name=$case.Name;Passed=$passed;Expected=$case.Expected;Reason=$reason}\n"
"});ConvertTo-Json -Depth 5 -InputObject $results\n", encoding='utf-8-sig')
completed = subprocess.run(['powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', str(script), '-Cases', str(fixtures)],
cwd=ROOT, check=True, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=30)
results = json.loads(completed.stdout)
self.assertEqual(len(cases), len(results))
for result in results:
with self.subTest(case=result['Name']):
self.assertEqual(result['Expected'], result['Passed'], result.get('Reason'))
if __name__ == '__main__':
unittest.main()
"""Offline negative controls for the Bing evidence reader; not guest acceptance."""
import copy
import hashlib
import json
from pathlib import Path
import subprocess
import tempfile
import time
import unittest
from PIL import Image, ImageDraw
from IndependentBingEvidence import (HOME, NIC_ARGUMENT, NAT_ARGUMENT, REMOTE, RESPONSE, json_blocks,
matching_presentation, native_network_profile, network_arguments, presentation_identity,
native_navigation_failure, remote_text_pixels, run_bing, stable_screenshot_presentation, validate_network_arguments,
validate_remote_page, validate_response_chain, validate_loaded_home, photograph_pixels, validate_search_results, validate_search_images, validate_wide_home)
from IndependentBingEvidence import validate_visible_images, validate_citation_layout, chevron_pixels, validate_scroll_coverage, validate_search_navigation
from IndependentBingEvidence import capture_matching_presentation
ROOT = Path(__file__).resolve().parents[2]
TEMP_ROOT = ROOT / 'build' / 'IndependentKernel' / 'ToolGuardTests'
def temporary_directory():
TEMP_ROOT.mkdir(parents=True, exist_ok=True)
return tempfile.TemporaryDirectory(dir=TEMP_ROOT)
def response(uri=HOME, status=200, location=None):
from urllib.parse import urlsplit
url = urlsplit(uri)
target = (url.path or '/') + ('?' + url.query if url.query else '')
headers = 'HTTP/1.1 ' + str(status) + ' OK\r\nContent-Type: text/html\r\n'
if location:
headers += 'Location: ' + location + '\r\n'
return {'Value': {'Kind': 'NativeResponse', 'RequestId': 1, 'RequestUri': uri, 'ResponseUri': uri,
'StatusCode': status, 'ContentType': 'text/html; charset=utf-8', 'CertificateValidated': True,
'NativeTransport': True, 'RequestHeaders': 'GET ' + target + ' HTTP/1.1\r\nHost: ' + url.hostname + '\r\n\r\n',
'ResponseHeaders': headers + '\r\n', 'BodyBytes': 1000, 'BodySha256': 'a' * 64,
'HtmlTitle': 'Bing', 'HasHtml': True, 'HasHead': True, 'Location': location}}
def remote():
return {'Address': HOME, 'Title': '必应', 'BodySha256': 'a' * 64, 'CertificateValidated': True,
'DocumentInitializationComplete': True, 'ScriptBudgetExceeded': False, 'PendingResources':0, 'NavigationLinks':[{'MenuId':7,'Painted':True,'ContainerBounds':{'X':0,'Y':5,'Width':200,'Height':20},'Bounds':{'X':i*25,'Y':5,'Width':20,'Height':15},'Text':'tab'} for i in range(5)], 'MissingVisibleImages': [], 'DocumentHeight': 100,
'SnapshotSequence': 12, 'ScrollY': 0, 'ViewportWidth': 200, 'ViewportHeight': 100,
'SearchInputs': [{'Name': 'q', 'Bounds': {'X': 30, 'Y': 20, 'Width': 120, 'Height': 20}}],
'TextRuns': [{'Text': text, 'Bounds': {'X': 10, 'Y': y, 'Width': 60, 'Height': 12}, 'Color': '#e0e0e0'} for text, y in (('Bing', 4), ('Search', 60))],
'LoadedStyles': [{'Address': 'https://www.bing.com/real.css', 'Characters': 1000}],
'DeclaredStyleLinks': ['https://www.bing.com/real.css'], 'InlineStyleCount': 2}
class IndependentBingEvidenceTests(unittest.TestCase):
def test_all_photograph_pixel_call_sites_bind_to_the_actual_helper(self):
import ast,inspect
source=(ROOT/'Tools/IndependentKernel/IndependentBingEvidence.py').read_text(encoding='utf-8')
signature=inspect.signature(photograph_pixels);calls=[]
for node in ast.walk(ast.parse(source)):
if isinstance(node,ast.Call) and isinstance(node.func,ast.Name) and node.func.id=='photograph_pixels':
signature.bind(*([None]*len(node.args)),**{item.arg:None for item in node.keywords});calls.append(node)
self.assertEqual(3,len(calls))
with self.assertRaises(TypeError):signature.bind(None,None,None,False)
def test_declared_navigation_requires_real_paint_after_clipping(self):
page=remote();self.assertEqual(5,len(validate_search_navigation(page)['Links']))
page['NavigationLinks'][2]['Painted']=False
with self.assertRaises(AssertionError):validate_search_navigation(page)
del page['NavigationLinks']
with self.assertRaises(AssertionError):validate_search_navigation(page)
def test_visible_images_and_citation_text_cannot_be_replaced_by_success_flags(self):
page=remote();validate_visible_images(page)
page['TextRuns']=[{'Text':'url','Ancestors':'strong|cite|div','ListItemId':4,'Bounds':{'X':10,'Y':20,'Width':80,'Height':20}},
{'Text':'title','Ancestors':'a|h2|li','ListItemId':4,'Bounds':{'X':10,'Y':44,'Width':100,'Height':24}}]
self.assertTrue(validate_citation_layout(page)['Passed'])
page['TextRuns'][0]['Bounds']['Height']=30
with self.assertRaises(AssertionError):validate_citation_layout(page)
page['MissingVisibleImages']=[{'Source':'not-decoded.png'}]
with self.assertRaises(AssertionError):validate_visible_images(page)
del page['MissingVisibleImages']
with self.assertRaises(AssertionError):validate_visible_images(page)
def test_full_scroll_coverage_requires_overlapping_actual_snapshots_through_the_bottom(self):
frames=[]
for sequence,y in enumerate([0,80,150],1):
page=remote();page.update(ScrollY=y,SnapshotSequence=sequence,DocumentHeight=250)
frames.append({'RemotePage':page})
self.assertTrue(validate_scroll_coverage(frames)['Passed'])
for index,field,value in [(1,'ScrollY',110),(2,'DocumentHeight',300),(2,'SnapshotSequence',1),(1,'BodySha256','b'*64)]:
changed=copy.deepcopy(frames);changed[index]['RemotePage'][field]=value
with self.subTest(field=field),self.assertRaises(AssertionError):validate_scroll_coverage(changed)
def test_chevron_requires_rotated_scaled_geometry_and_actual_thin_diagonal_pixels(self):
page=remote();frame=Image.new('RGB',(200,100),'#202020');draw=ImageDraw.Draw(frame)
page['LayoutSamples']=[{'Class':'nav-arrow','Bounds':{'X':x,'Y':60,'Width':14.142,'Height':14.142},'Transform':[.176777,.176777,-.176777,.176777,0,0]} for x in [20,60]]
draw.line([(28,62),(22,67),(28,72)],fill='#ffffff',width=1)
draw.line([(64,62),(70,67),(64,72)],fill='#999999',width=1)
self.assertTrue(chevron_pixels(frame,page,[0,0,200,100])['Passed'])
with self.assertRaises(AssertionError):chevron_pixels(Image.new('RGB',(200,100),'#808080'),page,[0,0,200,100])
page['LayoutSamples'][0]['Bounds']['Width']=40
with self.assertRaises(AssertionError):chevron_pixels(frame,page,[0,0,200,100])
def test_wide_home_requires_actual_navigation_and_six_distinct_hot_search_links(self):
page = remote(); page.update(ViewportWidth=1276, ViewportHeight=656)
page['Links'] = [{'Address': f'https://cn.bing.com/link/{i}', 'Bounds': {'X': i*40, 'Y': 20, 'Width': 30, 'Height': 20}} for i in range(4)]
with self.assertRaises(AssertionError): validate_wide_home(page)
page['Links'] += [{'Address': f'https://cn.bing.com/search?q=actual{i}', 'Bounds': {'X': i%3*200, 'Y': 280+i//3*30, 'Width': 180, 'Height': 20}} for i in range(6)]
self.assertTrue(validate_wide_home(page)['Passed'])
page['ViewportWidth'] = 796
with self.assertRaises(AssertionError): validate_wide_home(page)
def test_gray_home_queued_or_uniform_images_cannot_pass_visual_home_acceptance(self):
page = remote()
with self.assertRaises(AssertionError): validate_loaded_home(page)
page['PendingResources'] = 0; page['SearchInputs'][0]['Bounds']['Width'] = 160
photo = {'Source': 'https://images.example.test/actual.webp', 'Width': 800, 'Height': 600,
'Bounds': {'X': 0, 'Y': 0, 'Width': 200, 'Height': 100},
'Samples': [{'X': 10 + i * 10, 'Y': 75, 'Argb': f'ff{i * 8:02x}8070'} for i in range(16)]}
page['LoadedImages'] = [photo]
self.assertEqual([photo], validate_loaded_home(page))
frame = Image.new('RGB', (240, 160), '#808080'); viewport = [20, 30, 200, 100]
self.assertFalse(photograph_pixels(frame, [photo], viewport)['Passed'])
for sample in photo['Samples']: frame.putpixel((sample['X'] + 20, sample['Y'] + 30), tuple(bytes.fromhex(sample['Argb'][2:])))
self.assertTrue(photograph_pixels(frame, [photo], viewport)['Passed'])
for field, value in (('PendingResources', 1), ('Diagnostics', 'Event loop failed: btoa is not defined'), ('Diagnostics', 'JavaScript error: The operation has timed out.'), ('Diagnostics', 'Image failed: The underlying compression routine could not be loaded correctly.'), ('LoadedImages', [])):
changed = copy.deepcopy(page); changed[field] = value
with self.subTest(field=field), self.assertRaises(AssertionError): validate_loaded_home(changed)
def test_search_acceptance_requires_real_query_input_and_visible_result_link(self):
page = remote(); page.update(Address='https://cn.bing.com/search?q=Microsoft+.NET', PendingResources=0)
page['SearchInputs'][0]['Value'] = 'Microsoft .NET'
page['Links'] = [{'Address': 'https://dotnet.microsoft.com/download', 'Text': 'Download .NET',
'Bounds': {'X': 10, 'Y': 60, 'Width': 130, 'Height': 20}}]
self.assertEqual(page['Links'], validate_search_results(page, 'Microsoft .NET'))
for field, value in (('Address', HOME), ('Links', []), ('PendingResources', 1)):
changed = copy.deepcopy(page); changed[field] = value
with self.subTest(field=field), self.assertRaises(AssertionError): validate_search_results(changed, 'Microsoft .NET')
def test_search_images_require_visible_decoded_icon_instead_of_only_a_sprite(self):
page = remote()
with self.assertRaises(AssertionError): validate_search_images(page)
icon = {'Format': 'PNG', 'Width': 32, 'Height': 32, 'Bounds': {'X': 10, 'Y': 20, 'Width': 24, 'Height': 24},
'Samples': [{'Argb': 'ff6633cc'}, {'Argb': 'ffffffff'}]}
page['LoadedImages'] = [icon]; self.assertEqual([icon], validate_search_images(page))
for field, value in (('Width', 1030), ('Format', 'WebP'), ('Samples', [{'Argb': '006633cc'}])):
changed = copy.deepcopy(page); changed['LoadedImages'][0][field] = value
with self.subTest(field=field), self.assertRaises(AssertionError): validate_search_images(changed)
def test_native_validated_200_html_chain_is_required(self):
self.assertEqual(1, len(validate_response_chain([response()])))
for field, value in (('NativeTransport', False), ('CertificateValidated', False), ('StatusCode', 500),
('HasHtml', False), ('HasHead', False), ('HtmlTitle', 'Navigation failed'),
('BodySha256', 'missing'), ('BodyBytes', 0), ('ResponseHeaders', 'HTTP/1.1 404 Missing\r\n')):
item = response(); item['Value'][field] = value
with self.subTest(field=field), self.assertRaises(AssertionError):
validate_response_chain([item])
def test_redirects_require_matching_native_raw_headers_and_tls(self):
next_url = 'https://cn.bing.com/?ensearch=1'
chain = [response(status=302, location=next_url), response(next_url)]
self.assertEqual(2, len(validate_response_chain(chain)))
for change in ('downgrade', 'missing', 'loop', 'host', 'tls'):
invalid = copy.deepcopy(chain)
if change == 'downgrade':
invalid[0]['Value']['Location'] = 'http://cn.bing.com/'
elif change == 'missing':
invalid = invalid[:1]
elif change == 'loop':
invalid[0] = response(status=302, location=HOME); invalid.append(response(status=302, location=HOME))
elif change == 'host':
invalid[1]['Value']['RequestHeaders'] = 'GET /?ensearch=1 HTTP/1.1\r\nHost: attacker.test\r\n\r\n'
else:
invalid[1]['Value']['CertificateValidated'] = False
with self.subTest(change=change), self.assertRaises(AssertionError):
validate_response_chain(invalid)
def test_failure_page_wrong_html_and_missing_styles_are_not_remote_bing(self):
result = validate_remote_page(remote(), response()['Value'])
self.assertFalse(result['CssParityVerified'])
for change in ('failure', 'body', 'search', 'style', 'nonfinite', 'tls', 'snapshot'):
page = remote()
if change == 'failure': page['Title'] = 'Bing Navigation failed'
elif change == 'body': page['BodySha256'] = 'b' * 64
elif change == 'search': page['SearchInputs'][0]['Name'] = 'fake'
elif change == 'style': page['LoadedStyles'] = []
elif change == 'nonfinite': page['SearchInputs'][0]['Bounds']['X'] = float('nan')
elif change == 'tls': page['CertificateValidated'] = False
else: page['SnapshotSequence'] = 0
with self.subTest(change=change), self.assertRaises(AssertionError):
validate_remote_page(page, response()['Value'])
def test_non_bing_or_missing_packets_do_not_qualify(self):
for uri in ('https://bing.com.attacker.test/', 'https://example.com/', 'about:Start', 'http://www.bing.com/'):
with self.subTest(uri=uri), self.assertRaises(AssertionError):
validate_response_chain([response(uri)] if '://' in uri else [])
def test_stylesheet_redirects_bind_the_declared_request_and_actual_https_final(self):
page = remote()
style = page['LoadedStyles'][0]
requested = style['Address']; final = 'https://r.bing.com/styles/final.css?v=2'
style.update(RequestedAddress=requested, FinalAddress=final, Address=final)
evidence = validate_remote_page(page, response()['Value'])
self.assertEqual(final, evidence['LoadedStyles'][0]['FinalAddress'])
self.assertEqual(requested, evidence['LoadedStyles'][0]['RequestedAddress'])
self.assertFalse(evidence['CssParityVerified'])
for name in ('unrelated-request', 'wrong-final', 'http-final', 'http-request', 'empty-resource'):
changed = copy.deepcopy(page); receipt = changed['LoadedStyles'][0]
if name == 'unrelated-request': receipt['RequestedAddress'] = 'https://www.bing.com/unrelated.css'
elif name == 'wrong-final': receipt['FinalAddress'] = 'https://r.bing.com/different.css'
elif name == 'http-final': receipt['Address'] = receipt['FinalAddress'] = 'http://r.bing.com/styles.css'
elif name == 'http-request': receipt['RequestedAddress'] = 'http://www.bing.com/real.css'
else: receipt['Characters'] = 0
with self.subTest(case=name), self.assertRaises(AssertionError):
validate_remote_page(changed, response()['Value'])
def test_native_navigation_error_must_be_fresh_complete_current_window_and_bing(self):
failure = 'Navigation failed: TypeInitializerException: exact actual guest error'
def state(window=4, address=HOME, content=failure):
return f'XFE_BROWSER_STATE_BEGIN\n{window}|{address}\n{content}\nXFE_BROWSER_STATE_END\n'
actual = state()
self.assertEqual(failure, native_navigation_failure(actual, 0, 4)['Text'])
for serial, after in ((actual, len(actual)), (state(window=5), 0),
(state(address='https://example.com/'), 0), (state(address='about:Start'), 0),
(state(content='Actual Bing home'), 0), (actual[:-10], 0),
(actual + state(content='Actual Bing home'), 0)):
with self.subTest(serial=serial, after=after):
self.assertIsNone(native_navigation_failure(serial, after, 4))
prefixed = actual + 'NEW_INPUT\n' + actual
self.assertGreater(native_navigation_failure(prefixed, len(actual), 4)['SerialOffset'], len(actual))
def test_real_current_navigation_failure_exits_promptly_and_preserves_false_receipt(self):
serial = ('XFE_DESKTOP_ACTIVE|Browser\nXFE_WINDOW_STATE|4|Browser|216|132|800|500|0|0|1\nXFE_DESKTOP_STATE_END\n'
'XFE_BROWSER_STATE_BEGIN\n4|' + HOME + '\nNavigation failed: exact guest TLS init failure\nXFE_BROWSER_STATE_END\n')
with temporary_directory() as temporary:
started = time.monotonic()
with self.assertRaisesRegex(AssertionError, 'exact guest TLS init failure'):
run_bing(temporary, lambda: serial, lambda: serial, lambda *_: None,
lambda *_: self.fail('A failed navigation must not capture an acceptance screenshot'),
lambda *_: None, lambda *_: None, lambda *_: None,
timeout=180, open_browser=False, start_offset=0)
self.assertLess(time.monotonic() - started, 1)
result = json.loads((Path(temporary) / 'BingEvidence.json').read_text(encoding='utf-8'))
self.assertFalse(result['Passed']); self.assertFalse(result['NativeHttpsBingLoaded'])
self.assertEqual('Navigation failed: exact guest TLS init failure', result['NavigationFailure']['Text'])
self.assertEqual(1, result['RawGuestEvidence']['NavigationFailures'])
self.assertIn('exact guest TLS init failure', (Path(temporary) / 'BingRequestEvidence.log').read_text(encoding='utf-8'))
def test_parser_requires_complete_json_packet_and_retains_raw_evidence(self):
payload = json.dumps(response()['Value'])
packet = RESPONSE + '_BEGIN\n' + payload + '\n' + RESPONSE + '_END'
self.assertEqual(packet, json_blocks(packet, RESPONSE)[0]['Raw'])
self.assertEqual([], json_blocks(packet[:-4], RESPONSE))
self.assertEqual([], json_blocks(RESPONSE + '_BEGIN\n{} broken\n' + RESPONSE + '_END', RESPONSE))
self.assertEqual([], json_blocks(packet, RESPONSE, len(packet)))
def test_blank_uniform_wrong_color_and_wrong_geometry_do_not_count_as_glyphs(self):
page = remote(); viewport = [20, 30, 200, 100]
blank = Image.new('RGB', (240, 160), '#101010')
self.assertFalse(remote_text_pixels(blank, page, viewport)['Passed'])
self.assertFalse(remote_text_pixels(Image.new('RGB', blank.size, '#e0e0e0'), page, viewport)['Passed'])
# This generated image tests only the verifier's positive/negative
# controls; run_bing never uses it or calls a host image generator.
control = blank.copy(); draw = ImageDraw.Draw(control)
for run in page['TextRuns']:
x, y = run['Bounds']['X'] + viewport[0], run['Bounds']['Y'] + viewport[1]
for offset in range(0, 30, 5): draw.rectangle((x + offset, y + 2, x + offset + 1, y + 8), fill='#e0e0e0')
self.assertTrue(remote_text_pixels(control, page, viewport)['Passed'])
changed = copy.deepcopy(page)
for run in changed['TextRuns']: run['Color'] = '#ff0000'
self.assertFalse(remote_text_pixels(control, changed, viewport)['Passed'])
self.assertFalse(remote_text_pixels(control, page, [viewport[0] + 100, viewport[1], *viewport[2:]])['Passed'])
def test_real_run_without_https_always_keeps_false_timeout_and_raw_packets(self):
with temporary_directory() as temporary:
def no_op(*_): pass
with self.assertRaisesRegex(AssertionError, 'timed out'):
run_bing(temporary, lambda: '', lambda: '', no_op, no_op, no_op, no_op, no_op, timeout=0)
result = json.loads((Path(temporary) / 'BingEvidence.json').read_text(encoding='utf-8'))
self.assertFalse(result['Passed']); self.assertFalse(result['NativeHttpsBingLoaded'])
self.assertFalse(result['HostBrowserScreenshots']); self.assertFalse(result['AllUserRequirementsMet'])
self.assertEqual(0, result['RawGuestEvidence']['Packets'])
self.assertTrue((Path(temporary) / 'BingRequestEvidence.log').exists())
def test_native_qemu_profile_has_exact_e1000_nat_cpu_and_utc_arguments(self):
base = ['qemu-system-x86_64', '-cpu', 'max', '-rtc', 'base=utc']
arguments = base + network_arguments(True)
profile = validate_network_arguments(arguments, True)
self.assertEqual(native_network_profile(True), profile)
self.assertEqual('8086:100e', profile['PciVendorDevice'])
self.assertEqual('Static', profile['AddressConfiguration'])
self.assertFalse(profile['HostProxy']); self.assertFalse(profile['HostForwarding'])
self.assertEqual(['-netdev', NAT_ARGUMENT, '-device', NIC_ARGUMENT], network_arguments(True))
self.assertFalse(validate_network_arguments(base + network_arguments(False), False)['Enabled'])
def test_wrong_nic_proxy_forwarding_offline_duplicate_cpu_and_rtc_are_rejected(self):
valid = ['-cpu', 'max', '-rtc', 'base=utc', *network_arguments(True)]
invalid = [valid + ['-net', 'none'], valid + ['-nic', 'user'],
valid + ['-device', 'usb-net,netdev=NativeNet'],
valid + ['-netdev', NAT_ARGUMENT], valid + ['-cpu', 'max'],
[item.replace('e1000,', 'e1000e,') for item in valid],
[item.replace('dns=10.0.2.3', 'dns=127.0.0.1') for item in valid],
[item.replace('base=utc', 'base=localtime') for item in valid],
[item + ',hostfwd=tcp::8080-:80' if item == NAT_ARGUMENT else item for item in valid],
['-cpu', 'max', '-rtc', 'base=utc', '-net', 'none']]
for arguments in invalid:
with self.subTest(arguments=arguments), self.assertRaises(AssertionError):
validate_network_arguments(arguments, True)
def test_presentation_requires_latest_exact_snapshot_url_scroll_and_real_scanout(self):
def marker(snapshot=12, address=HOME, sequence=6, ticks=100, scroll=0, window=3):
return f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|{window}|Generation|2|Address|{address}|SnapshotSequence|{snapshot}|PresentationSequence|{sequence}|PresentationTicks|{ticks}|ScrollY|{scroll}\n'
valid = marker()
value = presentation_identity(matching_presentation(valid, remote(), 3))
self.assertEqual(12, value['SnapshotSequence']); self.assertEqual(100, value['PresentationTicks'])
for serial in ('', marker(snapshot=13), marker(address='https://example.com/'), marker(sequence=0),
marker(ticks=0), marker(scroll=5), marker(window=4), valid + marker(snapshot=13)):
with self.subTest(serial=serial), self.assertRaises(AssertionError):
matching_presentation(serial, remote(), 3)
with self.assertRaises(AssertionError): matching_presentation(valid, remote(), 3, len(valid))
def test_screenshot_requires_unchanged_snapshot_and_monotonic_presentation_counters(self):
initial = {'WindowId': 3, 'Generation': 2, 'Address': HOME, 'SnapshotSequence': 12,
'PresentationSequence': 6, 'PresentationTicks': 100, 'ScrollY': 0}
later = {**initial, 'PresentationSequence': 7, 'PresentationTicks': 110}
self.assertTrue(stable_screenshot_presentation(initial, later)['StableSnapshot'])
for field, value in (('WindowId', 4), ('Generation', 3), ('Address', 'about:Start'),
('SnapshotSequence', 13), ('PresentationSequence', 5), ('PresentationTicks', 99), ('ScrollY', 5)):
with self.subTest(field=field), self.assertRaises(AssertionError):
stable_screenshot_presentation(initial, {**later, field: value})
def test_capture_rejects_snapshot_advancing_during_the_actual_screenshot(self):
initial=f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|3|Generation|2|Address|{HOME}|SnapshotSequence|12|PresentationSequence|6|PresentationTicks|100|ScrollY|0\n'
changed=initial+initial.replace('SnapshotSequence|12','SnapshotSequence|13').replace('Sequence|6|','Sequence|7|').replace('Ticks|100','Ticks|110')
captures=[]
with self.assertRaisesRegex(AssertionError,'differs'):
capture_matching_presentation(initial,remote(),3,0,captures.append,lambda:changed,'attempt')
self.assertEqual(['attempt'],captures)
def test_capture_rejects_stale_dom_before_taking_any_screenshot(self):
latest=f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|3|Generation|2|Address|{HOME}|SnapshotSequence|13|PresentationSequence|7|PresentationTicks|110|ScrollY|0\n'
captures=[]
with self.assertRaisesRegex(AssertionError,'differs'):
capture_matching_presentation(latest,remote(),3,0,captures.append,lambda:latest,'attempt')
self.assertEqual([],captures)
def test_capture_accepts_only_a_stable_snapshot_on_both_sides(self):
initial=f'XFE_BROWSER_BACKGROUND_PRESENTED|Window|3|Generation|2|Address|{HOME}|SnapshotSequence|12|PresentationSequence|6|PresentationTicks|100|ScrollY|0\n'
later=initial+initial.replace('Sequence|6|','Sequence|7|').replace('Ticks|100','Ticks|110')
captures=[]
proof=capture_matching_presentation(initial,remote(),3,0,captures.append,lambda:later,'attempt')
self.assertEqual(['attempt'],captures);self.assertTrue(proof['StableSnapshot'])
self.assertEqual(12,proof['After']['SnapshotSequence']);self.assertEqual(7,proof['After']['PresentationSequence'])
def test_already_open_browser_requires_actual_navigation_start_offset(self):
with temporary_directory() as temporary, self.assertRaisesRegex(AssertionError, 'start offset'):
run_bing(temporary, lambda: '', lambda: '', lambda *_: None,
lambda *_: None, lambda *_: None, lambda *_: None, lambda *_: None,
open_browser=False)
def test_powershell_launcher_checks_actual_receipt_hashes_and_snapshot_bindings(self):
"""Execute only Open.ps1's real read-only Bing verifier against small guard fixtures.
These fixtures are neither guest evidence nor boot proofs. No launcher,
image copy, host network request, display or VM is invoked by this test.
"""
source = (ROOT / 'Tools/IndependentKernel/Open.ps1').read_text(encoding='utf-8-sig')
helpers = source[source.index('function Get-IndependentHash'):source.index('$sourceImage=')]
guard = source[source.index('$bingBinding='):source.index('foreach($capture in $documentWorkflow.Screenshots.')]
with temporary_directory() as temporary:
root = Path(temporary)
cases = []
for name in ('valid', 'tampered-png', 'tampered-log', 'wrong-body', 'wrong-snapshot',
'unstable', 'missing-generation', 'escaped-png', 'no-glyphs', 'unchecked-tls', 'parity-overclaim',
'old-acceptance', 'no-photograph', 'no-search', 'wrong-query', 'no-png', 'tampered-search',
'no-scroll', 'late-timeout', 'missing-icons', 'unpainted-icons', 'no-home-content', 'unpainted-home-content', 'unfinished-scripts', 'no-repeated-search', 'wrong-repeated-query', 'tampered-repeated-search','no-full-content','citation-overlap','visible-image-missing','content-gap','no-chevrons','clipped-navigation','repeated-placeholders'):
directory = root / name; directory.mkdir()
image = directory / 'Browser-Bing-Native-1.png'
Image.new('RGB', (2, 2), '#101010').save(image)
raw = directory / 'BingRequestEvidence.log'; raw.write_text('Verifier unit-test fixture only', encoding='utf-8')
sha = lambda path: hashlib.sha256(path.read_bytes()).hexdigest()
presentation = {'WindowId': 3, 'Generation': 2, 'Address': HOME, 'SnapshotSequence': 12,
'PresentationSequence': 6, 'PresentationTicks': 100, 'ScrollY': 0}
capture = {'Path': image.name, 'Sha256': sha(image), 'BodySha256': 'a' * 64,
'PresentationCorrelation': {'Before': dict(presentation), 'After': dict(presentation), 'StableSnapshot': True},
'TextPixels': {'Passed': True}, 'PhotographPixels': {'Passed': True}, 'HomeContentPixels': {'Passed': True}}
home_page = remote(); home_page.update(PendingResources=0, DecodedImageFormats=['WebP', 'PNG'], ViewportWidth=1276)
for link in home_page['NavigationLinks']:link['ContainerBounds']['Width']=1276
search_page = copy.deepcopy(home_page)
search_page.update(Address='https://cn.bing.com/search?q=Microsoft+.NET', SnapshotSequence=13)
search_page['SearchInputs'][0]['Value'] = 'Microsoft .NET'
search_page['LoadedImages'] = [{'Format': 'PNG', 'Width': 32, 'Height': 32}]
scrolled_page = copy.deepcopy(search_page); scrolled_page.update(SnapshotSequence=14, ScrollY=80)
def search_capture(filename, page):
path = directory / filename; Image.new('RGB', (2, 2), '#202020').save(path)
identity = {**presentation, 'Address': page['Address'], 'SnapshotSequence': page['SnapshotSequence'], 'ScrollY': page['ScrollY']}
return {'Path': filename, 'Sha256': sha(path), 'BodySha256': page['BodySha256'], 'ScrollY': page['ScrollY'],
'PresentationCorrelation': {'Before': dict(identity), 'After': dict(identity), 'StableSnapshot': True}, 'TextPixels': {'Passed': True}, 'IconPixels': {'Passed': True}}
search = {'Query': 'Microsoft .NET', 'Responses': [response(search_page['Address'])['Value']], 'RemotePage': search_page,
'ScrolledRemotePage': scrolled_page,
'VisibleResultLinks': [{'Address': 'https://dotnet.microsoft.com/download', 'Text': 'Download .NET', 'Bounds': {'Width': 130, 'Height': 20}}],
'Screenshot': search_capture('Search.png', search_page), 'ScrolledScreenshot': search_capture('Scroll.png', scrolled_page)}
repeated_page=copy.deepcopy(search_page);repeated_page.update(Address='https://cn.bing.com/search?q=Microsoft+CSharp',SnapshotSequence=15,BodySha256='b'*64)
repeated_page['SearchInputs'][0]['Value']='Microsoft CSharp'
repeated_response=response(repeated_page['Address'])['Value'];repeated_response['BodySha256']='b'*64
repeated={'Query':'Microsoft CSharp','Responses':[repeated_response],'RemotePage':repeated_page,'VisibleResultLinks':[{'Address':'https://learn.microsoft.com/dotnet/csharp/','Text':'C# programming','Bounds':{'Width':130,'Height':20}}],
'Screenshot':search_capture('Repeated.png',repeated_page)}
for item in (search_page,repeated_page):
item['TextRuns']=[{'Text':'domain','Bounds':{'X':10,'Y':55,'Width':40,'Height':10},'Ancestors':'cite|div|li','ListItemId':10},
{'Text':'Title','Bounds':{'X':10,'Y':70,'Width':50,'Height':15},'Ancestors':'a|h2|li','ListItemId':10}]
search_page['DocumentHeight']=180;scrolled_page['DocumentHeight']=180
search['FullContent']={'Passed':True,'ReachedBottom':True,'Frames':[{'RemotePage':search_page,'Screenshot':search['Screenshot']},{'RemotePage':scrolled_page,'Screenshot':search['ScrolledScreenshot']}]}
repeated['FullContent']={'Passed':True,'ReachedBottom':True,'Frames':[{'RemotePage':repeated_page,'Screenshot':repeated['Screenshot']}]}
record = {'Kind': 'XfeIndependentNativeBingEvidence', 'Passed': True, 'NativeHttpsBingLoaded': True,
'AcceptanceVersion': 9,'NavigationVisible':True, 'VisibleContentLoaded':True,'CssFixesVerified':True,'HomeChevronPixels':{'Passed':True}, 'VisualHomeLoaded': True, 'RealSearchCompleted': True, 'RepeatedSearchCompleted':True,'RepeatedSearch':repeated,'SearchScrolled': True, 'DecodedPngVerified': True, 'SearchImagesLoaded': True,
'HomeContent': {'Passed': True, 'NavigationLinks': [{}]*4, 'HotSearchLinks': [{}]*6},
'AllUserRequirementsMet': False, 'FullCssParityVerified': False, 'CompleteJavaScriptCompatibility': False,
'HostHttpRequests': False, 'HostBrowserScreenshots': False, 'HostJavaScriptInjection': False,
'VmStartedByThisTool': False, 'RequestedUrl': HOME, 'Responses': [response()['Value']],
'RemotePage': home_page, 'Presentation': presentation, 'Screenshots': [capture], 'Search': search,
'RawGuestEvidence': {'Path': raw.name, 'Sha256': sha(raw), 'Packets': 2, 'PresentationMarkers': 1}}
if name == 'tampered-png': image.write_bytes(b'changed PNG')
elif name == 'tampered-log': raw.write_text('changed log', encoding='utf-8')
elif name == 'wrong-body': capture['BodySha256'] = 'b' * 64
elif name == 'wrong-snapshot': capture['PresentationCorrelation']['Before']['SnapshotSequence'] = 13
elif name == 'unstable': capture['PresentationCorrelation']['StableSnapshot'] = False
elif name == 'missing-generation':
del capture['PresentationCorrelation']['Before']['Generation']; del capture['PresentationCorrelation']['After']['Generation']
elif name == 'escaped-png': capture['Path'] = '../outside.png'
elif name == 'no-glyphs': capture['TextPixels']['Passed'] = False
elif name == 'unchecked-tls': record['Responses'][0]['CertificateValidated'] = False
elif name == 'parity-overclaim': record['AllUserRequirementsMet'] = True
elif name == 'old-acceptance': record['AcceptanceVersion'] = 2
elif name == 'no-photograph': capture['PhotographPixels']['Passed'] = False
elif name == 'no-search': del record['Search']
elif name == 'wrong-query': search_page['SearchInputs'][0]['Value'] = 'different query'
elif name == 'no-png':
home_page['DecodedImageFormats'] = ['WebP']; search_page['DecodedImageFormats'] = ['WebP']
elif name == 'tampered-search': (directory / 'Search.png').write_bytes(b'changed search PNG')
elif name == 'no-scroll': scrolled_page['ScrollY'] = 0
elif name == 'late-timeout': scrolled_page['Diagnostics'] = 'JavaScript error: The operation has timed out.'
elif name == 'missing-icons': search_page['LoadedImages'] = []
elif name == 'unpainted-icons': search['Screenshot']['IconPixels']['Passed'] = False
elif name == 'no-home-content': record['HomeContent']['HotSearchLinks'] = []
elif name == 'unpainted-home-content': capture['HomeContentPixels']['Passed'] = False
elif name == 'unfinished-scripts': search_page['DocumentInitializationComplete'] = False
elif name == 'no-repeated-search': del record['RepeatedSearch']
elif name == 'wrong-repeated-query': repeated_page['SearchInputs'][0]['Value']='old query'
elif name == 'tampered-repeated-search': (directory/'Repeated.png').write_bytes(b'changed repeated search PNG')
if name=='no-full-content':del search['FullContent']
elif name=='citation-overlap':search_page['TextRuns'][0]['Bounds']['Height']=30
elif name=='visible-image-missing':repeated_page['MissingVisibleImages']=[{'Source':'missing.png'}]
elif name=='content-gap':scrolled_page['ScrollY']=150
elif name=='no-chevrons':record['HomeChevronPixels']['Passed']=False
elif name=='clipped-navigation':search_page['NavigationLinks'][2]['Painted']=False
elif name=='repeated-placeholders':repeated_page['LoadedImages']=[]
receipt = directory / 'BingEvidence.json'; receipt.write_text(json.dumps(record), encoding='utf-8')
binding = {'Path': receipt.name, 'Sha256': sha(receipt), 'Passed': True, 'NativeHttpsBingLoaded': True}
cases.append({'Name': name, 'Root': str(directory), 'Binding': binding, 'Expected': name == 'valid'})
fixtures = root / 'Cases.json'; fixtures.write_text(json.dumps(cases), encoding='utf-8')
script = root / 'Check.ps1'
script.write_text("param([string]$Cases)\n$ErrorActionPreference='Stop'\n"
"Import-Module (Join-Path $PSHOME 'Modules/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1')\n" + helpers +
"\n$check={\n" + guard + "\n}\n$results=@(foreach($case in (Get-Content -Raw -Encoding UTF8 $Cases | ConvertFrom-Json)){\n"
"$desktopInput=[pscustomobject]@{BingEvidence=$case.Binding};$guestEvidenceRoot=$case.Root;$passed=$false;$reason=$null\n"
"try{& $check;$passed=$true}catch{$reason=$_.Exception.Message}\n"
"[pscustomobject]@{Name=$case.Name;Passed=$passed;Expected=$case.Expected;Reason=$reason}\n"
"});ConvertTo-Json -Depth 5 -InputObject $results\n", encoding='utf-8-sig')
completed = subprocess.run(['powershell.exe', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', str(script), '-Cases', str(fixtures)],
cwd=ROOT, check=True, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=30)
results = json.loads(completed.stdout)
self.assertEqual(len(cases), len(results))
for result in results:
with self.subTest(case=result['Name']):
self.assertEqual(result['Expected'], result['Passed'], result.get('Reason'))
if __name__ == '__main__':
unittest.main()