ЛР 4 — БВТ2304 — Черников — на 4

This commit is contained in:
2026-11-05 13:41:57 +03:00
parent ec540a1a72
commit f878147188
61 changed files with 4622 additions and 21 deletions
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env python3
"""EXPLAIN (ANALYZE, BUFFERS) of the history page query on 10 000+ messages.
python scripts/explain_history.py --output evidence/explain.txt
The stand must be running. The script registers two synthetic users through the API,
creates a target chat and a noise chat, bulk-inserts messages with SQL, runs ANALYZE and
prints the plans of the same query the API runs (src/messenger/db.py, list_messages):
first page, a page in the middle, the last page, the generic plan of the parameterized
query, and for comparison the middle page without the index (DROP INDEX inside a
transaction that is rolled back, so the database is left unchanged).
Standard library only.
"""
import argparse
import base64
import json
from pathlib import Path
import subprocess
import sys
import time
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
BASE = 'http://127.0.0.1:8080'
PSQL = ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-X', '-v', 'ON_ERROR_STOP=1']
PAGE = 51 # limit 50 + 1 row to learn whether the next page exists, as the API does
lines = []
def out(text=''):
lines.append(text)
print(text, flush=True)
def psql(sql, echo=True):
"""Run SQL in one psql session; with echo the transcript shows every statement before its result."""
args = PSQL + (['--echo-all'] if echo else ['-At']) + ['-f', '-']
done = subprocess.run(args, cwd=str(ROOT), input=sql.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
text = done.stdout.decode('utf-8', 'replace').strip()
if done.returncode != 0:
out(text)
raise SystemExit('psql failed')
return text
def api(method, path, body=None, headers=None):
headers = dict(headers or {})
data = None
if body is not None:
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
with urllib.request.urlopen(request, timeout=10) as response:
return json.loads(response.read())
def user(label):
username = label + '_' + uuid.uuid4().hex[:10]
password = 'Explain-' + uuid.uuid4().hex
created = api('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
credential = base64.b64encode((username + ':' + password).encode()).decode()
session = api('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
return created['id'], session['token']
def page_sql(chat_id, after=None):
"""The same SQL text as Repository.list_messages, with literal values."""
sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' " % chat_id
if after:
sql += "AND (created_at, id) > ('%s', '%s') " % after
return sql + 'ORDER BY created_at, id LIMIT %d' % PAGE
def main():
parser = argparse.ArgumentParser(description='EXPLAIN of the history query; inserts synthetic messages')
parser.add_argument('--messages', type=int, default=10000, help='messages in the target chat (>= 10000)')
parser.add_argument('--noise', type=int, default=20000, help='messages in another chat')
parser.add_argument('--output', default=None)
args = parser.parse_args()
sys.stdout.reconfigure(errors='replace')
out('# %s UTC, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), psql('SELECT version()', echo=False)))
alice, alice_token = user('explain_a')
bob, _ = user('explain_b')
target = api('POST', '/api/v1/chats', {'title': 'EXPLAIN target', 'member_ids': [alice, bob]},
{'Authorization': 'Bearer ' + alice_token})['id']
noise = api('POST', '/api/v1/chats', {'title': 'EXPLAIN noise', 'member_ids': [alice]},
{'Authorization': 'Bearer ' + alice_token})['id']
out('# target chat %s: %d messages; noise chat %s: %d messages' % (target, args.messages, noise, args.noise))
started = time.monotonic()
psql('''
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '%(target)s', CASE WHEN i %% 2 = 0 THEN '%(alice)s'::uuid ELSE '%(bob)s'::uuid END,
'seed message ' || i, clock_timestamp() + i * interval '1 millisecond'
FROM generate_series(1, %(messages)d) AS i;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '%(noise)s', '%(alice)s', 'noise message ' || i, clock_timestamp() + i * interval '1 millisecond'
FROM generate_series(1, %(noise_count)d) AS i;
ANALYZE messages;
''' % {'target': target, 'noise': noise, 'alice': alice, 'bob': bob, 'messages': args.messages, 'noise_count': args.noise},
echo=False)
out('# inserted and analyzed in %.1f s' % (time.monotonic() - started))
out(psql("SELECT 'messages total: ' || count(*) || ', in target chat: ' || count(*) FILTER (WHERE chat_id = '%s') FROM messages" % target, echo=False))
def key_at(position):
row = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET %d LIMIT 1"
% (target, position - 1), echo=False)
return tuple(row.split('|'))
middle, last = key_at(args.messages // 2), key_at(args.messages - 20)
explain = 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) '
out()
out('== 1. First page, middle page and last page: index range scan, no sort ==')
out(psql('\n'.join(explain + page_sql(target, after) + ';' for after in (None, middle, last))))
out()
out('== 2. Generic plan of the parameterized query (how a prepared statement with $1..$4 runs) ==')
out(psql('''SET plan_cache_mode = force_generic_plan;
PREPARE history_page(uuid, timestamptz, uuid, int) AS
SELECT id, chat_id, sender_id, text, created_at FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4;
%sEXECUTE history_page('%s', '%s', '%s', %d);''' % (explain, target, middle[0], middle[1], PAGE)))
out()
out('== 3. For comparison: the same middle page without the index (rolled back) ==')
out(psql('''BEGIN;
DROP INDEX messages_chat_history_idx;
%s%s;
ROLLBACK;''' % (explain, page_sql(target, middle))))
out(psql("SELECT 'index still exists: ' || (to_regclass('messages_chat_history_idx') IS NOT NULL)", echo=False))
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
if __name__ == '__main__':
main()
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Create local secret files for the compose stand.
python scripts/init_secrets.py
Passwords are random and synthetic; they are created only if missing and never printed.
The URL/config files are always rebuilt from the password files, so the pair stays consistent.
secrets/ is listed in .gitignore and .dockerignore.
PostgreSQL applies its password only when the data volume is created. If secrets/ is
deleted while the pgdata volume is kept, restore the old files or recreate the volume
(docker compose down -v deletes ALL lab data).
"""
from pathlib import Path
import json
import secrets
ROOT = Path(__file__).resolve().parents[1]
DIR = ROOT / 'secrets'
def write(name, content):
path = DIR / name
path.write_text(content, encoding='utf-8', newline='\n')
# Readable for the non-root users inside containers (files are bind-mounted as is).
path.chmod(0o644)
def password(name):
path = DIR / name
if path.is_file() and path.read_text(encoding='utf-8').strip():
print('keep secrets/' + name)
return path.read_text(encoding='utf-8').strip()
value = secrets.token_hex(24) # hex: safe inside URLs without escaping
write(name, value + '\n')
print('created secrets/' + name)
return value
def main():
DIR.mkdir(exist_ok=True)
postgres = password('postgres_password')
valkey = password('valkey_password')
write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger\n' % postgres)
write('valkey_auth', 'requirepass %s\n' % valkey)
write('cache_url', 'valkey://:%s@valkey:6379/0\n' % valkey)
admin_key = password('s3_admin_access_key_id')
admin_secret = password('s3_admin_secret_access_key')
api_key = password('s3_access_key_id')
api_secret = password('s3_secret_access_key')
identities = [
dict(name='bootstrap', credentials=[dict(accessKey=admin_key, secretKey=admin_secret)],
actions=['Admin', 'Read', 'Write', 'List', 'Tagging']),
dict(name='messenger-api', credentials=[dict(accessKey=api_key, secretKey=api_secret)],
actions=['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'])]
# Нет anonymous identity: ни объект, ни список bucket не доступны анонимно.
write('s3_config', json.dumps(dict(identities=identities)) + '\n')
print('written secrets/database_url, secrets/valkey_auth, secrets/cache_url')
if __name__ == '__main__':
main()
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""Повторяемые интеграционные проверки lab4. Запуск на локальном синтетическом стенде.
Ничего не удаляет из volumes. Отказы восстанавливаются в finally; токены не печатаются.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import random
import socket
import struct
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
def command(*args):
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace')
if result.returncode:
raise RuntimeError('Command failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1500:])
return result.stdout.strip()
def compose(*args):
return command('docker', 'compose', *args)
def sql(query):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
def ready(c):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.5)
raise AssertionError('API did not become ready')
def stack(c):
config = json.loads(compose('config', '--format', 'json'))
for name in ('api', 'postgres', 'valkey', 's3'):
check(not config['services'][name].get('ports'), name + ': internal ports must not be published')
check(config['services']['api']['deploy']['replicas'] >= 2, 'two replicas required')
compose('run', '--rm', '--no-deps', 'bucket')
print('PASS stack: internal ports, 2 replicas, repeated bucket bootstrap')
def private(c):
a, b, eve, chat = c.bootstrap()
att, data = c.attachment(a, chat)
# Ключ доступен только оператору через БД, через публичный API его нет.
key = sql("SELECT object_key FROM attachments WHERE id='%s'" % att['id'])
code = """import urllib.request, urllib.error
from messenger.objects import ObjectStore
from botocore.exceptions import ClientError
s=ObjectStore()
try:
urllib.request.urlopen('http://s3:8333/'+s.bucket+'/'+%r)
raise AssertionError('anonymous GET allowed')
except urllib.error.HTTPError as e:
assert e.code==403, e.code
try:
s.client.create_bucket(Bucket='lab4-forbidden-bucket')
raise AssertionError('API has admin rights')
except ClientError as e:
assert e.response['ResponseMetadata']['HTTPStatusCode']==403
print('PASS private: anonymous object GET 403; API cannot create buckets')
""" % key
print(compose('exec', '-T', 'api', 'python', '-c', code))
def failover(c):
a, b, eve, chat = c.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', 201,
actor=a, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
target = result['headers']['X-Instance-Id']
ids = compose('ps', '-q', 'api').splitlines()
check(any(i.startswith(target) for i in ids), 'upload instance must be a replica of this stack')
try:
command('docker', 'stop', target)
ready(c)
download = c.call('GET', '/api/v1/attachments/' + result['json']['id'] + '/content', actor=b)
check(download['headers']['X-Instance-Id'] != target, 'download used stopped replica')
check(download['bytes'] == data, 'failover changed bytes')
print('PASS upload replica stopped -> another replica download; SHA256=' + hashlib.sha256(data).hexdigest())
finally:
command('docker', 'start', target)
compose('up', '-d', '--wait')
def outage(c):
a, b, eve, chat = c.bootstrap()
att, content = c.attachment(a, chat)
start = time.monotonic()
try:
compose('stop', 's3')
# После остановки dependency LB тоже исключает реплики. Проверяем обычным HTTP:
# созданный LB ответ вправе не иметь X-Instance-Id.
import urllib.request
import urllib.error
req = urllib.request.Request(c.base + '/api/v1/attachments/' + att['id'] + '/content',
headers={'Authorization': 'Bearer ' + b['token']})
try:
urllib.request.urlopen(req, timeout=6)
raise AssertionError('S3 down but content succeeded')
except urllib.error.HTTPError as e:
check(e.code == 503, 'S3 down must return 503')
payload = json.loads(e.read())
check(payload['error']['code'] == 'unavailable', 'invalid outage error')
print('PASS S3 unavailable -> 503; elapsed_s=' + str(round(time.monotonic()-start, 3)))
finally:
compose('start', 's3')
ready(c)
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=b)['bytes'] == content,
'S3 outage lost object')
print('PASS same token and original after S3 recovery')
def png_noise():
# 3 MiB несжимаемых пикселей, реальный PNG. Случайность фиксирована.
rng = random.Random(42)
raw = b''.join(b'\0' + rng.randbytes(1024 * 3) for _ in range(1024))
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1024, 1024, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(raw)) + chunk(b'IEND', b''))
def memory(c):
a, b, eve, chat = c.bootstrap()
data = png_noise()
ids = compose('ps', '-q', 'api').splitlines()
# Kernel memory.peak учитывает дочерний decoder и tmpfs; sampling не пропустит пик.
before = [int(command('docker', 'exec', i, 'cat', '/sys/fs/cgroup/memory.current')) for i in ids]
barrier = threading.Barrier(10)
def upload(_):
barrier.wait()
start = time.monotonic()
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=memory.png', 201,
actor=a, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
return round(time.monotonic() - start, 3), r['headers']['X-Instance-Id'], r['json']
start = time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(upload, range(10)))
elapsed = time.monotonic() - start
peak = [int(command('docker', 'exec', i, 'cat', '/sys/fs/cgroup/memory.peak')) for i in ids]
for _, _, att in results:
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=b)
check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'], 'concurrent checksum mismatch')
print(json.dumps(dict(concurrent=10, successful=10, bytes_per_file=len(data), dimensions=[1024, 1024],
seconds=round(elapsed, 3), successful_rps=round(10/elapsed, 3),
latency_seconds=sorted(r[0] for r in results),
replicas=len(set(r[1] for r in results)), memory_before_bytes=before,
memory_peak_bytes=peak, peak_scope='container lifetime; cgroup includes decoder and tmpfs')))
def compensation(c):
a, b, eve, chat = c.bootstrap()
before = int(sql('SELECT count(*) FROM attachments'))
sql("""CREATE OR REPLACE FUNCTION lab4_force_failure() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN IF NEW.filename='lab4-force-rollback.png' THEN RAISE EXCEPTION 'synthetic lab4 failure'; END IF;
RETURN NEW; END $$;
CREATE TRIGGER lab4_force_failure BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_force_failure();""")
try:
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=lab4-force-rollback.png', 500,
actor=a, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'})
check(int(sql('SELECT count(*) FROM attachments')) == before, 'failed transaction left metadata')
finally:
sql('DROP TRIGGER IF EXISTS lab4_force_failure ON attachments; DROP FUNCTION IF EXISTS lab4_force_failure()')
print('PASS injected DB failure after S3 PUT: metadata rolled back, no 201; fresh orphan protected by grace')
print(compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'cleanup'))
# Обрыв клиента до завершения тела: INSERT/PUT ещё не выполнялись.
address = urlsplit(c.base)
with socket.create_connection((address.hostname, address.port or 80), timeout=5) as sock:
body = (ROOT / 'tests/fixtures/sample.png').read_bytes()
headers = ('POST /api/v1/chats/%s/attachments?filename=interrupted.png HTTP/1.1\r\n'
'Host: local\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\n'
'Content-Length: %d\r\n\r\n') % (chat['id'], a['token'], len(body))
sock.sendall(headers.encode() + body[:len(body)//2])
time.sleep(1)
check(int(sql('SELECT count(*) FROM attachments')) == before, 'interrupted upload committed metadata')
print('PASS interrupted upload: no attachment committed')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
parser.add_argument('--only', nargs='+', choices=['stack', 'private', 'failover', 'outage', 'memory', 'compensation'])
args = parser.parse_args()
c = Client(args.base_url, timeout=30)
for name in args.only or ['stack', 'private', 'failover', 'outage', 'memory', 'compensation']:
globals()[name](c)
if __name__ == '__main__':
main()
+10
View File
@@ -0,0 +1,10 @@
@echo off
rem Windows twin of scripts/restart.sh: tests/resilience.py starts the hook directly,
rem and Windows cannot execute a .sh file. Same actions, volumes are kept.
setlocal
cd /d "%~dp0.."
docker compose stop api || exit /b 1
docker compose restart postgres valkey s3 || exit /b 1
docker compose start api || exit /b 1
docker compose restart lb || exit /b 1
exit /b 0
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
# Hook for: make test-persistence ACTION_SCRIPT=scripts/restart.sh
# Restarts the whole stand: all API replicas, PostgreSQL, Valkey and the load balancer,
# keeping the named volumes: no "down -v", no FLUSHALL, no re-seed.
# The test itself waits for API readiness through the load balancer afterwards.
set -eu
cd "$(dirname "$0")/.."
docker compose stop api
docker compose restart postgres valkey s3
docker compose start api
docker compose restart lb
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Sanitized runtime snapshot: real processes, start times, limits and volumes; no environment/secrets."""
import datetime as dt
import json
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8')
def main():
ids = command('docker', 'compose', 'ps', '-a', '-q').split()
containers = json.loads(command('docker', 'inspect', *ids))
result = {'captured_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'containers': []}
for item in containers:
# Only these selected fields are persisted; Config.Env is deliberately excluded.
result['containers'].append(dict(
name=item['Name'].lstrip('/'), image=item['Config']['Image'],
status=item['State']['Status'], started_at=item['State']['StartedAt'],
health=item['State'].get('Health', {}).get('Status'),
memory_limit_bytes=item['HostConfig']['Memory'], cpu_limit_nanos=item['HostConfig']['NanoCpus'],
volumes=[dict(name=m.get('Name'), destination=m['Destination'])
for m in item['Mounts'] if m['Type'] == 'volume'],
published_ports=item['NetworkSettings']['Ports']))
text = json.dumps(result, indent=2)
(ROOT / 'evidence/stack.json').write_text(text + '\n', encoding='utf-8')
print('Saved evidence/stack.json (process start times, versions, limits, volumes and ports)')
if __name__ == '__main__':
main()
+19
View File
@@ -0,0 +1,19 @@
@echo off
rem Windows twin of scripts/stop-one.sh: tests/resilience.py starts the hook directly,
rem and Windows cannot execute a .sh file. Stops ONE api replica by X-Instance-Id (container ID).
setlocal
if "%TARGET_INSTANCE_ID%"=="" (
echo TARGET_INSTANCE_ID is not set 1>&2
exit /b 2
)
cd /d "%~dp0.."
set PROJECT=%COMPOSE_PROJECT_NAME%
if "%PROJECT%"=="" set PROJECT=mtusi-lab4
set ID=
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=%PROJECT%" --filter "label=com.docker.compose.service=api"') do set ID=%%i
if "%ID%"=="" (
echo TARGET_INSTANCE_ID is not a running api replica of %PROJECT% 1>&2
exit /b 1
)
docker stop %ID% >nul || exit /b 1
exit /b 0
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
# Hook for: make test-failover ACTION_SCRIPT=scripts/stop-one.sh
# Stops exactly ONE API replica and leaves it stopped. TARGET_INSTANCE_ID is the X-Instance-Id
# of the replica, which is its container hostname = short container ID. The hook checks that
# the ID belongs to a running "api" container of this compose project before stopping it.
# docker stop sends SIGTERM: the replica reports not ready, HAProxy removes it, then it exits.
# Restore afterwards: docker compose up -d (or docker start <id>)
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
project="${COMPOSE_PROJECT_NAME:-mtusi-lab4}"
id=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
--filter "label=com.docker.compose.project=$project" \
--filter "label=com.docker.compose.service=api")
if [ -z "$id" ]; then
echo "TARGET_INSTANCE_ID is not a running api replica of $project" >&2
exit 1
fi
docker stop "$id" >/dev/null
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Run acceptance checks and save real UTF-8 evidence. Requires an already running stand."""
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def main():
url = os.environ.get('BASE_URL', 'http://127.0.0.1:8080')
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
checks = [
('offline.txt', ['scripts/check_contract.py']),
('smoke.txt', ['tests/smoke.py', '--base-url', url]),
('tests.txt', ['-m', 'unittest', 'discover', '-s', 'student-tests', '-v']),
('lab4-checks.txt', ['scripts/lab4_checks.py', '--base-url', url]),
('persistence.txt', ['tests/resilience.py', '--mode', 'persistence', '--action-script', hook, '--base-url', url]),
('stack.txt', ['scripts/stack_evidence.py']),
]
for filename, args in checks:
result = subprocess.run([sys.executable, '-X', 'utf8', *args], cwd=ROOT,
env=dict(os.environ, BASE_URL=url), capture_output=True)
content = result.stdout.decode('utf-8', errors='replace') + result.stderr.decode('utf-8', errors='replace')
(ROOT / 'evidence' / filename).write_text(content, encoding='utf-8')
print(filename + ': ' + ('PASS' if result.returncode == 0 else 'FAIL'), flush=True)
if result.returncode:
print(content[-6000:], flush=True)
return result.returncode
return 0
if __name__ == '__main__':
sys.exit(main())