forked from mtusi2026/mtusi2026lab5
80 lines
3.7 KiB
Python
80 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Временная ошибка S3: attempt растёт, backoff+jitter, затем ready."""
|
|
import subprocess, sys, time
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from tests.client import Client, check
|
|
from student_tests.common import upload, wait_status, psql, docker, ok, fail
|
|
|
|
def main():
|
|
c = Client()
|
|
a, b, e, ch = c.bootstrap()
|
|
content = Path('tests/fixtures/sample.png').read_bytes()
|
|
|
|
# 1) Настраиваем mc с кредами — иначе удалить/положить объект не сможем.
|
|
root_user = open('secrets/minio_root_user.txt').read().strip()
|
|
root_pass = open('secrets/minio_root_password.txt').read().strip()
|
|
docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'loc',
|
|
'http://localhost:9000', root_user, root_pass, '--api', 'S3v4')
|
|
|
|
# 2) Пауза worker’а: job не будет взят до unpause.
|
|
docker('pause', 'worker')
|
|
try:
|
|
att = upload(c, a, ch, content)
|
|
staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
|
|
check(staging.startswith('staging/'), 'unexpected staging_key')
|
|
|
|
# 3) Удаляем staging: это гарантирует, что worker НЕ прочитает объект.
|
|
# Если worker ещё не стартовал — GetObject вернёт NoSuchKey.
|
|
subprocess.run(
|
|
['docker', 'compose', 'exec', '-T', 's3', 'mc', 'rm',
|
|
'loc/messenger-attachments/' + staging],
|
|
capture_output=True, text=True, check=True)
|
|
|
|
# 4) Отдаём job worker’у.
|
|
docker('unpause', 'worker')
|
|
|
|
# 5) Ждём, пока attempt >= 2.
|
|
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
|
deadline = time.monotonic() + 90
|
|
attempt = 1
|
|
while time.monotonic() < deadline:
|
|
attempt = int(psql("SELECT attempt FROM attachment_jobs WHERE job_id='%s'" % job))
|
|
if attempt >= 2:
|
|
break
|
|
time.sleep(1)
|
|
check(attempt >= 2,
|
|
'retry did not happen on missing staging object (attempt=%d)' % attempt)
|
|
ok('transient error retried: attempt=%d' % attempt)
|
|
|
|
# 6) Логи retry именно для нашего job_id.
|
|
logs = docker('logs', '--tail=500', 'worker')
|
|
lines = [l for l in logs.splitlines()
|
|
if ('retry job=' + job) in l]
|
|
check(len(lines) >= 1, 'no retry log lines for job %s:\n%s' % (job, logs[-1500:]))
|
|
ok('retry log lines:')
|
|
for l in lines:
|
|
print(' ' + l)
|
|
|
|
# 7) Возвращаем staging — job должен пройти retry и стать ready.
|
|
# Перезаписываем объект с правильным содержимым.
|
|
# Через mc pipe: передаём байты на stdin.
|
|
r = subprocess.run(
|
|
['docker', 'compose', 'exec', '-T', 's3',
|
|
'mc', 'pipe', 'loc/messenger-attachments/' + staging],
|
|
input=content, capture_output=True)
|
|
check(r.returncode == 0, 'failed to restore staging: %s' % r.stderr.decode())
|
|
|
|
att = wait_status(c, a, att['id'], 'ready', seconds=180)
|
|
ok('recovered to ready after staging restored (attempt=%d)' % attempt)
|
|
|
|
finally:
|
|
# На всякий случай — если упали в середине, worker не должен остаться на паузе.
|
|
subprocess.run(['docker', 'compose', 'unpause', 'worker'],
|
|
capture_output=True)
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as exc:
|
|
fail(str(exc)) |