Files
mtusi2026lab7/student-tests/reprocess_v2.py
T

80 lines
5.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Трек 2, versioned pipeline: выкатка pipeline v2 без потери работы и без перезаписи.
1. Новые загрузки сразу получают v2: api пересоздаётся с PIPELINE_VERSION=2 (worker уже поддерживает v1 и v2).
2. Старые ready-вложения: `jobs reprocess --version=2`; повтор команды ничего не ставит (ключ attachment_id + версия).
3. Пока идёт переобработка, вложение остаётся ready и отдаёт прежние варианты: в processing не откатывается.
4. Итог: у всех ready версия 2; для одинаковых байтов v2 детерминирована (sha256 совпадает с новой загрузкой);
у каждой пары (вложение, версия) ровно 3 варианта, варианты v1 остаются историей и не отдаются.
В конце api возвращается к PIPELINE_VERSION из .env. Запуск: python3 student-tests/reprocess_v2.py --base-url http://localhost:8080"""
import hashlib
import time
from failures import PNG, sql
from pipeline_faults import compose, healthy, jobs, rows, wait_for
from client import Client, arguments, check, run
def main():
args = arguments('Pipeline v1 -> v2 rollout and reprocess')
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, _, chat = c.bootstrap()
def upload(name):
att = c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), 202, actor=alice, raw=PNG,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
return c.wait_ready(bob, att)
def thumbnail(att):
return hashlib.sha256(c.call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'], 200, actor=bob)['bytes']).hexdigest()
def version(att):
return sql("SELECT pipeline_version FROM attachments WHERE id = '%s'" % att['id'])
def by_version():
return ', '.join('v%s: %s' % tuple(r) for r in rows("SELECT pipeline_version, count(*) FROM attachments WHERE state = 'ready' GROUP BY 1 ORDER BY 1"))
old = upload('before-v2.png')
old_thumb = thumbnail(old)
check(version(old) == '1', 'expected v1 before rollout')
print('до выкатки: ready по версиям — %s; вложение %s v1' % (by_version(), old['id']))
try:
compose('up', '-d', '--no-deps', 'api', env={'PIPELINE_VERSION': '2'})
wait_for('api healthy with PIPELINE_VERSION=2', lambda: healthy('api'), 90)
new = upload('after-v2.png')
check(version(new) == '2' and thumbnail(new) != old_thumb, 'new upload is not v2')
print('api с PIPELINE_VERSION=2: новая загрузка %s сразу v2 (thumbnail отличается от v1)' % new['id'])
started = time.monotonic()
first = jobs('reprocess', '--version=2')[-1]
states = set()
while True:
states.add(c.call('GET', '/api/v1/attachments/' + old['id'], 200, actor=bob, schema='Attachment')['json']['status'])
thumbnail(old) # во время переобработки вариант доступен
if not int(sql("SELECT count(*) FROM jobs WHERE pipeline_version = 2 AND state IN ('queued', 'processing')")):
break
check(time.monotonic() - started < 600, 'reprocess did not finish')
time.sleep(0.5)
elapsed = time.monotonic() - started
second = jobs('reprocess', '--version=2')[-1]
check(states == {'ready'}, 'attachment left ready during reprocess: %s' % states)
check(second['enqueued'] == 0, 'second reprocess enqueued %s' % second)
print('reprocess --version=2: поставлено %d, все done за %.1f с (%.1f вложений/с); статус во время: %s; повтор: поставлено %d'
% (first['enqueued'], elapsed, first['enqueued'] / max(elapsed, 0.001), ', '.join(sorted(states)), second['enqueued']))
left = rows("SELECT id FROM attachments WHERE state = 'ready' AND pipeline_version < 2")
check(not left, 'ready attachments still below v2: %d' % len(left))
bad = rows("SELECT attachment_id, pipeline_version, count(*) FROM attachment_variants GROUP BY 1, 2 HAVING count(*) <> 3")
check(not bad, 'variant sets with count != 3: %s' % bad)
failed = rows("SELECT state, count(*) FROM jobs WHERE pipeline_version = 2 GROUP BY 1")
check(version(old) == '2' and thumbnail(old) == thumbnail(new) != old_thumb, 'reprocessed result differs from fresh v2')
print('итог: ready по версиям — %s; jobs v2: %s; у всех пар (вложение, версия) по 3 варианта;'
% (by_version(), ', '.join('%s %s' % tuple(r) for r in failed)))
print(' %s после reprocess отдаёт v2, thumbnail побайтно равен свежей загрузке v2 (детерминизм pipeline)' % old['id'])
finally:
compose('up', '-d', '--no-deps', 'api')
wait_for('api healthy', lambda: healthy('api'), 90)
print('PASS: v2 rollout — new uploads v2, reprocess idempotent, no rollback of ready, deterministic result')
if __name__ == '__main__':
run(main)