Files
mtusi2026lab7/src/ops.py
T
zemiles 9cc80d2529 ЛР 7 — БПИ2303 — Лазарев — на 3
Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS.

- Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2.
- Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку
  internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1.
- TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full
  (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert,
  mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape.
- OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором,
  auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login.
- Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и
  пользователи брокера выдаются из OpenBao.
- Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе.
- Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL,
  no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API.
- Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao,
  hardening; evidence, в т.ч. прогон из чистого clone.
2026-11-18 23:59:35 +03:00

221 lines
9.8 KiB
Python

#!/usr/bin/env python3
"""Операторский CLI (compose-сервис `ops`, profile tools). Вывод — JSON без секретов.
ops inspect <attachment_id> состояние: attachment, job, события, варианты, outbox, объекты S3
ops failed [--limit N] реестр failed-заданий (DLQ на стороне БД)
ops replay <attachment_id>... [--all-failed] [--error-code C] [--dry-run] [--reason TEXT]
явный replay failed -> queued (ready/queued/processing не трогаются)
ops dlq [--replay] [--purge-invalid]
сообщения в RabbitMQ DLQ: показать; --replay — разобрать каждое
по состоянию jobs (failed -> replay, остальное -> отбросить с audit)
ops publish-raw <json> учебное: опубликовать произвольное тело (проверка DLQ)
Replay всегда проходит через строку jobs под FOR UPDATE: повтор ready-задания отклоняется,
поэтому «повторить» не означает исполнить дубль уже успешной работы.
"""
import argparse
import json
import os
import sys
import pika
import psycopg
from psycopg.rows import dict_row
import common
import telemetry
import jobs
import objstore
EXTRA_ATTEMPTS = common.env_int("JOB_MAX_ATTEMPTS", 5)
def db():
return psycopg.connect(common.database_conninfo(), row_factory=dict_row)
def jsonable(value):
if isinstance(value, dict):
return {k: jsonable(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [jsonable(v) for v in value]
if isinstance(value, (str, int, float, bool)) or value is None:
return value
if hasattr(value, "isoformat"):
return value.isoformat().replace("+00:00", "Z")
return str(value)
def out(data):
print(json.dumps(jsonable(data), ensure_ascii=False, indent=1))
def s3_objects(prefix):
"""Только имена/размеры (ключ janitor: List, без чтения содержимого)."""
store = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments"))
try:
return sorted((o.object_name, o.size) for o in store.client.list_objects(store.bucket, prefix=prefix, recursive=True))
except Exception as exc:
return [("error", type(exc).__name__)]
def cmd_inspect(args):
with db() as conn:
att = conn.execute(
"""SELECT id, chat_id, status, error_code, content_type, size_bytes, sha256, width, height,
object_key, staging_key, pipeline_version, created_at, updated_at
FROM attachments WHERE id = %s""", (args.attachment_id,)).fetchone()
if att is None:
out({"error": "not_found"})
return 1
job = conn.execute(
"""SELECT id, pipeline_version, status, attempt, max_attempts, next_attempt_at, lease_owner,
lease_until, error_code, last_error, replay_count, created_at, updated_at, finished_at
FROM jobs WHERE attachment_id = %s ORDER BY pipeline_version""", (args.attachment_id,)).fetchall()
job_ids = [j["id"] for j in job]
events = conn.execute(
"SELECT job_id, at, event, attempt, actor, detail FROM job_events WHERE job_id = ANY(%s) ORDER BY id",
(job_ids,)).fetchall()
outbox = conn.execute(
"""SELECT id, payload->>'attempt' AS attempt, available_at, created_at, published_at, publish_count,
payload->>'traceparent' AS traceparent
FROM outbox WHERE job_id = ANY(%s) ORDER BY id""", (job_ids,)).fetchall()
variants = conn.execute(
"""SELECT pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256
FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name""",
(args.attachment_id,)).fetchall()
objects = s3_objects("attachments/%s/" % args.attachment_id)
staging = s3_objects(att["staging_key"]) if att["staging_key"] else []
out({"attachment": att, "jobs": job, "events": events, "outbox": outbox, "variants": variants,
"s3_final_objects": objects, "s3_staging_objects": staging})
return 0
def cmd_failed(args):
with db() as conn:
rows = conn.execute(
"""SELECT j.attachment_id, j.id AS job_id, j.error_code, j.attempt, j.max_attempts, j.replay_count,
j.last_error, j.updated_at, a.staging_key IS NOT NULL AS replayable
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.status = 'failed' ORDER BY j.updated_at DESC LIMIT %s""", (args.limit,)).fetchall()
out({"failed": rows, "count": len(rows)})
return 0
def cmd_replay(args):
results = []
with db() as conn:
ids = list(args.attachment_ids)
if args.all_failed:
q = "SELECT attachment_id FROM jobs WHERE status = 'failed'"
params = []
if args.error_code:
q += " AND error_code = %s"
params.append(args.error_code)
ids += [str(r["attachment_id"]) for r in conn.execute(q, params)]
for att_id in ids:
with conn.transaction():
result, job = jobs.replay(conn, att_id, "ops:" + common.HOSTNAME, EXTRA_ATTEMPTS,
args.reason, dry_run=args.dry_run)
results.append({"attachment_id": att_id, "result": result,
"job_id": str(job["job_id"]) if job else None})
out({"replay": results})
return 0 if all(r["result"] in ("replayed", "would_replay") for r in results) else 3
def broker_channel():
conn = pika.BlockingConnection(common.broker_parameters())
return conn, conn.channel()
def cmd_dlq(args):
conn, ch = broker_channel()
seen = []
tags = []
try:
with db() as dbc:
while True:
method, props, body = ch.basic_get(common.DLQ, auto_ack=False)
if method is None:
break
deaths = (props.headers or {}).get("x-death") or []
reason = deaths[0].get("reason") if deaths else None
item = {"reason": reason.decode() if isinstance(reason, bytes) else reason,
"delivery_count": (props.headers or {}).get("x-delivery-count"), "action": "kept"}
try:
msg = jobs.parse_message(body)
item["job_id"] = msg["job_id"]
if args.replay:
with dbc.transaction():
job = jobs.load_job(dbc, msg["job_id"], lock=True)
if job is None:
item["action"] = "kept_unknown_job"
elif job["job_status"] == "failed":
result, _ = jobs.replay(dbc, str(job["attachment_id"]), "ops-dlq", EXTRA_ATTEMPTS,
"dlq replay")
item["action"] = result
else:
# queued/processing/ready: состояние уже ведёт БД, дубль не нужен.
jobs.add_event(dbc, job["job_id"], "dlq_message_dropped", msg["attempt"], "ops-dlq",
{"job_status": job["job_status"], "reason": item["reason"]})
item["action"] = "dropped_job_" + job["job_status"]
except jobs.InvalidMessage as exc:
item["invalid"] = str(exc)
if args.purge_invalid:
item["action"] = "purged_invalid"
seen.append(item)
if item["action"].startswith("kept"):
tags.append(method.delivery_tag) # вернём в DLQ после обхода
else:
ch.basic_ack(method.delivery_tag)
for tag in tags:
ch.basic_nack(tag, requeue=True)
finally:
conn.close()
out({"dlq_messages": seen, "count": len(seen)})
return 0
def cmd_publish_raw(args):
conn, ch = broker_channel()
try:
ch.confirm_delivery()
ch.basic_publish(common.EXCHANGE, common.ROUTING_KEY, args.body.encode(),
pika.BasicProperties(delivery_mode=2, content_type="application/json"), mandatory=True)
finally:
conn.close()
out({"published": True})
return 0
def main():
telemetry.configure_logging(stream=sys.stderr) # stdout — только JSON-результат команды
p = argparse.ArgumentParser(prog="ops")
sub = p.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("inspect")
s.add_argument("attachment_id")
s.set_defaults(func=cmd_inspect)
s = sub.add_parser("failed")
s.add_argument("--limit", type=int, default=50)
s.set_defaults(func=cmd_failed)
s = sub.add_parser("replay")
s.add_argument("attachment_ids", nargs="*")
s.add_argument("--all-failed", action="store_true")
s.add_argument("--error-code")
s.add_argument("--dry-run", action="store_true")
s.add_argument("--reason", default="operator replay")
s.set_defaults(func=cmd_replay)
s = sub.add_parser("dlq")
s.add_argument("--replay", action="store_true")
s.add_argument("--purge-invalid", action="store_true")
s.set_defaults(func=cmd_dlq)
s = sub.add_parser("publish-raw")
s.add_argument("body")
s.set_defaults(func=cmd_publish_raw)
args = p.parse_args()
return args.func(args)
if __name__ == "__main__":
sys.exit(main())