forked from mtusi2026/mtusi2026lab7
Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS. - Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2. - Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1. - TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert, mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape. - OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором, auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login. - Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и пользователи брокера выдаются из OpenBao. - Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе. - Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL, no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API. - Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao, hardening; evidence, в т.ч. прогон из чистого clone.
221 lines
9.8 KiB
Python
221 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Операторский CLI (compose-сервис `ops`, profile tools). Вывод — JSON без секретов.
|
|
|
|
ops inspect <attachment_id> состояние: attachment, job, события, варианты, outbox, объекты S3
|
|
ops failed [--limit N] реестр failed-заданий (DLQ на стороне БД)
|
|
ops replay <attachment_id>... [--all-failed] [--error-code C] [--dry-run] [--reason TEXT]
|
|
явный replay failed -> queued (ready/queued/processing не трогаются)
|
|
ops dlq [--replay] [--purge-invalid]
|
|
сообщения в RabbitMQ DLQ: показать; --replay — разобрать каждое
|
|
по состоянию jobs (failed -> replay, остальное -> отбросить с audit)
|
|
ops publish-raw <json> учебное: опубликовать произвольное тело (проверка DLQ)
|
|
|
|
Replay всегда проходит через строку jobs под FOR UPDATE: повтор ready-задания отклоняется,
|
|
поэтому «повторить» не означает исполнить дубль уже успешной работы.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
import pika
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
|
|
import common
|
|
import telemetry
|
|
import jobs
|
|
import objstore
|
|
|
|
EXTRA_ATTEMPTS = common.env_int("JOB_MAX_ATTEMPTS", 5)
|
|
|
|
|
|
def db():
|
|
return psycopg.connect(common.database_conninfo(), row_factory=dict_row)
|
|
|
|
|
|
def jsonable(value):
|
|
if isinstance(value, dict):
|
|
return {k: jsonable(v) for k, v in value.items()}
|
|
if isinstance(value, (list, tuple)):
|
|
return [jsonable(v) for v in value]
|
|
if isinstance(value, (str, int, float, bool)) or value is None:
|
|
return value
|
|
if hasattr(value, "isoformat"):
|
|
return value.isoformat().replace("+00:00", "Z")
|
|
return str(value)
|
|
|
|
|
|
def out(data):
|
|
print(json.dumps(jsonable(data), ensure_ascii=False, indent=1))
|
|
|
|
|
|
def s3_objects(prefix):
|
|
"""Только имена/размеры (ключ janitor: List, без чтения содержимого)."""
|
|
store = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments"))
|
|
try:
|
|
return sorted((o.object_name, o.size) for o in store.client.list_objects(store.bucket, prefix=prefix, recursive=True))
|
|
except Exception as exc:
|
|
return [("error", type(exc).__name__)]
|
|
|
|
|
|
def cmd_inspect(args):
|
|
with db() as conn:
|
|
att = conn.execute(
|
|
"""SELECT id, chat_id, status, error_code, content_type, size_bytes, sha256, width, height,
|
|
object_key, staging_key, pipeline_version, created_at, updated_at
|
|
FROM attachments WHERE id = %s""", (args.attachment_id,)).fetchone()
|
|
if att is None:
|
|
out({"error": "not_found"})
|
|
return 1
|
|
job = conn.execute(
|
|
"""SELECT id, pipeline_version, status, attempt, max_attempts, next_attempt_at, lease_owner,
|
|
lease_until, error_code, last_error, replay_count, created_at, updated_at, finished_at
|
|
FROM jobs WHERE attachment_id = %s ORDER BY pipeline_version""", (args.attachment_id,)).fetchall()
|
|
job_ids = [j["id"] for j in job]
|
|
events = conn.execute(
|
|
"SELECT job_id, at, event, attempt, actor, detail FROM job_events WHERE job_id = ANY(%s) ORDER BY id",
|
|
(job_ids,)).fetchall()
|
|
outbox = conn.execute(
|
|
"""SELECT id, payload->>'attempt' AS attempt, available_at, created_at, published_at, publish_count,
|
|
payload->>'traceparent' AS traceparent
|
|
FROM outbox WHERE job_id = ANY(%s) ORDER BY id""", (job_ids,)).fetchall()
|
|
variants = conn.execute(
|
|
"""SELECT pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256
|
|
FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name""",
|
|
(args.attachment_id,)).fetchall()
|
|
objects = s3_objects("attachments/%s/" % args.attachment_id)
|
|
staging = s3_objects(att["staging_key"]) if att["staging_key"] else []
|
|
out({"attachment": att, "jobs": job, "events": events, "outbox": outbox, "variants": variants,
|
|
"s3_final_objects": objects, "s3_staging_objects": staging})
|
|
return 0
|
|
|
|
|
|
def cmd_failed(args):
|
|
with db() as conn:
|
|
rows = conn.execute(
|
|
"""SELECT j.attachment_id, j.id AS job_id, j.error_code, j.attempt, j.max_attempts, j.replay_count,
|
|
j.last_error, j.updated_at, a.staging_key IS NOT NULL AS replayable
|
|
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
|
WHERE j.status = 'failed' ORDER BY j.updated_at DESC LIMIT %s""", (args.limit,)).fetchall()
|
|
out({"failed": rows, "count": len(rows)})
|
|
return 0
|
|
|
|
|
|
def cmd_replay(args):
|
|
results = []
|
|
with db() as conn:
|
|
ids = list(args.attachment_ids)
|
|
if args.all_failed:
|
|
q = "SELECT attachment_id FROM jobs WHERE status = 'failed'"
|
|
params = []
|
|
if args.error_code:
|
|
q += " AND error_code = %s"
|
|
params.append(args.error_code)
|
|
ids += [str(r["attachment_id"]) for r in conn.execute(q, params)]
|
|
for att_id in ids:
|
|
with conn.transaction():
|
|
result, job = jobs.replay(conn, att_id, "ops:" + common.HOSTNAME, EXTRA_ATTEMPTS,
|
|
args.reason, dry_run=args.dry_run)
|
|
results.append({"attachment_id": att_id, "result": result,
|
|
"job_id": str(job["job_id"]) if job else None})
|
|
out({"replay": results})
|
|
return 0 if all(r["result"] in ("replayed", "would_replay") for r in results) else 3
|
|
|
|
|
|
def broker_channel():
|
|
conn = pika.BlockingConnection(common.broker_parameters())
|
|
return conn, conn.channel()
|
|
|
|
|
|
def cmd_dlq(args):
|
|
conn, ch = broker_channel()
|
|
seen = []
|
|
tags = []
|
|
try:
|
|
with db() as dbc:
|
|
while True:
|
|
method, props, body = ch.basic_get(common.DLQ, auto_ack=False)
|
|
if method is None:
|
|
break
|
|
deaths = (props.headers or {}).get("x-death") or []
|
|
reason = deaths[0].get("reason") if deaths else None
|
|
item = {"reason": reason.decode() if isinstance(reason, bytes) else reason,
|
|
"delivery_count": (props.headers or {}).get("x-delivery-count"), "action": "kept"}
|
|
try:
|
|
msg = jobs.parse_message(body)
|
|
item["job_id"] = msg["job_id"]
|
|
if args.replay:
|
|
with dbc.transaction():
|
|
job = jobs.load_job(dbc, msg["job_id"], lock=True)
|
|
if job is None:
|
|
item["action"] = "kept_unknown_job"
|
|
elif job["job_status"] == "failed":
|
|
result, _ = jobs.replay(dbc, str(job["attachment_id"]), "ops-dlq", EXTRA_ATTEMPTS,
|
|
"dlq replay")
|
|
item["action"] = result
|
|
else:
|
|
# queued/processing/ready: состояние уже ведёт БД, дубль не нужен.
|
|
jobs.add_event(dbc, job["job_id"], "dlq_message_dropped", msg["attempt"], "ops-dlq",
|
|
{"job_status": job["job_status"], "reason": item["reason"]})
|
|
item["action"] = "dropped_job_" + job["job_status"]
|
|
except jobs.InvalidMessage as exc:
|
|
item["invalid"] = str(exc)
|
|
if args.purge_invalid:
|
|
item["action"] = "purged_invalid"
|
|
seen.append(item)
|
|
if item["action"].startswith("kept"):
|
|
tags.append(method.delivery_tag) # вернём в DLQ после обхода
|
|
else:
|
|
ch.basic_ack(method.delivery_tag)
|
|
for tag in tags:
|
|
ch.basic_nack(tag, requeue=True)
|
|
finally:
|
|
conn.close()
|
|
out({"dlq_messages": seen, "count": len(seen)})
|
|
return 0
|
|
|
|
|
|
def cmd_publish_raw(args):
|
|
conn, ch = broker_channel()
|
|
try:
|
|
ch.confirm_delivery()
|
|
ch.basic_publish(common.EXCHANGE, common.ROUTING_KEY, args.body.encode(),
|
|
pika.BasicProperties(delivery_mode=2, content_type="application/json"), mandatory=True)
|
|
finally:
|
|
conn.close()
|
|
out({"published": True})
|
|
return 0
|
|
|
|
|
|
def main():
|
|
telemetry.configure_logging(stream=sys.stderr) # stdout — только JSON-результат команды
|
|
p = argparse.ArgumentParser(prog="ops")
|
|
sub = p.add_subparsers(dest="cmd", required=True)
|
|
s = sub.add_parser("inspect")
|
|
s.add_argument("attachment_id")
|
|
s.set_defaults(func=cmd_inspect)
|
|
s = sub.add_parser("failed")
|
|
s.add_argument("--limit", type=int, default=50)
|
|
s.set_defaults(func=cmd_failed)
|
|
s = sub.add_parser("replay")
|
|
s.add_argument("attachment_ids", nargs="*")
|
|
s.add_argument("--all-failed", action="store_true")
|
|
s.add_argument("--error-code")
|
|
s.add_argument("--dry-run", action="store_true")
|
|
s.add_argument("--reason", default="operator replay")
|
|
s.set_defaults(func=cmd_replay)
|
|
s = sub.add_parser("dlq")
|
|
s.add_argument("--replay", action="store_true")
|
|
s.add_argument("--purge-invalid", action="store_true")
|
|
s.set_defaults(func=cmd_dlq)
|
|
s = sub.add_parser("publish-raw")
|
|
s.add_argument("body")
|
|
s.set_defaults(func=cmd_publish_raw)
|
|
args = p.parse_args()
|
|
return args.func(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|