From 934dfefee78179b52e8440af5d397e6c7d0270ef Mon Sep 17 00:00:00 2001 From: dizi Date: Sun, 1 Nov 2026 17:06:32 +0300 Subject: [PATCH 1/8] feat: add OpenTelemetry instrumentation --- cmd/lab5experiment/api_scenario.go | 2 +- cmd/lab5experiment/main.go | 18 +- cmd/messenger/main.go | 4 +- go.mod | 29 +- go.sum | 65 ++++- internal/bootstrap/migration.go | 4 +- internal/bootstrap/pipeline.go | 51 ++-- internal/bootstrap/reconciliation.go | 10 +- internal/bootstrap/replay.go | 6 +- internal/bootstrap/server.go | 26 +- internal/bootstrap/telemetry.go | 23 ++ internal/broker/redisstream/broker.go | 20 +- internal/database/database.go | 3 +- internal/database/telemetry.go | 28 ++ .../messenger/attachment/pipeline/metrics.go | 140 +++++----- internal/messenger/attachment/service.go | 20 +- internal/messenger/attachment/service_test.go | 2 +- .../messenger/attachment/worker/worker.go | 83 ++++-- .../attachment/worker/worker_test.go | 6 +- internal/messenger/httpapi/attachments.go | 19 +- internal/messenger/httpapi/auth.go | 2 +- internal/messenger/httpapi/handlers.go | 4 +- .../messenger/httpapi/messenger_interface.go | 4 +- internal/messenger/httpapi/metrics.go | 111 +++----- internal/messenger/httpapi/model.go | 2 +- internal/messenger/httpapi/server.go | 37 ++- internal/messenger/httpapi/server_test.go | 4 +- internal/objectstore/store.go | 53 +++- internal/outbox/dispatcher.go | 56 +++- internal/outbox/dispatcher_test.go | 2 +- internal/postgres/attachment_jobs.go | 8 +- internal/postgres/attachment_lease.go | 2 +- internal/postgres/attachment_recovery.go | 2 +- internal/postgres/attachments.go | 2 +- internal/postgres/errors.go | 2 +- internal/postgres/idempotency.go | 2 +- internal/postgres/outbox.go | 2 +- internal/postgres/store.go | 2 +- internal/postgres/store_integration_test.go | 8 +- internal/reconcile/reconcile.go | 2 +- internal/reconcile/reconcile_test.go | 2 +- internal/session/session.go | 2 +- internal/session/session_integration_test.go | 2 +- internal/telemetry/database.go | 39 +++ internal/telemetry/logging.go | 74 +++++ internal/telemetry/propagation.go | 70 +++++ internal/telemetry/propagation_test.go | 33 +++ internal/telemetry/runtime.go | 252 ++++++++++++++++++ internal/telemetry/runtime_test.go | 59 ++++ 49 files changed, 1113 insertions(+), 286 deletions(-) create mode 100644 internal/bootstrap/telemetry.go create mode 100644 internal/database/telemetry.go create mode 100644 internal/telemetry/database.go create mode 100644 internal/telemetry/logging.go create mode 100644 internal/telemetry/propagation.go create mode 100644 internal/telemetry/propagation_test.go create mode 100644 internal/telemetry/runtime.go create mode 100644 internal/telemetry/runtime_test.go diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go index 5dc3072..048a121 100644 --- a/cmd/lab5experiment/api_scenario.go +++ b/cmd/lab5experiment/api_scenario.go @@ -17,7 +17,7 @@ import ( "github.com/google/uuid" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type apiState struct { diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go index 3749235..35c5241 100644 --- a/cmd/lab5experiment/main.go +++ b/cmd/lab5experiment/main.go @@ -21,15 +21,15 @@ import ( "github.com/google/uuid" "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" ) const ( diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 80230b8..c2c6c96 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -5,8 +5,8 @@ import ( "log/slog" "os" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/bootstrap" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/logger" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger" ) func main() { diff --git a/go.mod b/go.mod index 563b37a..37c0a6e 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gitea.privcloud.tech/dizi/mtusi2026lab5 +module gitea.privcloud.tech/dizi/mtusi2026lab6 go 1.26.0 @@ -12,19 +12,36 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/valkey-io/valkey-go v1.0.78 + go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 + go.opentelemetry.io/otel/exporters/prometheus v0.60.0 + go.opentelemetry.io/otel/metric v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 golang.org/x/crypto v0.57.0 golang.org/x/sync v0.23.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect @@ -39,7 +56,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/otlptranslator v0.0.2 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect @@ -48,11 +66,18 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tinylib/msgp v1.6.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/log v0.14.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index edfd0ad..63eb5b2 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -21,12 +23,23 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -70,12 +83,14 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= +github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= @@ -101,6 +116,40 @@ github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8= github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= @@ -117,6 +166,14 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/bootstrap/migration.go b/internal/bootstrap/migration.go index 4cf7712..0e7dd49 100644 --- a/internal/bootstrap/migration.go +++ b/internal/bootstrap/migration.go @@ -3,8 +3,8 @@ package bootstrap import ( "context" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" ) func RunMigrations() error { diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go index 97ce3ec..1539107 100644 --- a/internal/bootstrap/pipeline.go +++ b/internal/bootstrap/pipeline.go @@ -14,15 +14,16 @@ import ( "github.com/google/uuid" "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) func RunDispatcher(logger *slog.Logger) error { @@ -30,11 +31,23 @@ func RunDispatcher(logger *slog.Logger) error { if err != nil { return err } + instanceID, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-dispatcher", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings)) if err != nil { return err } defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil { + return err + } client, err := newRedisClient(settings.BrokerURL) if err != nil { return err @@ -51,7 +64,7 @@ func RunDispatcher(logger *slog.Logger) error { if err := broker.Ensure(context.Background()); err != nil { return err } - metrics := pipeline.NewMetrics() + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0) dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, @@ -70,11 +83,23 @@ func RunWorker(logger *slog.Logger) error { if err != nil { return err } + hostname, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-worker", hostname) + if err != nil { + return err + } + defer shutdownTelemetry(observability) pool, err := database.Open(context.Background(), workerDatabaseConfig(settings)) if err != nil { return err } defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil { + return err + } objects, err := objectstore.New(objectstore.Config{ Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, @@ -88,10 +113,6 @@ func RunWorker(logger *slog.Logger) error { return err } defer func() { _ = client.Close() }() - hostname, err := os.Hostname() - if err != nil { - return err - } consumer := hostname + "-" + uuid.NewString() broker, err := redisstream.New(client, redisstream.Config{ Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, @@ -108,7 +129,7 @@ func RunWorker(logger *slog.Logger) error { if err != nil { return err } - metrics := pipeline.NewMetrics() + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency) processorWorker, err := worker.New( logger, postgres.New(pool), broker, objects, processor, metrics, worker.Config{ diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go index 447dd23..3f09a3f 100644 --- a/internal/bootstrap/reconciliation.go +++ b/internal/bootstrap/reconciliation.go @@ -10,11 +10,11 @@ import ( "syscall" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" ) type reconciliationOutput struct { diff --git a/internal/bootstrap/replay.go b/internal/bootstrap/replay.go index 56b9f2e..70cc3e8 100644 --- a/internal/bootstrap/replay.go +++ b/internal/bootstrap/replay.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" ) func RunAttachmentReplay(arguments []string, output io.Writer) error { diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go index 121e328..473c0b7 100644 --- a/internal/bootstrap/server.go +++ b/internal/bootstrap/server.go @@ -9,14 +9,15 @@ import ( "os/signal" "syscall" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/httpapi" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/password" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) func RunServer(logger *slog.Logger) error { @@ -28,11 +29,19 @@ func RunServer(logger *slog.Logger) error { if err != nil { return err } + observability, logger, err := startTelemetry("messenger-api", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) pool, err := database.Open(context.Background(), databaseConfig(settings)) if err != nil { return err } defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil { + return err + } sessions, err := session.New(settings.CacheURL) if err != nil { return err @@ -62,6 +71,7 @@ func RunServer(logger *slog.Logger) error { app := httpapi.NewServer(logger, httpapi.Dependencies{ Users: repository, Chats: repository, Messages: repository, Attachments: attachments, Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects, + Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(), }, httpapi.Config{ SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL, RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort, diff --git a/internal/bootstrap/telemetry.go b/internal/bootstrap/telemetry.go new file mode 100644 index 0000000..bb1dcd9 --- /dev/null +++ b/internal/bootstrap/telemetry.go @@ -0,0 +1,23 @@ +package bootstrap + +import ( + "context" + "log/slog" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) { + runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID)) + if err != nil { + return nil, nil, err + } + return runtime, runtime.Logger(), nil +} + +func shutdownTelemetry(runtime *telemetry.Runtime) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = runtime.Shutdown(ctx) +} diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go index 35d46f0..0788997 100644 --- a/internal/broker/redisstream/broker.go +++ b/internal/broker/redisstream/broker.go @@ -11,7 +11,12 @@ import ( "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" ) const ( @@ -103,6 +108,10 @@ func isBusyGroup(err error) bool { } func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() envelope, err := DecodeEnvelope(payload) if err != nil { return "", err @@ -117,12 +126,19 @@ func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, Values: map[string]any{payloadField: string(payload)}, }).Result() if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XADD failed") return "", fmt.Errorf("publish redis stream job: %w", err) } + span.SetAttributes(attribute.String("messaging.message.id", id)) return id, nil } func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: b.config.Group, Consumer: b.config.Consumer, @@ -134,6 +150,8 @@ func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { return nil, nil } if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XREADGROUP failed") return nil, fmt.Errorf("read redis stream jobs: %w", err) } return b.deliveries(streams) diff --git a/internal/database/database.go b/internal/database/database.go index ff1f8bd..c631b06 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab5/migrations" + "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" ) var ( @@ -40,6 +40,7 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) { if err != nil { return nil, fmt.Errorf("parse database configuration: %w", err) } + poolConfig.ConnConfig.Tracer = queryTracer{} poolConfig.MinConns = config.MinConnections poolConfig.MaxConns = config.MaxConnections poolConfig.MaxConnLifetime = config.MaxConnectionAge diff --git a/internal/database/telemetry.go b/internal/database/telemetry.go new file mode 100644 index 0000000..e2c34f8 --- /dev/null +++ b/internal/database/telemetry.go @@ -0,0 +1,28 @@ +package database + +import ( + "context" + + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +type queryTracer struct{} + +func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "postgresql"))) + return ctx +} + +func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + span := trace.SpanFromContext(ctx) + if data.Err != nil { + span.RecordError(data.Err) + span.SetStatus(codes.Error, "query failed") + } + span.End() +} diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go index 8e8d432..07a91a4 100644 --- a/internal/messenger/attachment/pipeline/metrics.go +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -1,104 +1,106 @@ package pipeline import ( + "context" "net/http" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} type Metrics struct { - workerInFlight prometheus.Gauge - workerJobs *prometheus.CounterVec - workerDuration *prometheus.HistogramVec - workerRetries prometheus.Counter - workerReclaimed prometheus.Counter - outboxPublished prometheus.Counter - outboxPublishFailure prometheus.Counter - outboxRecovered prometheus.Counter + meter metric.Meter + workerInFlight metric.Int64UpDownCounter + workerJobs metric.Int64Counter + workerDuration metric.Float64Histogram + workerTimeToReady metric.Float64Histogram + workerRetries metric.Int64Counter + workerReclaimed metric.Int64Counter + outboxPublished metric.Int64Counter + outboxPublishFailure metric.Int64Counter + outboxRecovered metric.Int64Counter handler http.Handler } -func NewMetrics() *Metrics { - registry := prometheus.NewRegistry() - metrics := &Metrics{ - workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight", - Help: "Attachment jobs currently processed by this worker.", - }), - workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total", - Help: "Completed attachment job deliveries by result.", - }, []string{"result"}), - workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds", - Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets, - }, []string{"result"}), - workerRetries: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total", - Help: "Attachment jobs scheduled for another attempt.", - }), - workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total", - Help: "Redis Stream deliveries reclaimed after their idle timeout.", - }), - outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "outbox", Name: "published_total", - Help: "Outbox events confirmed by Redis Streams.", - }), - outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total", - Help: "Outbox events whose Redis Streams publish failed.", - }), - outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total", - Help: "Expired outbox leases returned for publication.", - }), +func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics { + if meter == nil { + meter = otel.Meter("messenger-pipeline") } - registry.MustRegister( - metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration, - metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished, - metrics.outboxPublishFailure, metrics.outboxRecovered, - collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + if handler == nil { + handler = http.NotFoundHandler() + } + workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker.")) + workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result.")) + workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration", + metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(workerDurationBuckets...), ) - metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) - return metrics + workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready", + metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(workerDurationBuckets...), + ) + workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt.")) + workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout.")) + outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams.")) + outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed.")) + outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication.")) + if workerCapacity > 0 { + _, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity", + metric.WithDescription("Maximum concurrent attachment jobs for this worker."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(int64(workerCapacity)) + return nil + }), + ) + } + return &Metrics{ + meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs, + workerDuration: workerDuration, workerTimeToReady: workerTimeToReady, + workerRetries: workerRetries, workerReclaimed: workerReclaimed, + outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure, + outboxRecovered: outboxRecovered, handler: handler, + } } func (m *Metrics) Handler() http.Handler { return m.handler } -func (m *Metrics) ObserveStarted() { - m.workerInFlight.Inc() +func (m *Metrics) ObserveStarted(ctx context.Context) { + m.workerInFlight.Add(ctx, 1) } -func (m *Metrics) ObserveFinished(result string, duration time.Duration) { - m.workerInFlight.Dec() - m.workerJobs.WithLabelValues(result).Inc() - m.workerDuration.WithLabelValues(result).Observe(duration.Seconds()) +func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) { + attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result)) + m.workerInFlight.Add(ctx, -1) + m.workerJobs.Add(ctx, 1, attributes) + m.workerDuration.Record(ctx, duration.Seconds(), attributes) } -func (m *Metrics) ObserveRetry() { - m.workerRetries.Inc() +func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) { + m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process"))) } -func (m *Metrics) ObserveReclaimed(count int) { - m.workerReclaimed.Add(float64(count)) +func (m *Metrics) ObserveRetry(ctx context.Context) { + m.workerRetries.Add(ctx, 1) } -func (m *Metrics) ObservePublished() { - m.outboxPublished.Inc() +func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) { + m.workerReclaimed.Add(ctx, int64(count)) } -func (m *Metrics) ObservePublishFailure() { - m.outboxPublishFailure.Inc() +func (m *Metrics) ObservePublished(ctx context.Context) { + m.outboxPublished.Add(ctx, 1) } -func (m *Metrics) ObserveRecovered(count int) { - m.outboxRecovered.Add(float64(count)) +func (m *Metrics) ObservePublishFailure(ctx context.Context) { + m.outboxPublishFailure.Add(ctx, 1) +} + +func (m *Metrics) ObserveRecovered(ctx context.Context, count int) { + m.outboxRecovered.Add(ctx, int64(count)) } diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go index 7e0bbf8..38e5bdc 100644 --- a/internal/messenger/attachment/service.go +++ b/internal/messenger/attachment/service.go @@ -19,7 +19,8 @@ import ( "github.com/google/uuid" "golang.org/x/sync/semaphore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) type Repository interface { @@ -53,8 +54,7 @@ type Upload struct { ContentType string ContentLength int64 Body io.Reader - Traceparent string - Tracestate string + TraceContext telemetry.MessageCarrier } type Content struct { @@ -112,27 +112,27 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e return model.Attachment{}, mapDependencyError(err) } if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil { - s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err) - s.deletePending(pending.ID, stagingKey) + s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err) + s.deletePending(ctx, pending.ID, stagingKey) return model.Attachment{}, model.ErrUnavailable } createdAt := time.Now().UTC() queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{ ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion, Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt, - }, input.Traceparent, input.Tracestate) + }, input.TraceContext.Traceparent, input.TraceContext.Tracestate) if err != nil { - s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err) + s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err) return model.Attachment{}, mapDependencyError(err) } return queued, nil } -func (s *Service) deletePending(attachmentID, stagingKey string) { - cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) +func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout) defer cancel() if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil { - s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) + s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) } } diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go index da14eb0..296f283 100644 --- a/internal/messenger/attachment/service_test.go +++ b/internal/messenger/attachment/service_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func TestUploadLifecycle(t *testing.T) { diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go index f94f525..45ff600 100644 --- a/internal/messenger/attachment/worker/worker.go +++ b/internal/messenger/attachment/worker/worker.go @@ -18,10 +18,16 @@ import ( "golang.org/x/sync/semaphore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) type Repository interface { @@ -29,7 +35,7 @@ type Repository interface { ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) - RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error) + RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error) FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error) } @@ -50,10 +56,11 @@ type ImageProcessor interface { } type Metrics interface { - ObserveStarted() - ObserveFinished(string, time.Duration) - ObserveRetry() - ObserveReclaimed(int) + ObserveStarted(context.Context) + ObserveFinished(context.Context, string, time.Duration) + ObserveTimeToReady(context.Context, time.Duration) + ObserveRetry(context.Context) + ObserveReclaimed(context.Context, int) } type Config struct { @@ -166,7 +173,7 @@ func (w *Worker) Run(ctx context.Context) error { } return err } - w.metrics.ObserveReclaimed(len(deliveries)) + w.metrics.ObserveReclaimed(ctx, len(deliveries)) w.processBatch(ctx, deliveries) reclaimCursor = next if reclaimCursor == "" { @@ -189,25 +196,48 @@ func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Deli go func(delivery redisstream.Delivery) { defer group.Done() defer func() { <-limit }() - if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil { - w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err) - } + _ = w.ProcessDelivery(ctx, delivery, FaultNone) }(delivery) } group.Wait() } -func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error { +func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) { started := w.now() - w.metrics.ObserveStarted() result := "transient_error" - defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }() - if delivery.DecodeError != nil { + w.metrics.ObserveStarted(ctx) + defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }() result = "invalid_job" return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) } envelope := delivery.Envelope + ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{ + Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate, + }) + ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer), + trace.WithAttributes( + attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()), + attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt), + attribute.Int("pipeline.version", envelope.PipelineVersion), + )) + w.metrics.ObserveStarted(ctx) + defer func() { + w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) + span.SetAttributes(attribute.String("job.outcome", result)) + if resultErr != nil { + span.RecordError(resultErr) + span.SetStatus(codes.Error, result) + if ctx.Err() == nil { + w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr) + } + } else { + w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result) + } + span.End() + }() + lease, err := w.repository.ClaimAttachmentJob( ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt, w.config.WorkerID, w.config.LeaseDuration, @@ -246,7 +276,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv } if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") { if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil { - w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) + w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) } } if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale { @@ -260,6 +290,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return err } result = "ready" + w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt)) return nil } @@ -444,7 +475,10 @@ func (w *Worker) resolveProcessingFailure( return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) } nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt)) - mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt) + traceContext := telemetry.ContextCarrier(ctx) + mutation, err := w.repository.RetryAttachmentJob( + ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate, + ) if err != nil { return "transient_error", fmt.Errorf("retry attachment job: %w", err) } @@ -454,7 +488,7 @@ func (w *Worker) resolveProcessingFailure( if mutation != model.JobMutationApplied { return "transient_error", fmt.Errorf("unexpected retry result %q", mutation) } - w.metrics.ObserveRetry() + w.metrics.ObserveRetry(ctx) return "retry", delivery.Ack(ctx) } @@ -502,7 +536,8 @@ func (w *Worker) retryDelay(jobID string, attempt int) time.Duration { type nopMetrics struct{} -func (nopMetrics) ObserveStarted() {} -func (nopMetrics) ObserveFinished(string, time.Duration) {} -func (nopMetrics) ObserveRetry() {} -func (nopMetrics) ObserveReclaimed(int) {} +func (nopMetrics) ObserveStarted(context.Context) {} +func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {} +func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {} +func (nopMetrics) ObserveRetry(context.Context) {} +func (nopMetrics) ObserveReclaimed(context.Context, int) {} diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go index 7a573f1..c2ca384 100644 --- a/internal/messenger/attachment/worker/worker_test.go +++ b/internal/messenger/attachment/worker/worker_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" ) func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) { diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go index f9250c8..2e832d6 100644 --- a/internal/messenger/httpapi/attachments.go +++ b/internal/messenger/httpapi/attachments.go @@ -13,16 +13,21 @@ import ( "unicode" "unicode/utf8" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { started := time.Now() result := "client_error" defer func() { - s.metrics.attachmentUploads.WithLabelValues(result).Inc() - s.metrics.attachmentDuration.Observe(time.Since(started).Seconds()) + attributes := metric.WithAttributes(attribute.String("result", result)) + s.metrics.attachmentUploads.Add(r.Context(), 1, attributes) + s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes) }() if r.Method != http.MethodPost { @@ -47,7 +52,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ ChatID: chatID, OwnerID: user.ID, Filename: filename, ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, - Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")), + TraceContext: telemetry.ContextCarrier(r.Context()), }) if err != nil { if errors.Is(err, model.ErrCapacity) { @@ -59,7 +64,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr return attachmentError(err) } result = "success" - s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes)) + s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes) writeJSON(w, http.StatusAccepted, uploaded) return nil } @@ -128,7 +133,7 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, content.Body); err != nil { - s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err) + s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err) } return nil } diff --git a/internal/messenger/httpapi/auth.go b/internal/messenger/httpapi/auth.go index 5baa066..29419a1 100644 --- a/internal/messenger/httpapi/auth.go +++ b/internal/messenger/httpapi/auth.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go index 1b41574..9084d13 100644 --- a/internal/messenger/httpapi/handlers.go +++ b/internal/messenger/httpapi/handlers.go @@ -9,7 +9,7 @@ import ( "time" "unicode/utf8" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) @@ -229,7 +229,7 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { return unavailable() } if replay { - s.metrics.replays.Inc() + s.metrics.replays.Add(r.Context(), 1) } writeJSON(w, http.StatusCreated, message) return nil diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go index 055b80b..9844c25 100644 --- a/internal/messenger/httpapi/messenger_interface.go +++ b/internal/messenger/httpapi/messenger_interface.go @@ -4,8 +4,8 @@ import ( "context" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type UserRepository interface { diff --git a/internal/messenger/httpapi/metrics.go b/internal/messenger/httpapi/metrics.go index f1b058e..5047ebd 100644 --- a/internal/messenger/httpapi/metrics.go +++ b/internal/messenger/httpapi/metrics.go @@ -1,14 +1,15 @@ package httpapi import ( + "context" "net/http" "strconv" "strings" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) var ( @@ -17,80 +18,50 @@ var ( ) type metrics struct { - requests *prometheus.CounterVec - duration *prometheus.HistogramVec - inFlight prometheus.Gauge - replays prometheus.Counter - attachmentUploads *prometheus.CounterVec - attachmentDuration prometheus.Histogram - attachmentBytes prometheus.Counter + requests metric.Int64Counter + duration metric.Float64Histogram + inFlight metric.Int64UpDownCounter + replays metric.Int64Counter + attachmentUploads metric.Int64Counter + attachmentDuration metric.Float64Histogram + attachmentBytes metric.Int64Counter handler http.Handler } -func newMetrics() *metrics { - registry := prometheus.NewRegistry() - result := &metrics{ - requests: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "requests_total", - Help: "Completed HTTP requests.", - }, []string{"method", "route", "status_class"}), - duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "request_duration_seconds", - Help: "HTTP request duration in seconds.", - Buckets: requestDurationBuckets, - }, []string{"method", "route"}), - inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "requests_in_flight", - Help: "HTTP requests currently being served.", - }), - replays: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "idempotency_replays_total", - Help: "Message requests served from an idempotency record.", - }), - attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "messenger", - Name: "attachment_uploads_total", - Help: "Completed attachment uploads by result.", - }, []string{"result"}), - attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: "messenger", - Name: "attachment_upload_duration_seconds", - Help: "Attachment upload duration in seconds.", - Buckets: uploadDurationBuckets, - }), - attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", - Name: "attachment_upload_bytes_total", - Help: "Bytes accepted by successful attachment uploads.", - }), +func newMetrics(meter metric.Meter, handler http.Handler) *metrics { + if meter == nil { + meter = otel.Meter("messenger-api") } - registry.MustRegister( - result.requests, - result.duration, - result.inFlight, - result.replays, - result.attachmentUploads, - result.attachmentDuration, - result.attachmentBytes, - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + if handler == nil { + handler = http.NotFoundHandler() + } + requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests.")) + duration, _ := meter.Float64Histogram("messenger_http_request_duration", + metric.WithDescription("HTTP request duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(requestDurationBuckets...), ) - result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) - return result + inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served.")) + replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record.")) + attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result.")) + attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration", + metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(uploadDurationBuckets...), + ) + attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By")) + return &metrics{ + requests: requests, duration: duration, inFlight: inFlight, replays: replays, + attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration, + attachmentBytes: attachmentBytes, handler: handler, + } } -func (m *metrics) observe(r *http.Request, status int, started time.Time) { - route := routeTemplate(r.URL.Path) - m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc() - m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds()) +func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) { + attributes := metric.WithAttributes( + attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)), + attribute.String("status_class", strconv.Itoa(status/100)+"xx"), + ) + m.requests.Add(ctx, 1, attributes) + m.duration.Record(ctx, time.Since(started).Seconds(), attributes) } func routeTemplate(path string) string { diff --git a/internal/messenger/httpapi/model.go b/internal/messenger/httpapi/model.go index 9ecc48d..7f3f6dc 100644 --- a/internal/messenger/httpapi/model.go +++ b/internal/messenger/httpapi/model.go @@ -1,7 +1,7 @@ package httpapi import ( - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" "github.com/google/uuid" ) diff --git a/internal/messenger/httpapi/server.go b/internal/messenger/httpapi/server.go index 9af19cf..fcc18bf 100644 --- a/internal/messenger/httpapi/server.go +++ b/internal/messenger/httpapi/server.go @@ -6,6 +6,16 @@ import ( "net/http" "sync/atomic" "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) type Dependencies struct { @@ -17,6 +27,8 @@ type Dependencies struct { Passwords PasswordService Database HealthChecker Objects HealthChecker + Meter metric.Meter + Metrics http.Handler } type Config struct { @@ -67,7 +79,8 @@ func NewServer( attachments: dependencies.Attachments, sessions: dependencies.Sessions, passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects, sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL, - instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(), + instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, + metrics: newMetrics(dependencies.Meter, dependencies.Metrics), } s.ready.Store(true) return s @@ -82,27 +95,39 @@ func (s *Server) Handler() http.Handler { } func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header)) + tracer := otel.Tracer("messenger-api/http") + route := routeTemplate(r.URL.Path) + requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route))) + defer span.End() + r = r.WithContext(requestContext) w.Header().Set("X-Instance-Id", s.instanceID) requestID, err := newID() if err != nil { http.Error(w, "server unavailable", http.StatusServiceUnavailable) return } + span.SetAttributes(attribute.String("request.id", requestID)) w.Header().Set("X-Request-Id", requestID) w.Header().Set("Cache-Control", "no-store") wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK} started := time.Now() - s.metrics.inFlight.Inc() + s.metrics.inFlight.Add(r.Context(), 1) defer func() { - s.metrics.inFlight.Dec() + s.metrics.inFlight.Add(r.Context(), -1) if recovered := recover(); recovered != nil { - s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path) + s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route) if wrapped.status == http.StatusOK { writeError(wrapped, requestID, internalError()) } } - s.metrics.observe(r, wrapped.status, started) - s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) + s.metrics.observe(r.Context(), r, wrapped.status, started) + span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status)) + if wrapped.status >= http.StatusInternalServerError { + span.SetStatus(codes.Error, http.StatusText(wrapped.status)) + } + s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) }() if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" { wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index d2e0a21..c1bf91d 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) var errDependency = errors.New("dependency unavailable") diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go index ac41f65..e2d6ded 100644 --- a/internal/objectstore/store.go +++ b/internal/objectstore/store.go @@ -13,7 +13,12 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) var ErrNotFound = errors.New("object not found") @@ -85,30 +90,38 @@ func (s *Store) Ping(ctx context.Context) error { } func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error { + ctx, span := s.startSpan(ctx, "PUT") + defer span.End() ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() _, err := s.client.PutObject( ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType}, ) if err != nil { + recordSpanError(span, err) return fmt.Errorf("put S3 object: %w", err) } return nil } func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { + ctx, span := s.startSpan(ctx, "GET") ctx, cancel := context.WithTimeout(ctx, s.timeout) object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) if err != nil { cancel() + recordSpanError(span, err) + span.End() return nil, objectError("get S3 object", err) } if _, err := object.Stat(); err != nil { cancel() _ = object.Close() + recordSpanError(span, err) + span.End() return nil, objectError("stat S3 object", err) } - return &timeoutReader{ReadCloser: object, cancel: cancel}, nil + return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil } func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) { @@ -122,9 +135,12 @@ func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error } func (s *Store) Delete(ctx context.Context, key string) error { + ctx, span := s.startSpan(ctx, "DELETE") + defer span.End() ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { + recordSpanError(span, err) return objectError("delete S3 object", err) } return nil @@ -161,6 +177,28 @@ type timeoutReader struct { cancel context.CancelFunc } +type tracedReader struct { + io.ReadCloser + span trace.Span +} + +func (r *tracedReader) Read(buffer []byte) (int, error) { + read, err := r.ReadCloser.Read(buffer) + if err != nil && !errors.Is(err, io.EOF) { + recordSpanError(r.span, err) + } + return read, err +} + +func (r *tracedReader) Close() error { + err := r.ReadCloser.Close() + if err != nil { + recordSpanError(r.span, err) + } + r.span.End() + return err +} + func (r *timeoutReader) Close() error { err := r.ReadCloser.Close() r.cancel() @@ -174,3 +212,14 @@ func objectError(action string, err error) error { } return fmt.Errorf("%s: %w", action, err) } + +func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) { + return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation), + attribute.String("s3.bucket.name", s.bucket))) +} + +func recordSpanError(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, "S3 operation failed") +} diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go index 22aa75d..ea74f61 100644 --- a/internal/outbox/dispatcher.go +++ b/internal/outbox/dispatcher.go @@ -6,7 +6,13 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) var errLeaseLost = errors.New("outbox lease lost") @@ -24,9 +30,9 @@ type Publisher interface { } type Metrics interface { - ObservePublished() - ObservePublishFailure() - ObserveRecovered(int) + ObservePublished(context.Context) + ObservePublishFailure(context.Context) + ObserveRecovered(context.Context, int) } type Config struct { @@ -156,15 +162,32 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { if err := ctx.Err(); err != nil { return published, nil } - _, err := d.publisher.PublishConfirmed(ctx, event.Payload) + publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload) + publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish", + trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt), + )) + payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload) + if injectErr != nil { + span.RecordError(injectErr) + span.SetStatus(codes.Error, "inject trace context") + span.End() + return published, fmt.Errorf("inject outbox trace context: %w", injectErr) + } + _, err := d.publisher.PublishConfirmed(publishContext, payload) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "publish failed") + span.End() if ctx.Err() != nil { return published, nil } - d.metrics.ObservePublishFailure() + d.metrics.ObservePublishFailure(publishContext) nextAttemptAt := d.now().Add(d.config.RetryDelay) released, releaseErr := d.store.ReleaseOutboxEvent( - ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, + publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, ) if releaseErr != nil { return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr) @@ -174,14 +197,21 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { } continue } - marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken) + marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "mark published") + span.End() return published, fmt.Errorf("mark outbox event published: %w", err) } if !marked { + span.SetStatus(codes.Error, "outbox lease lost") + span.End() return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost) } - d.metrics.ObservePublished() + span.SetAttributes(attribute.String("messaging.operation.type", "publish")) + span.End() + d.metrics.ObservePublished(publishContext) published++ } return published, nil @@ -192,12 +222,12 @@ func (d *Dispatcher) RecoverExpired(ctx context.Context) error { if err != nil { return fmt.Errorf("recover expired outbox leases: %w", err) } - d.metrics.ObserveRecovered(recovered) + d.metrics.ObserveRecovered(ctx, recovered) return nil } type nopMetrics struct{} -func (nopMetrics) ObservePublished() {} -func (nopMetrics) ObservePublishFailure() {} -func (nopMetrics) ObserveRecovered(int) {} +func (nopMetrics) ObservePublished(context.Context) {} +func (nopMetrics) ObservePublishFailure(context.Context) {} +func (nopMetrics) ObserveRecovered(context.Context, int) {} diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go index 206790d..b9d792c 100644 --- a/internal/outbox/dispatcher_test.go +++ b/internal/outbox/dispatcher_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" "github.com/stretchr/testify/require" ) diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go index 3bcfb39..6c959de 100644 --- a/internal/postgres/attachment_jobs.go +++ b/internal/postgres/attachment_jobs.go @@ -10,7 +10,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const attachmentJobTopic = "attachment.jobs" @@ -447,6 +447,8 @@ func (s *Store) RetryAttachmentJob( lease model.AttachmentJobLease, errorCode string, nextAttemptAt time.Time, + traceparent string, + tracestate string, ) (model.JobMutationResult, error) { if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { return "", model.ErrInvalidInput @@ -483,9 +485,7 @@ func (s *Store) RetryAttachmentJob( if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil { return "", fmt.Errorf("requeue attachment: %w", err) } - eventJob := job - eventJob.CreatedAt = time.Now().UTC() - if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { return "", err } if err := tx.Commit(ctx); err != nil { diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go index b6883eb..9e631a9 100644 --- a/internal/postgres/attachment_lease.go +++ b/internal/postgres/attachment_lease.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Store) RenewAttachmentJobLease( diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go index d7d84ca..e305a79 100644 --- a/internal/postgres/attachment_recovery.go +++ b/internal/postgres/attachment_recovery.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Store) RecoverExpiredAttachmentJobs( diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index e7c2531..2765eda 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const attachmentColumns = ` diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go index e4daccb..39509ad 100644 --- a/internal/postgres/errors.go +++ b/internal/postgres/errors.go @@ -1,6 +1,6 @@ package postgres -import "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" var ( ErrConflict = model.ErrConflict diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go index a8168f5..4bbb95b 100644 --- a/internal/postgres/idempotency.go +++ b/internal/postgres/idempotency.go @@ -12,7 +12,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const expiredIdempotencyCleanupBatch = 100 diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go index 9eda865..0817f7c 100644 --- a/internal/postgres/outbox.go +++ b/internal/postgres/outbox.go @@ -8,7 +8,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Store) ClaimOutboxEvents( diff --git a/internal/postgres/store.go b/internal/postgres/store.go index 5413f28..44980b7 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -10,7 +10,7 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type Store struct { diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 4867523..e4184db 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -15,10 +15,10 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" ) type PostgresSuite struct { diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 9c4ee8c..15607f8 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type MetadataStore interface { diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 8cbef1f..36a2955 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func TestReconcilerLifecycleAndRepeat(t *testing.T) { diff --git a/internal/session/session.go b/internal/session/session.go index b10261b..a7df4b9 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -14,7 +14,7 @@ import ( "github.com/valkey-io/valkey-go" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const ( diff --git a/internal/session/session_integration_test.go b/internal/session/session_integration_test.go index c7d874e..a82e5eb 100644 --- a/internal/session/session_integration_test.go +++ b/internal/session/session_integration_test.go @@ -11,7 +11,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" ) type SessionSuite struct { diff --git a/internal/telemetry/database.go b/internal/telemetry/database.go new file mode 100644 index 0000000..6390cd3 --- /dev/null +++ b/internal/telemetry/database.go @@ -0,0 +1,39 @@ +package telemetry + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + connectionStateUsed = attribute.String("state", "used") + connectionStateIdle = attribute.String("state", "idle") + connectionStateConnecting = attribute.String("state", "connecting") +) + +func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error { + _, err := meter.Int64ObservableGauge("messenger_db_pool_connections", + metric.WithDescription("PostgreSQL pool connections by state."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + stats := pool.Stat() + observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed)) + observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle)) + observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting)) + return nil + }), + ) + if err != nil { + return err + } + _, err = meter.Int64ObservableCounter("messenger_db_pool_waited", + metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(pool.Stat().EmptyAcquireCount()) + return nil + }), + ) + return err +} diff --git a/internal/telemetry/logging.go b/internal/telemetry/logging.go new file mode 100644 index 0000000..880d349 --- /dev/null +++ b/internal/telemetry/logging.go @@ -0,0 +1,74 @@ +package telemetry + +import ( + "context" + "log/slog" + "strings" + + "go.opentelemetry.io/otel/trace" +) + +type correlationHandler struct { + stdout slog.Handler + otel slog.Handler +} + +func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler { + return correlationHandler{stdout: stdout, otel: otel} +} + +func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level) +} + +func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error { + span := trace.SpanContextFromContext(ctx) + if span.IsValid() { + record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String())) + } + record = sanitizedRecord(record) + if err := h.stdout.Handle(ctx, record); err != nil { + return err + } + return h.otel.Handle(ctx, record) +} + +func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + attributes = sanitizedAttributes(attributes) + return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)} +} + +func (h correlationHandler) WithGroup(name string) slog.Handler { + return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)} +} + +func sanitizedRecord(record slog.Record) slog.Record { + copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC) + record.Attrs(func(attribute slog.Attr) bool { + if !sensitiveKey(attribute.Key) { + copy.AddAttrs(attribute) + } + return true + }) + return copy +} + +func sanitizedAttributes(attributes []slog.Attr) []slog.Attr { + result := make([]slog.Attr, 0, len(attributes)) + for _, attribute := range attributes { + if !sensitiveKey(attribute.Key) { + result = append(result, attribute) + } + } + return result +} + +func sensitiveKey(key string) bool { + key = strings.ToLower(key) + for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} { + if strings.Contains(key, marker) { + return true + } + } + return false +} diff --git a/internal/telemetry/propagation.go b/internal/telemetry/propagation.go new file mode 100644 index 0000000..92c3dac --- /dev/null +++ b/internal/telemetry/propagation.go @@ -0,0 +1,70 @@ +package telemetry + +import ( + "context" + "encoding/json" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +type MessageCarrier struct { + Traceparent string + Tracestate string +} + +func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, carrier) +} + +func Inject(ctx context.Context, carrier propagation.TextMapCarrier) { + otel.GetTextMapPropagator().Inject(ctx, carrier) +} + +func ContextCarrier(ctx context.Context) MessageCarrier { + carrier := propagation.MapCarrier{} + Inject(ctx, carrier) + return MessageCarrier{ + Traceparent: carrier.Get("traceparent"), + Tracestate: carrier.Get("tracestate"), + } +} + +func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context { + headers := propagation.MapCarrier{ + "traceparent": carrier.Traceparent, + "tracestate": carrier.Tracestate, + } + return Extract(ctx, headers) +} + +func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context { + var carrier struct { + Traceparent string `json:"traceparent"` + Tracestate string `json:"tracestate"` + } + if json.Unmarshal(payload, &carrier) != nil { + return ctx + } + return ExtractMessage(ctx, MessageCarrier(carrier)) +} + +func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) { + var value map[string]json.RawMessage + if err := json.Unmarshal(payload, &value); err != nil { + return nil, err + } + carrier := ContextCarrier(ctx) + setJSONField(value, "traceparent", carrier.Traceparent) + setJSONField(value, "tracestate", carrier.Tracestate) + return json.Marshal(value) +} + +func setJSONField(value map[string]json.RawMessage, key, field string) { + if field == "" { + delete(value, key) + return + } + encoded, _ := json.Marshal(field) + value[key] = encoded +} diff --git a/internal/telemetry/propagation_test.go b/internal/telemetry/propagation_test.go new file mode 100644 index 0000000..31895a5 --- /dev/null +++ b/internal/telemetry/propagation_test.go @@ -0,0 +1,33 @@ +package telemetry + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) { + otel.SetTextMapPropagator(propagation.TraceContext{}) + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + + payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`)) + require.NoError(t, err) + var value map[string]any + require.NoError(t, json.Unmarshal(payload, &value)) + require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"]) + require.NotContains(t, value, "baggage") + + extracted := ExtractJSONMessage(context.Background(), payload) + require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID()) +} diff --git a/internal/telemetry/runtime.go b/internal/telemetry/runtime.go new file mode 100644 index 0000000..90267e6 --- /dev/null +++ b/internal/telemetry/runtime.go @@ -0,0 +1,252 @@ +package telemetry + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/bridges/otelslog" + otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + otelprometheus "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" +) + +const ( + defaultEnvironment = "lab" + defaultNamespace = "mtusi-highload" +) + +type Config struct { + ServiceName string + Version string + InstanceID string + Endpoint string + Environment string + Namespace string + Insecure bool +} + +type Runtime struct { + traces *trace.TracerProvider + metrics *metric.MeterProvider + logs *log.LoggerProvider + logger *slog.Logger + handler http.Handler +} + +func ConfigFromEnvironment(serviceName, instanceID string) Config { + resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES")) + return Config{ + ServiceName: serviceName, + Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"), + InstanceID: instanceID, + Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")), + Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment), + Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace), + Insecure: true, + } +} + +func parseResourceAttributes(raw string) map[string]string { + result := make(map[string]string) + for _, pair := range strings.Split(raw, ",") { + key, value, found := strings.Cut(pair, "=") + if found && strings.TrimSpace(key) != "" { + result[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + } + return result +} + +func firstValue(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func New(ctx context.Context, config Config) (*Runtime, error) { + if err := config.validate(); err != nil { + return nil, err + } + res, err := newResource(ctx, config) + if err != nil { + return nil, err + } + registry := prometheus.NewRegistry() + prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry)) + if err != nil { + return nil, fmt.Errorf("create Prometheus reader: %w", err) + } + + traceOptions := []trace.TracerProviderOption{trace.WithResource(res)} + metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)} + logOptions := []log.LoggerProviderOption{log.WithResource(res)} + if config.Endpoint != "" { + traceExporter, err := newTraceExporter(ctx, config) + if err != nil { + return nil, err + } + metricExporter, err := newMetricExporter(ctx, config) + if err != nil { + return nil, err + } + logExporter, err := newLogExporter(ctx, config) + if err != nil { + return nil, err + } + traceOptions = append(traceOptions, trace.WithBatcher(traceExporter, + trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512), + trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second), + )) + metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader( + metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second), + ))) + logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter, + log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512), + log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second), + ))) + } + + traces := trace.NewTracerProvider(traceOptions...) + metrics := metric.NewMeterProvider(metricOptions...) + logs := log.NewLoggerProvider(logOptions...) + propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}) + otel.SetTracerProvider(traces) + otel.SetMeterProvider(metrics) + otel.SetTextMapPropagator(propagator) + if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil { + return nil, fmt.Errorf("start Go runtime metrics: %w", err) + } + + stdout := slog.NewJSONHandler(os.Stdout, nil) + otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs)) + logger := slog.New(newCorrelationHandler(stdout, otelHandler)) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + return &Runtime{ + traces: traces, metrics: metrics, logs: logs, logger: logger, + handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), + }, nil +} + +func (c Config) validate() error { + switch { + case strings.TrimSpace(c.ServiceName) == "": + return errors.New("telemetry service name is required") + case strings.TrimSpace(c.InstanceID) == "": + return errors.New("telemetry instance ID is required") + case strings.TrimSpace(c.Environment) == "": + return errors.New("telemetry environment is required") + case strings.TrimSpace(c.Namespace) == "": + return errors.New("telemetry service namespace is required") + default: + return nil + } +} + +func newResource(ctx context.Context, config Config) (*resource.Resource, error) { + res, err := resource.New(ctx, + resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(), + resource.WithAttributes( + semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version), + semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace), + semconv.DeploymentEnvironmentName(config.Environment), + ), + ) + if err != nil { + return nil, fmt.Errorf("create telemetry resource: %w", err) + } + return res, nil +} + +func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) { + var options []otlptracegrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlptracegrpc.WithInsecure()) + } + exporter, err := otlptracegrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP trace exporter: %w", err) + } + return exporter, nil +} + +func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) { + var options []otlpmetricgrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlpmetricgrpc.WithInsecure()) + } + exporter, err := otlpmetricgrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP metric exporter: %w", err) + } + return exporter, nil +} + +func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) { + var options []otlploggrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlploggrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlploggrpc.WithInsecure()) + } + exporter, err := otlploggrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP log exporter: %w", err) + } + return exporter, nil +} + +func (r *Runtime) Logger() *slog.Logger { + return r.logger +} + +func (r *Runtime) MeterProvider() *metric.MeterProvider { + return r.metrics +} + +func (r *Runtime) MetricsHandler() http.Handler { + return r.handler +} + +func (r *Runtime) Shutdown(ctx context.Context) error { + var result error + for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} { + if err := shutdown(ctx); err != nil { + result = errors.Join(result, err) + } + } + return result +} diff --git a/internal/telemetry/runtime_test.go b/internal/telemetry/runtime_test.go new file mode 100644 index 0000000..e8b41b1 --- /dev/null +++ b/internal/telemetry/runtime_test.go @@ -0,0 +1,59 @@ +package telemetry + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" +) + +func TestConfigValidation(t *testing.T) { + valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"} + require.NoError(t, valid.validate()) + + invalid := valid + invalid.ServiceName = "" + require.EqualError(t, invalid.validate(), "telemetry service name is required") +} + +func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) { + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + first := &recordHandler{} + second := &recordHandler{} + handler := newCorrelationHandler(first, second) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0) + record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret")) + + require.NoError(t, handler.Handle(ctx, record)) + for _, captured := range []*recordHandler{first, second} { + require.Equal(t, traceID.String(), captured.attributes["trace_id"]) + require.Equal(t, spanID.String(), captured.attributes["span_id"]) + require.Equal(t, "http_request_completed", captured.attributes["event"]) + require.NotContains(t, captured.attributes, "authorization") + } +} + +type recordHandler struct { + attributes map[string]string +} + +func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h *recordHandler) Handle(_ context.Context, record slog.Record) error { + h.attributes = make(map[string]string) + record.Attrs(func(attribute slog.Attr) bool { + h.attributes[attribute.Key] = attribute.Value.String() + return true + }) + return nil +} +func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *recordHandler) WithGroup(string) slog.Handler { return h } -- 2.54.0 From 4bf383bd519cbd2caaafe45f030854ea2d50cfb4 Mon Sep 17 00:00:00 2001 From: dizi Date: Sun, 1 Nov 2026 17:06:35 +0300 Subject: [PATCH 2/8] ops: add observability stack and SLO alerts --- Dockerfile | 3 +- compose.yaml | 103 ++- infra/loki/loki.yaml | 48 ++ infra/otel/collector.yaml | 73 +++ infra/tempo/tempo.yaml | 48 ++ lab5.env.example => lab6.env.example | 21 +- .../{messenger.json => messenger-lab6.json} | 610 +++++++++++++++++- .../grafana/provisioning/alerting/.gitkeep | 1 + .../datasources/observability.yml | 74 +++ .../provisioning/datasources/prometheus.yml | 10 - .../grafana/provisioning/plugins/.gitkeep | 1 + monitoring/prometheus.yml | 46 +- monitoring/rules/messenger.yml | 178 +++++ monitoring/runbooks/api-availability.md | 72 +++ monitoring/runbooks/attachment-pipeline.md | 85 +++ 15 files changed, 1341 insertions(+), 32 deletions(-) create mode 100644 infra/loki/loki.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/tempo/tempo.yaml rename lab5.env.example => lab6.env.example (80%) rename monitoring/grafana/dashboards/{messenger.json => messenger-lab6.json} (55%) create mode 100644 monitoring/grafana/provisioning/alerting/.gitkeep create mode 100644 monitoring/grafana/provisioning/datasources/observability.yml delete mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/grafana/provisioning/plugins/.gitkeep create mode 100644 monitoring/rules/messenger.yml create mode 100644 monitoring/runbooks/api-availability.md create mode 100644 monitoring/runbooks/attachment-pipeline.md diff --git a/Dockerfile b/Dockerfile index efd6b0d..ab37655 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,9 @@ ENV SWAGGER_JSON=/spec/openapi.json FROM prom/prometheus:v3.13.3 AS prometheus COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml +COPY monitoring/rules /etc/prometheus/rules -FROM grafana/grafana:13.1.6 AS grafana +FROM grafana/grafana:13.2.4 AS grafana COPY monitoring/grafana/provisioning /etc/grafana/provisioning COPY monitoring/grafana/dashboards /etc/grafana/dashboards diff --git a/compose.yaml b/compose.yaml index 291eaec..d21c2eb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume target: /etc/messenger/messenger.yaml read_only: true +x-telemetry-environment: &telemetry-environment + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL} + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION} + services: postgres: image: postgres:17.6-alpine3.22 @@ -118,7 +123,7 @@ services: networks: [data] migrate: - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/messenger", "migrate"] environment: MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml @@ -137,8 +142,10 @@ services: api: build: . - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-api MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml APP_HOST: ${APP_HOST} APP_PORT: ${APP_PORT} @@ -176,9 +183,11 @@ services: networks: [application, data] dispatcher: - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/messenger", "dispatch"] environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-dispatcher MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} BROKER_URL: ${BROKER_URL} @@ -202,9 +211,11 @@ services: networks: [data, monitoring] worker: - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/messenger", "worker"] environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-worker MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} S3_ENDPOINT: ${S3_ENDPOINT} @@ -239,7 +250,7 @@ services: build: context: . target: loadbalancer - image: ${NGINX_IMAGE:-mtusi-nginx:lab5} + image: ${NGINX_IMAGE:-mtusi-nginx:lab6} ports: - "${API_BIND_ADDRESS}:8080" depends_on: @@ -259,7 +270,7 @@ services: build: context: . target: swagger - image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5} + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6} ports: - "${SWAGGER_BIND_ADDRESS}:8080" depends_on: @@ -289,7 +300,7 @@ services: lab5-experiment: profiles: ["experiments"] - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/lab5experiment"] environment: MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml @@ -375,11 +386,12 @@ services: build: context: . target: prometheus - image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5} + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6} command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=7d + - --web.enable-remote-write-receiver volumes: - prometheus_data:/prometheus ports: @@ -398,6 +410,9 @@ services: - dispatcher - worker - s3 + - otel-collector + - loki + - tempo cpus: ${PROMETHEUS_CPUS} mem_limit: ${PROMETHEUS_MEMORY} restart: unless-stopped @@ -407,28 +422,98 @@ services: build: context: . target: grafana - image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5} + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6} environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} GF_USERS_ALLOW_SIGN_UP: "false" + GF_EXPLORE_ENABLED: "true" volumes: - grafana_data:/var/lib/grafana ports: - "${GRAFANA_BIND_ADDRESS}:3000" depends_on: - prometheus + - loki + - tempo + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"] + interval: 5s + timeout: 3s + retries: 20 cpus: ${GRAFANA_CPUS} mem_limit: ${GRAFANA_MEMORY} restart: unless-stopped networks: [edge, monitoring] + loki: + image: grafana/loki:3.7.9 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + ports: + - "${LOKI_BIND_ADDRESS}:3100" + healthcheck: + test: ["CMD", "/usr/bin/loki", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${LOKI_CPUS} + mem_limit: ${LOKI_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + tempo: + image: grafana/tempo:2.10.10 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + ports: + - "${TEMPO_BIND_ADDRESS}:3200" + healthcheck: + test: ["CMD", "/tempo", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${TEMPO_CPUS} + mem_limit: ${TEMPO_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.160.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + expose: + - "4317" + - "4318" + - "8888" + - "8889" + - "13133" + depends_on: + - loki + - tempo + healthcheck: + test: ["CMD", "/otelcol-contrib", "components"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${OTEL_COLLECTOR_CPUS} + mem_limit: ${OTEL_COLLECTOR_MEMORY} + restart: unless-stopped + networks: [application, data, monitoring] + volumes: postgres_data: valkey_data: redis_broker_data: prometheus_data: grafana_data: + loki_data: + tempo_data: s3_data: networks: diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..4c04d5d --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,48 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9096 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + volume_enabled: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - service.name + - deployment.environment.name + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e912195 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,73 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_constant_labels: + included: + - service.name + - service.version + - deployment.environment.name + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_grpc/tempo] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..0d8a229 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,48 @@ +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9095 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +metrics_generator: + registry: + external_labels: + source: tempo + deployment_environment: lab + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] + +usage_report: + reporting_enabled: false diff --git a/lab5.env.example b/lab6.env.example similarity index 80% rename from lab5.env.example rename to lab6.env.example index 5d3809a..ca34a40 100644 --- a/lab5.env.example +++ b/lab6.env.example @@ -1,3 +1,6 @@ +# Изолированное имя Compose-проекта и его volumes/networks. +COMPOSE_PROJECT_NAME=mtusi2026lab6 + # Адрес и внутренний порт HTTP-сервера. APP_HOST=0.0.0.0 APP_PORT=8080 @@ -8,6 +11,14 @@ API_BIND_ADDRESS=127.0.0.1:8080 SWAGGER_BIND_ADDRESS=127.0.0.1:8081 PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 GRAFANA_BIND_ADDRESS=127.0.0.1:3000 +LOKI_BIND_ADDRESS=127.0.0.1:3100 +TEMPO_BIND_ADDRESS=127.0.0.1:3200 + +# Ресурсные атрибуты и внутренний OTLP endpoint приложений. +DEPLOYMENT_ENVIRONMENT=lab +SERVICE_VERSION=lab6 +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc # Число API-реплик за Nginx. API_REPLICAS=2 @@ -55,8 +66,14 @@ EXPORTER_CPUS=0.25 EXPORTER_MEMORY=128m PROMETHEUS_CPUS=1 PROMETHEUS_MEMORY=512m -GRAFANA_CPUS=0.5 -GRAFANA_MEMORY=256m +GRAFANA_CPUS=1 +GRAFANA_MEMORY=512m +OTEL_COLLECTOR_CPUS=0.5 +OTEL_COLLECTOR_MEMORY=256m +LOKI_CPUS=1 +LOKI_MEMORY=512m +TEMPO_CPUS=1 +TEMPO_MEMORY=512m POSTGRES_CPUS=2 POSTGRES_MEMORY=1g VALKEY_CPUS=1 diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger-lab6.json similarity index 55% rename from monitoring/grafana/dashboards/messenger.json rename to monitoring/grafana/dashboards/messenger-lab6.json index de0ef2d..f20cabe 100644 --- a/monitoring/grafana/dashboards/messenger.json +++ b/monitoring/grafana/dashboards/messenger-lab6.json @@ -67,7 +67,7 @@ }, { "type": "timeseries", - "title": "API latency p95 by route", + "title": "API latency p50 / p95 / p99 by route", "id": 3, "gridPos": { "h": 8, @@ -77,9 +77,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", - "legendFormat": "{{method}} {{route}}", + "expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50 {{method}} {{route}}", "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95 {{method}} {{route}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p99 {{method}} {{route}}", + "refId": "C" } ], "fieldConfig": { @@ -895,14 +905,563 @@ } } ] + }, + { + "type": "row", + "title": "SLO and alerts", + "id": 105, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 109 + }, + "panels": [ + { + "type": "timeseries", + "title": "Business API availability", + "id": 31, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 110 + }, + "targets": [ + { + "expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "availability", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "API error-budget burn rate (99.5% SLO)", + "id": 32, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 110 + }, + "targets": [ + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachments ready within 30 seconds", + "id": 33, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 118 + }, + "targets": [ + { + "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "ready in 30s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachment ready-time error-budget burn", + "id": 34, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 118 + }, + "targets": [ + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "SLO alert state", + "id": 35, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 126 + }, + "targets": [ + { + "expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})", + "legendFormat": "{{alertname}} {{alertstate}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Runtime resources", + "id": 107, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 134 + }, + "panels": [ + { + "type": "timeseries", + "title": "Process CPU by service", + "id": 38, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Go heap by service", + "id": 39, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Database pool used and wait rate", + "id": 40, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 143 + }, + "targets": [ + { + "expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})", + "legendFormat": "{{service_name}} used", + "refId": "A" + }, + { + "expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))", + "legendFormat": "{{service_name}} waits/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Worker saturation", + "id": 41, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 143 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "busy", + "refId": "A" + }, + { + "expr": "sum(messenger_attachment_worker_capacity)", + "legendFormat": "capacity", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Logs and traces", + "id": 106, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 151 + }, + "panels": [ + { + "type": "logs", + "title": "Correlated application logs", + "id": 36, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 152 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=~\"messenger-.+\"} | json", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + } + }, + { + "type": "traces", + "title": "Recent application traces (TraceQL)", + "id": 37, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 152 + }, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "query": "{ resource.service.name =~ \"messenger-.*\" }", + "queryType": "traceql", + "refId": "A", + "limit": 50, + "tableType": "traces" + } + ], + "options": { + "showSpanFilters": true + } + }, + { + "type": "timeseries", + "title": "Trace-derived service call rate", + "id": 42, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))", + "legendFormat": "{{service}} {{span_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "type": "timeseries", + "title": "Collector delivery failures and queue pressure", + "id": 43, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))", + "legendFormat": "{{exporter}} spans failed/s", + "refId": "A" + }, + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))", + "legendFormat": "{{exporter}} logs failed/s", + "refId": "B" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "{{exporter}} queue", + "refId": "C" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_capacity)", + "legendFormat": "{{exporter}} capacity", + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] } ], "refresh": "5s", "schemaVersion": 41, "tags": [ "highload", - "lab5", - "pipeline" + "lab6", + "observability", + "slo" ], "templating": { "list": [] @@ -912,7 +1471,42 @@ "to": "now" }, "timezone": "browser", - "title": "Messenger lab5 overview", - "uid": "messenger-lab5", - "version": 3 + "title": "Messenger lab6 observability", + "uid": "messenger-lab6", + "version": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Explore logs and traces", + "type": "link", + "url": "/explore" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "API runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Pipeline runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md" + } + ] } diff --git a/monitoring/grafana/provisioning/alerting/.gitkeep b/monitoring/grafana/provisioning/alerting/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/alerting/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/grafana/provisioning/datasources/observability.yml b/monitoring/grafana/provisioning/datasources/observability.yml new file mode 100644 index 0000000..99af68b --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/observability.yml @@ -0,0 +1,74 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + derivedFields: + - name: TraceID + matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})' + datasourceUid: tempo + url: '$${__value.raw}' + + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + nodeGraph: + enabled: true + search: + hide: false + serviceMap: + datasourceUid: prometheus + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"' + tags: + - key: service.name + value: service_name + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + tags: + - key: service.name + value: service + queries: + - name: Request rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))' + - name: Error rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))' diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 8704d9e..0000000 --- a/monitoring/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - uid: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: false diff --git a/monitoring/grafana/provisioning/plugins/.gitkeep b/monitoring/grafana/provisioning/plugins/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/plugins/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 1869184..70b5dfd 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -1,6 +1,11 @@ global: - scrape_interval: 1s - evaluation_interval: 1s + scrape_interval: 2s + evaluation_interval: 5s + external_labels: + deployment_environment: lab + +rule_files: + - /etc/prometheus/rules/*.yml scrape_configs: - job_name: messenger @@ -9,6 +14,13 @@ scrape_configs: type: A port: 8080 refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-api + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep - job_name: nginx static_configs: @@ -32,12 +44,42 @@ scrape_configs: type: A port: 9091 refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-worker + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep - job_name: outbox-dispatcher static_configs: - targets: ["dispatcher:9091"] + relabel_configs: + - target_label: service_name + replacement: messenger-dispatcher + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep - job_name: minio metrics_path: /minio/v2/metrics/cluster static_configs: - targets: ["s3:9000"] + + - job_name: otel-applications + static_configs: + - targets: ["otel-collector:8889"] + + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] diff --git a/monitoring/rules/messenger.yml b/monitoring/rules/messenger.yml new file mode 100644 index 0000000..4999ddb --- /dev/null +++ b/monitoring/rules/messenger.yml @@ -0,0 +1,178 @@ +groups: + - name: messenger-api-slo + interval: 5s + rules: + - alert: MessengerAPIBusiness5xxHigh + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0)) + / + clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001) + ) > 0.05 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: Business API 5xx ratio is above 5% + description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnFast + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001) + ) > 0.072 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001) + ) > 0.072 + for: 30s + labels: + severity: critical + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning fast + description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnSlow + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001) + ) > 0.03 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001) + ) > 0.03 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning steadily + description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - name: messenger-attachment-slo + interval: 5s + rules: + - alert: AttachmentReadyHighErrorBudgetBurnFast + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)), + 0.000001 + ) > 0.144 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)), + 0.000001 + ) > 0.144 + for: 30s + labels: + severity: critical + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning fast + description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentReadyHighErrorBudgetBurnSlow + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)), + 0.000001 + ) > 0.06 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)), + 0.000001 + ) > 0.06 + for: 1m + labels: + severity: warning + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning steadily + description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentWorkerUnavailable + expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1 + for: 30s + labels: + severity: critical + service: messenger-worker + annotations: + summary: No attachment worker is available + description: Prometheus has not observed a healthy worker target for 30 seconds. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentQueueBacklogGrowing + expr: | + max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30 + and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0 + for: 30s + labels: + severity: warning + service: messenger-worker + annotations: + summary: Attachment queue backlog is growing + description: The oldest queued attachment is older than 30 seconds and its age continues to increase. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentRedisStreamLagHigh + expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20 + for: 1m + labels: + severity: warning + service: redis-broker + annotations: + summary: Redis Stream consumer lag is high + description: The attachment worker group has more than 20 messages waiting for delivery. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md diff --git a/monitoring/runbooks/api-availability.md b/monitoring/runbooks/api-availability.md new file mode 100644 index 0000000..2ff5453 --- /dev/null +++ b/monitoring/runbooks/api-availability.md @@ -0,0 +1,72 @@ +# API availability SLO + +## Цель и сигналы + +Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%. + +Основные сигналы: + +- `messenger_http_requests_total{method,route,status_class}` — число запросов; +- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency; +- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets; +- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения; +- trace `messenger-api` — путь одного запроса по зависимостям; +- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`. + +`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента. +2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью. +3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler. +4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL: + + ```text + { resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error } + ``` + +5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют. +6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki: + + ```text + {service_name="messenger-api"} | json | trace_id="" + ``` + +7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют. + +## Проверка alert + +Состояние правил видно в Grafana Alerting и в Prometheus: + +```bash +curl -s http://127.0.0.1:9090/api/v1/alerts +``` + +Контролируемый отказ PostgreSQL: + +```bash +ENV_FILE=lab6.env.example scripts/incident-database.sh +``` + +Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются. + +Ожидаемая последовательность: + +```text +нормальный error ratio +→ PostgreSQL остановлен +→ 5xx растёт +→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing +→ PostgreSQL восстановлен +→ успешные ответы возвращаются +→ оперативный alert resolved; burn-rate хранит историю длинного окна +``` + +## Действия + +1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи. +2. Если API targets недоступны, восстановить реплики по одной и проверить readiness. +3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины. +4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline. +5. Не удалять volumes и не сбрасывать очередь ради закрытия alert. diff --git a/monitoring/runbooks/attachment-pipeline.md b/monitoring/runbooks/attachment-pipeline.md new file mode 100644 index 0000000..c2ae007 --- /dev/null +++ b/monitoring/runbooks/attachment-pipeline.md @@ -0,0 +1,85 @@ +# Attachment pipeline SLO + +## Цель и сигналы + +Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget. + +Основные сигналы: + +- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker; +- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки; +- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время; +- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL; +- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job; +- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker; +- `messenger_outbox_pending` — задержка до публикации; +- Redis Stream lag и PEL — доставка и неподтверждённые сообщения. + +Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента. +2. Определить место задержки: + - растёт `messenger_outbox_pending` — проблема до публикации в broker; + - растут stream lag и `queued` — worker не успевает или недоступен; + - растут PEL и `processing` — delivery получена, но не завершена; + - растёт `failed` — постоянные ошибки или исчерпаны retry; + - busy равен capacity — worker насыщен. +3. В Explore → Tempo найти upload по route или worker consumer span: + + ```text + { resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" } + ``` + + ```text + { resource.service.name = "messenger-worker" && name = "attachment.process" } + ``` + +4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label. +5. Перейти из выбранного span в Loki или выполнить: + + ```text + {service_name="messenger-worker"} | json | trace_id="" + ``` + +6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются. + +## Проверка alert + +Короткая деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh short +``` + +Длительная деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh long +``` + +Ожидаемая последовательность: + +```text +worker up, backlog около нуля +→ worker остановлен +→ queued и oldest age растут +→ short: AttachmentWorkerUnavailable firing +→ long: AttachmentQueueBacklogGrowing firing +→ worker запущен +→ backlog дренируется +→ все принятые attachments достигают ready +→ alert resolved +``` + +Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload. + +## Действия + +1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования. +2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency. +3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина. +4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim. +5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины. +6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment. -- 2.54.0 From 441b752257656818bd3bb40a869861c11e973938 Mon Sep 17 00:00:00 2001 From: dizi Date: Sun, 1 Nov 2026 17:06:37 +0300 Subject: [PATCH 3/8] test: add SLO incident verification --- .gitea/workflows/ci.yml | 12 +- .gitea/workflows/deploy.yml | 32 ++-- .gitea/workflows/incidents.yml | 73 +++++++++ .gitignore | 2 + Makefile | 11 +- evidence/lab6-results.json | 59 +++++++ evidence/lab6-schema.json | 37 +++++ scripts/incident-database.sh | 91 +++++++++++ scripts/incident-telemetry.sh | 73 +++++++++ scripts/incident-worker.sh | 107 ++++++++++++ scripts/lab6-alert.py | 67 ++++++++ scripts/lab6-correlation.py | 148 +++++++++++++++++ scripts/lab6-observe.py | 86 ++++++++++ scripts/lab6-traffic.py | 261 ++++++++++++++++++++++++++++++ scripts/test-lab6.sh | 73 +++++++++ scripts/validate-lab6-evidence.py | 101 ++++++++++++ 16 files changed, 1216 insertions(+), 17 deletions(-) create mode 100644 .gitea/workflows/incidents.yml create mode 100644 evidence/lab6-results.json create mode 100644 evidence/lab6-schema.json create mode 100755 scripts/incident-database.sh create mode 100755 scripts/incident-telemetry.sh create mode 100755 scripts/incident-worker.sh create mode 100755 scripts/lab6-alert.py create mode 100755 scripts/lab6-correlation.py create mode 100644 scripts/lab6-observe.py create mode 100755 scripts/lab6-traffic.py create mode 100755 scripts/test-lab6.sh create mode 100755 scripts/validate-lab6-evidence.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index f2d5cad..33018c7 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -16,11 +16,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: - COMPOSE_PROJECT_NAME: mtusi2026lab5-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }} API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + LOKI_BIND_ADDRESS: 127.0.0.1:0 + TEMPO_BIND_ADDRESS: 127.0.0.1:0 steps: - name: Checkout exact commit run: | @@ -46,7 +48,8 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py - name: Lint uses: golangci/golangci-lint-action@v7 @@ -62,11 +65,12 @@ jobs: - name: Prepare local stack run: | - cp lab5.env.example .env + cp lab6.env.example .env docker compose config --quiet docker compose up -d --build --scale api=2 --wait --wait-timeout 150 docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" - name: Integration checks run: | @@ -82,6 +86,7 @@ jobs: docker compose up -d --scale api=2 --wait --wait-timeout 90 make test-lab5-acceptance make test-lab5 + ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 - name: Container logs if: failure() @@ -92,4 +97,5 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 54da2be..b1b9a29 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -22,19 +22,21 @@ jobs: s3_image: ${{ steps.publish.outputs.s3_image }} s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }} env: - COMPOSE_PROJECT_NAME: mtusi2026lab5-deploy-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }} COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 GRAFANA_BIND_ADDRESS: 127.0.0.1:0 - API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-api:${{ github.sha }} - NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-nginx:${{ github.sha }} - SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-swagger:${{ github.sha }} - PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-prometheus:${{ github.sha }} - GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-grafana:${{ github.sha }} - S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio:${{ github.sha }} - S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio-client:${{ github.sha }} + LOKI_BIND_ADDRESS: 127.0.0.1:0 + TEMPO_BIND_ADDRESS: 127.0.0.1:0 + API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }} + NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }} + SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }} + PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }} + GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }} + S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }} + S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }} steps: - name: Checkout exact commit run: | @@ -60,7 +62,8 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py - name: Lint uses: golangci/golangci-lint-action@v7 @@ -76,11 +79,12 @@ jobs: - name: Prepare local stack run: | - cp lab5.env.example .env + cp lab6.env.example .env docker compose config --quiet docker compose up -d --build --scale api=2 --wait --wait-timeout 150 docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" - name: Integration checks run: | @@ -95,6 +99,7 @@ jobs: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 make test-lab5-acceptance make test-lab5 + ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 - name: Publish verified images id: publish @@ -132,6 +137,7 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi deploy: @@ -171,17 +177,17 @@ jobs: run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin" - name: Upload image manifest - run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab5-${{ github.run_id }}.env" + run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab6-${{ github.run_id }}.env" - name: Deploy and verify run: | ssh "$DEPLOY_USER@$DEPLOY_HOST" " set -eu - cd ~/mtusi2026lab5 + cd ~/mtusi2026lab6 git fetch origin main git cat-file -e '$DEPLOY_SHA^{commit}' git reset --hard '$DEPLOY_SHA' - manifest='/tmp/mtusi2026lab5-${{ github.run_id }}.env' + manifest='/tmp/mtusi2026lab6-${{ github.run_id }}.env' if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then rm -f \"\$manifest\" else diff --git a/.gitea/workflows/incidents.yml b/.gitea/workflows/incidents.yml new file mode 100644 index 0000000..244f11e --- /dev/null +++ b/.gitea/workflows/incidents.yml @@ -0,0 +1,73 @@ +name: Lab 6 incidents + +on: + workflow_dispatch: + +concurrency: + group: lab6-incidents-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + incidents: + runs-on: ubuntu-latest + timeout-minutes: 35 + env: + COMPOSE_PROJECT_NAME: mtusi2026lab6-incidents-${{ github.run_id }} + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + LOKI_BIND_ADDRESS: 127.0.0.1:0 + TEMPO_BIND_ADDRESS: 127.0.0.1:0 + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Prepare stack + run: | + cp lab6.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 180 + docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + + - name: Reproduce SLO incidents + run: | + ENV_FILE=.env \ + BASE_URL=http://nginx:8080 \ + PROMETHEUS_URL=http://prometheus:9090 \ + TEMPO_URL=http://tempo:3200 \ + LOKI_URL=http://loki:3100 \ + EVIDENCE_DIR=evidence/runtime/ci \ + make test-lab6 + + - name: Upload sanitized evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: lab6-incidents-${{ github.sha }} + path: evidence/runtime/ci + if-no-files-found: warn + retention-days: 14 + + - name: Container logs + if: failure() + run: docker compose logs --no-color --tail=200 + + - name: Stop local stack + if: always() + run: | + docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker compose down -v --remove-orphans diff --git a/.gitignore b/.gitignore index b5a7189..7913498 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +evidence/runtime/ diff --git a/Makefile b/Makefile index 43e0a2a..8a72b7e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance +.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -34,3 +34,12 @@ test-lab5: test-lab5-acceptance: ./scripts/test-lab5-acceptance.sh + +lab6-load: + $(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" --mode mixed --duration 60 --concurrency 4 --wait-ready + +test-lab6-smoke: + BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh smoke + +test-lab6: + BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh full diff --git a/evidence/lab6-results.json b/evidence/lab6-results.json new file mode 100644 index 0000000..3ae353c --- /dev/null +++ b/evidence/lab6-results.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "captured_at": "2026-11-01T14:05:43Z", + "baseline": { + "api_request_rate_per_second": 1.1485206460733426, + "firing_alerts": 0, + "traces_by_service": { + "messenger-api": 39, + "messenger-worker": 23, + "messenger-dispatcher": 22 + }, + "logs_by_service": { + "messenger-api": 162, + "messenger-worker": 40, + "messenger-dispatcher": 1 + }, + "api_worker_shared_traces": 3, + "traces_with_correlated_logs": 37 + }, + "incidents": [ + { + "name": "worker_unavailable_short", + "injected_duration_seconds": 76, + "alert_detection_upper_bound_seconds": 76, + "alert_resolution_seconds": 15, + "recovery_observation_seconds": 36 + }, + { + "name": "postgres_unavailable", + "injected_duration_seconds": 76, + "alert_detection_upper_bound_seconds": 76, + "operational_alert_resolution_seconds": 127, + "recovery_observation_seconds": 148, + "fast_burn_alert_observed": true, + "database_error_traces": 4, + "database_error_traces_with_logs": 4 + }, + { + "name": "worker_unavailable_long", + "injected_duration_seconds": 180, + "alert_detection_upper_bound_seconds": 180, + "alert_resolution_seconds": 15, + "recovery_observation_seconds": 65, + "fast_burn_alert_observed": true, + "slow_burn_alert_observed": true + }, + { + "name": "telemetry_backends_unavailable", + "injected_duration_seconds": 30, + "api_ready_http_status": 200, + "recovery_observation_seconds": 31 + } + ], + "validation": { + "valid": true, + "errors": [], + "incident_summaries": 4 + } +} diff --git a/evidence/lab6-schema.json b/evidence/lab6-schema.json new file mode 100644 index 0000000..c119c53 --- /dev/null +++ b/evidence/lab6-schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/evidence/lab6-schema.json", + "title": "Lab 6 incident summary", + "type": "object", + "required": [ + "schema_version", + "incident", + "started_at", + "recovery_started_at", + "recovered_at", + "injected_duration_seconds", + "recovery_observation_seconds", + "artifacts" + ], + "properties": { + "schema_version": {"const": 1}, + "incident": { + "enum": ["worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"] + }, + "profile": {"enum": ["short", "long"]}, + "started_at": {"type": "string", "format": "date-time"}, + "recovery_started_at": {"type": "string", "format": "date-time"}, + "recovered_at": {"type": "string", "format": "date-time"}, + "planned_injection_seconds": {"type": "integer", "minimum": 1}, + "injected_duration_seconds": {"type": "integer", "minimum": 1}, + "recovery_observation_seconds": {"type": "integer", "minimum": 0}, + "api_ready_http_status_during_outage": {"type": "integer", "minimum": 100, "maximum": 599}, + "artifacts": { + "type": "array", + "items": {"type": "string", "pattern": "^[A-Za-z0-9._-]+$"}, + "minItems": 4, + "uniqueItems": true + } + }, + "additionalProperties": false +} diff --git a/scripts/incident-database.sh b/scripts/incident-database.sh new file mode 100755 index 0000000..bc3c6e0 --- /dev/null +++ b/scripts/incident-database.sh @@ -0,0 +1,91 @@ +#!/bin/sh + +set -eu + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} +loki_url=${LOKI_URL:-http://127.0.0.1:3100} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +outage_seconds=${DB_OUTAGE_SECONDS:-75} +require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} +api_alert_name=${API_ALERT_NAME:-MessengerAPIBusiness5xxHigh} +burn_alert_name=${API_BURN_ALERT_NAME:-MessengerAPIHighErrorBudgetBurnFast} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/database-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d postgres migrate api nginx >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode messages \ + --duration "$((outage_seconds + 20))" \ + --concurrency 4 \ + --rate "${DB_LOAD_RATE:-20}" \ + --started-file "$output_dir/load-started" \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +remaining=60 +while [ ! -f "$output_dir/load-started" ] && [ "$remaining" -gt 0 ]; do + sleep 0.5 + remaining=$((remaining - 1)) +done +test -f "$output_dir/load-started" +sleep 2 +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop postgres >/dev/null +sleep "$outage_seconds" +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$burn_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-alert-firing.json" >/dev/null +fi +python3 scripts/lab6-correlation.py --mode database --tempo-url "$tempo_url" --loki-url "$loki_url" --output "$output_dir/database-correlation.json" --require >/dev/null +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 postgres migrate api nginx >/dev/null +wait "$load_pid" || true +load_pid= +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null +fi +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" <<'PY' +import datetime +import json +import pathlib +import sys + +output, started, recovery_started, recovered, duration = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "postgres_unavailable", + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "planned_injection_seconds": int(duration), + "injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "database-correlation.json", "alert-firing.json", "alert-resolved.json", "burn-alert-firing.json"], +} +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/incident-telemetry.sh b/scripts/incident-telemetry.sh new file mode 100755 index 0000000..1f0cc50 --- /dev/null +++ b/scripts/incident-telemetry.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +set -eu + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/telemetry-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d tempo loki >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop tempo loki >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode mixed \ + --duration "$outage_seconds" \ + --concurrency 4 \ + --rate "${TELEMETRY_LOAD_RATE:-10}" \ + --wait-ready \ + --ready-timeout 120 \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +sleep "$outage_seconds" +wait "$load_pid" +load_pid= +health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready") +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 tempo loki >/dev/null +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" "$health_code" <<'PY' +import datetime +import json +import pathlib +import sys + +output, started, recovery_started, recovered, duration, health_code = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "telemetry_backends_unavailable", + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "injected_duration_seconds": int(duration), + "api_ready_http_status_during_outage": int(health_code), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json"], +} +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/incident-worker.sh b/scripts/incident-worker.sh new file mode 100755 index 0000000..016df05 --- /dev/null +++ b/scripts/incident-worker.sh @@ -0,0 +1,107 @@ +#!/bin/sh + +set -eu + +mode=${1:-short} +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} +case "$mode" in + short) + incident_seconds=${SHORT_WORKER_OUTAGE_SECONDS:-75} + load_seconds=${SHORT_WORKER_LOAD_SECONDS:-60} + requests=${SHORT_WORKER_REQUESTS:-300} + default_alert=AttachmentWorkerUnavailable + ;; + long) + incident_seconds=${LONG_WORKER_OUTAGE_SECONDS:-180} + load_seconds=${LONG_WORKER_LOAD_SECONDS:-120} + requests=${LONG_WORKER_REQUESTS:-1000} + default_alert=AttachmentQueueBacklogGrowing + ;; + *) + echo "usage: $0 [short|long]" >&2 + exit 2 + ;; +esac +worker_alert_name=${WORKER_ALERT_NAME:-$default_alert} + +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/worker-$mode-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d worker >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop worker >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode attachments \ + --duration "$load_seconds" \ + --requests "$requests" \ + --concurrency 4 \ + --rate "${WORKER_LOAD_RATE:-5}" \ + --wait-ready \ + --ready-timeout 480 \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +sleep "$incident_seconds" +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null +fi +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 worker >/dev/null +wait "$load_pid" +load_pid= +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null + if [ "$mode" = "long" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnFast --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-fast-firing.json" >/dev/null + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnSlow --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-slow-firing.json" >/dev/null + fi +fi +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$mode" "$started_at" "$recovery_started_at" "$recovered_at" "$incident_seconds" <<'PY' +import datetime +import json +import pathlib +import sys + +output, mode, started, recovery_started, recovered, duration = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "worker_unavailable", + "profile": mode, + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "planned_injection_seconds": int(duration), + "injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "alert-firing.json", "alert-resolved.json"], +} +if mode == "long": + report["artifacts"].extend(["burn-fast-firing.json", "burn-slow-firing.json"]) +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/lab6-alert.py b/scripts/lab6-alert.py new file mode 100755 index 0000000..1e75ce9 --- /dev/null +++ b/scripts/lab6-alert.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import time +import urllib.error +import urllib.request + + +def fetch(prometheus_url): + url = prometheus_url.rstrip("/") + "/api/v1/alerts" + with urllib.request.urlopen(url, timeout=10) as response: + payload = json.loads(response.read(4 << 20).decode("utf-8")) + return payload.get("data", {}).get("alerts", []) + + +def matching(alerts, name): + return [alert for alert in alerts if alert.get("labels", {}).get("alertname") == name] + + +def main(): + parser = argparse.ArgumentParser(description="Wait for a Prometheus alert transition") + parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090") + parser.add_argument("--name", required=True) + parser.add_argument("--state", choices=("firing", "resolved"), required=True) + parser.add_argument("--timeout", type=float, default=120) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + deadline = time.monotonic() + args.timeout + last = [] + error = None + observed = False + while time.monotonic() < deadline: + try: + last = matching(fetch(args.prometheus_url), args.name) + error = None + if args.state == "firing" and any(item.get("state") == "firing" for item in last): + observed = True + break + if args.state == "resolved" and not any(item.get("state") in ("pending", "firing") for item in last): + observed = True + break + except (urllib.error.URLError, json.JSONDecodeError) as current_error: + error = str(current_error) + time.sleep(2) + + report = { + "schema_version": 1, + "alert": args.name, + "expected_state": args.state, + "observed": observed, + "observed_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "last_states": sorted({item.get("state", "") for item in last}), + "error": error, + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-correlation.py b/scripts/lab6-correlation.py new file mode 100755 index 0000000..9b981a7 --- /dev/null +++ b/scripts/lab6-correlation.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import re +import time +import urllib.error +import urllib.parse +import urllib.request + +TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b') +SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher") + + +def get_json(url): + with urllib.request.urlopen(url, timeout=10) as response: + return json.loads(response.read(8 << 20).decode("utf-8")) + + +def tempo_query(base_url, query): + url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100}) + payload = get_json(url) + return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")} + + +def extract_trace_ids(payload): + result = set() + for stream in payload.get("data", {}).get("result", []): + for value in stream.get("values", []): + candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""] + if len(value) > 2: + candidates.append(value[2]) + for candidate in candidates: + if isinstance(candidate, dict): + trace_id = candidate.get("trace_id") or candidate.get("traceid") + if trace_id: + result.add(str(trace_id)) + candidate = json.dumps(candidate) + result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate))) + return result + + +def loki_logs(base_url, service, start_ns): + query = '{service_name="' + service + '"}' + url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({ + "query": query, + "start": str(start_ns), + "limit": 5000, + "direction": "backward", + }) + payload = get_json(url) + count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", [])) + return count, extract_trace_ids(payload) + + +def capture(tempo_url, loki_url, lookback_seconds): + start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) + traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES} + logs = {} + log_trace_ids = {} + for service in SERVICES: + count, ids = loki_logs(loki_url, service, start_ns) + logs[service] = count + log_trace_ids[service] = ids + api_worker = traces["messenger-api"] & traces["messenger-worker"] + all_trace_ids = set().union(*traces.values()) + all_log_trace_ids = set().union(*log_trace_ids.values()) + return { + "traces_by_service": {service: len(ids) for service, ids in traces.items()}, + "logs_by_service": logs, + "api_worker_shared_traces": len(api_worker), + "traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids), + } + + +def capture_database(tempo_url, loki_url, lookback_seconds): + start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) + traces = tempo_query( + tempo_url, + '{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }', + ) + log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns) + return { + "database_error_traces": len(traces), + "api_logs": log_count, + "database_error_traces_with_logs": len(traces & log_trace_ids), + } + + +def complete(report): + return ( + report["traces_by_service"]["messenger-api"] > 0 + and report["traces_by_service"]["messenger-worker"] > 0 + and report["logs_by_service"]["messenger-api"] > 0 + and report["logs_by_service"]["messenger-worker"] > 0 + and report["api_worker_shared_traces"] > 0 + and report["traces_with_correlated_logs"] > 0 + ) + + +def database_complete(report): + return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0 + + +def main(): + parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki") + parser.add_argument("--tempo-url", default="http://127.0.0.1:3200") + parser.add_argument("--loki-url", default="http://127.0.0.1:3100") + parser.add_argument("--lookback", type=float, default=900) + parser.add_argument("--wait", type=float, default=45) + parser.add_argument("--output", required=True) + parser.add_argument("--require", action="store_true") + parser.add_argument("--mode", choices=("baseline", "database"), default="baseline") + args = parser.parse_args() + + deadline = time.monotonic() + args.wait + error = None + report = None + while time.monotonic() < deadline: + try: + report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback) + error = None + if database_complete(report) if args.mode == "database" else complete(report): + break + except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error: + error = str(current_error) + time.sleep(2) + + result = { + "schema_version": 1, + "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "lookback_seconds": args.lookback, + "signals": report, + "mode": args.mode, + "complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))), + "error": error, + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, ensure_ascii=False, indent=2)) + if args.require and not result["complete"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-observe.py b/scripts/lab6-observe.py new file mode 100644 index 0000000..4614176 --- /dev/null +++ b/scripts/lab6-observe.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import time +import urllib.error +import urllib.parse +import urllib.request + +QUERIES = { + "api_error_ratio_1m": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)', + "api_request_rate_1m": 'sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))', + "queued_jobs": 'sum(messenger_attachment_jobs_count{status="queued"})', + "processing_jobs": 'sum(messenger_attachment_jobs_count{status="processing"})', + "oldest_queued_job_seconds": 'max(messenger_attachment_jobs_oldest_age_seconds{status="queued"})', + "worker_jobs_in_flight": 'sum(messenger_attachment_worker_jobs_in_flight)', + "worker_capacity": 'sum(messenger_attachment_worker_capacity)', + "pending_outbox": 'sum(messenger_outbox_pending)', + "worker_up": 'min(up{job="attachment-worker"})', + "collector_export_failures_5m": 'sum(increase(otelcol_exporter_send_failed_spans_total[5m])) + sum(increase(otelcol_exporter_send_failed_log_records_total[5m]))', +} + + +def get_json(url): + try: + with urllib.request.urlopen(url, timeout=10) as response: + return json.loads(response.read(4 << 20).decode("utf-8")), None + except (urllib.error.URLError, json.JSONDecodeError) as error: + return None, str(error) + + +def query(prometheus_url, expression): + url = prometheus_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode({"query": expression}) + payload, error = get_json(url) + if error: + return {"error": error} + if payload.get("status") != "success": + return {"error": payload.get("error", "query failed")} + rows = payload.get("data", {}).get("result", []) + return { + "series": [ + {"labels": row.get("metric", {}), "value": row.get("value", [None, None])[1]} + for row in rows + ] + } + + +def alerts(prometheus_url): + payload, error = get_json(prometheus_url.rstrip("/") + "/api/v1/alerts") + if error: + return {"error": error, "items": []} + items = [] + for alert in payload.get("data", {}).get("alerts", []): + labels = alert.get("labels", {}) + items.append({ + "name": labels.get("alertname", ""), + "state": alert.get("state", ""), + "severity": labels.get("severity", ""), + "active_at": alert.get("activeAt", ""), + }) + return {"items": sorted(items, key=lambda item: (item["name"], item["state"]))} + + +def main(): + parser = argparse.ArgumentParser(description="Capture a bounded Prometheus snapshot for Lab 6 evidence") + parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090") + parser.add_argument("--phase", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + report = { + "schema_version": 1, + "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "phase": args.phase, + "queries": {name: query(args.prometheus_url, expression) for name, expression in QUERIES.items()}, + "alerts": alerts(args.prometheus_url), + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-traffic.py b/scripts/lab6-traffic.py new file mode 100755 index 0000000..244d53b --- /dev/null +++ b/scripts/lab6-traffic.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import json +import math +from pathlib import Path +import secrets +import struct +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import zlib + +MAX_RESPONSE_BYTES = 1 << 20 + + +def png_fixture(width=96, height=64): + rows = [] + for y in range(height): + pixels = bytearray() + for x in range(width): + pixels.extend(((x * 3) % 256, (y * 5) % 256, (x + y) % 256)) + rows.append(b"\x00" + bytes(pixels)) + raw = b"".join(rows) + + def chunk(name, payload): + return struct.pack(">I", len(payload)) + name + payload + struct.pack(">I", zlib.crc32(name + payload) & 0xFFFFFFFF) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"") + + +def request(method, url, body=None, headers=None, timeout=10): + req = urllib.request.Request(url, data=body, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + payload = response.read(MAX_RESPONSE_BYTES) + return response.status, payload + except urllib.error.HTTPError as error: + return error.code, error.read(MAX_RESPONSE_BYTES) + except urllib.error.URLError: + return 0, b"" + + +def decode_json(payload): + return json.loads(payload.decode("utf-8")) + + +def json_request(method, url, value=None, token=None, headers=None): + request_headers = {"Accept": "application/json"} + body = None + if value is not None: + body = json.dumps(value, separators=(",", ":")).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if token: + request_headers["Authorization"] = "Bearer " + token + if headers: + request_headers.update(headers) + status, payload = request(method, url, body, request_headers) + return status, decode_json(payload) if payload else {} + + +def bootstrap(base_url): + suffix = secrets.token_hex(6) + password = "Lab6-" + secrets.token_urlsafe(18) + users = [] + for prefix in ("telemetry_a_", "telemetry_b_"): + username = prefix + suffix + status, user = json_request("POST", base_url + "/api/v1/users", { + "username": username, + "display_name": "Synthetic telemetry load", + "password": password, + }) + if status != 201: + raise RuntimeError("register user returned HTTP %d" % status) + users.append((username, user["id"])) + + credentials = base64.b64encode((users[0][0] + ":" + password).encode("utf-8")).decode("ascii") + status, payload = request("POST", base_url + "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + credentials, + "Accept": "application/json", + }) + if status != 201: + raise RuntimeError("create session returned HTTP %d" % status) + token = decode_json(payload)["token"] + + status, chat = json_request("POST", base_url + "/api/v1/chats", { + "title": "Lab 6 synthetic telemetry", + "member_ids": [users[0][1], users[1][1]], + }, token) + if status != 201: + raise RuntimeError("create chat returned HTTP %d" % status) + return token, chat["id"] + + +class Results: + def __init__(self): + self.lock = threading.Lock() + self.latencies = [] + self.statuses = {} + self.attachment_ids = [] + + def record(self, status, elapsed, attachment_id=None): + with self.lock: + self.latencies.append(elapsed) + key = str(status) + self.statuses[key] = self.statuses.get(key, 0) + 1 + if attachment_id: + self.attachment_ids.append(attachment_id) + + +def percentile(values, quantile): + if not values: + return None + ordered = sorted(values) + index = max(0, math.ceil(quantile * len(ordered)) - 1) + return round(ordered[index] * 1000, 3) + + +def run_request(base_url, token, chat_id, mode, sequence, fixture): + if mode == "mixed": + mode = "attachments" if sequence % 3 == 0 else "messages" + started = time.monotonic() + if mode == "messages": + status, _ = json_request("POST", base_url + "/api/v1/chats/" + chat_id + "/messages", { + "text": "synthetic telemetry event", + "attachment_ids": [], + }, token, {"Idempotency-Key": secrets.token_hex(16)}) + return status, time.monotonic() - started, None + + upload_url = base_url + "/api/v1/chats/" + chat_id + "/attachments?" + urllib.parse.urlencode({ + "filename": "synthetic.png", + }) + status, payload = request("POST", upload_url, fixture, { + "Authorization": "Bearer " + token, + "Content-Type": "image/png", + "Accept": "application/json", + }) + attachment_id = None + if status == 202: + attachment_id = decode_json(payload).get("id") + return status, time.monotonic() - started, attachment_id + + +def wait_for_attachments(base_url, token, attachment_ids, timeout): + remaining = set(attachment_ids) + states = {"ready": 0, "failed": 0, "timeout": 0} + deadline = time.monotonic() + timeout + while remaining and time.monotonic() < deadline: + for attachment_id in list(remaining): + status, metadata = json_request("GET", base_url + "/api/v1/attachments/" + attachment_id, token=token) + if status != 200: + continue + state = metadata.get("status") + if state in ("ready", "failed"): + states[state] += 1 + remaining.remove(attachment_id) + if remaining: + time.sleep(0.25) + states["timeout"] = len(remaining) + return states + + +def main(): + parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents") + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed") + parser.add_argument("--duration", type=float, default=30) + parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--rate", type=float, default=0, help="Approximate aggregate request rate; zero is unthrottled") + parser.add_argument("--wait-ready", action="store_true") + parser.add_argument("--ready-timeout", type=float, default=120) + parser.add_argument("--output") + parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready") + args = parser.parse_args() + + if args.duration <= 0 or args.duration > 1800: + parser.error("duration must be in (0, 1800]") + if args.requests < 0: + parser.error("requests must be non-negative") + if args.concurrency < 1 or args.concurrency > 64: + parser.error("concurrency must be in [1, 64]") + if args.rate < 0 or args.rate > 1000: + parser.error("rate must be in [0, 1000]") + + base_url = args.base_url.rstrip("/") + token, chat_id = bootstrap(base_url) + if args.started_file: + started_file = Path(args.started_file) + started_file.parent.mkdir(parents=True, exist_ok=True) + started_file.touch() + fixture = png_fixture() + results = Results() + next_sequence = 0 + sequence_lock = threading.Lock() + deadline = time.monotonic() + args.duration + + def worker(): + nonlocal next_sequence + while time.monotonic() < deadline: + with sequence_lock: + if args.requests and next_sequence >= args.requests: + return + sequence = next_sequence + next_sequence += 1 + status, elapsed, attachment_id = run_request(base_url, token, chat_id, args.mode, sequence, fixture) + results.record(status, elapsed, attachment_id) + if args.rate: + delay = args.concurrency / args.rate - elapsed + if delay > 0: + time.sleep(delay) + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [pool.submit(worker) for _ in range(args.concurrency)] + for future in futures: + future.result() + elapsed = time.monotonic() - started + + attachment_states = None + if args.wait_ready and results.attachment_ids: + attachment_states = wait_for_attachments(base_url, token, results.attachment_ids, args.ready_timeout) + + total = len(results.latencies) + successes = sum(count for status, count in results.statuses.items() if status.startswith("2")) + report = { + "schema_version": 1, + "workload": args.mode, + "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - elapsed)), + "duration_seconds": round(elapsed, 3), + "concurrency": args.concurrency, + "requested_rate": args.rate or None, + "requests": total, + "successful_requests": successes, + "request_rate": round(total / elapsed, 3) if elapsed else None, + "status_counts": dict(sorted(results.statuses.items())), + "latency_ms": { + "p50": percentile(results.latencies, 0.50), + "p95": percentile(results.latencies, 0.95), + "p99": percentile(results.latencies, 0.99), + }, + "attachments_accepted": len(results.attachment_ids), + "attachment_terminal_states": attachment_states, + } + encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(encoded, encoding="utf-8") + print(encoded, end="") + if not total: + raise SystemExit("no requests were sent") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-lab6.sh b/scripts/test-lab6.sh new file mode 100755 index 0000000..c2ffb45 --- /dev/null +++ b/scripts/test-lab6.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +set -eu + +profile=${1:-full} +case "$profile" in + smoke|full) ;; + *) + echo "usage: $0 [smoke|full]" >&2 + exit 2 + ;; +esac + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} +loki_url=${LOKI_URL:-http://127.0.0.1:3100} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir=${EVIDENCE_DIR:-evidence/runtime/lab6-$run_id} +mkdir -p "$output_dir" + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + compose up -d postgres redis-broker s3 valkey api nginx dispatcher worker otel-collector prometheus loki tempo grafana >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +test -f "$env_file" +compose config --quiet +compose up -d --build --wait --wait-timeout 180 +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode mixed \ + --duration "${BASELINE_LOAD_SECONDS:-15}" \ + --requests "${BASELINE_REQUESTS:-120}" \ + --concurrency 4 \ + --rate "${BASELINE_LOAD_RATE:-10}" \ + --wait-ready \ + --ready-timeout 120 \ + --output "$output_dir/baseline-traffic.json" >/dev/null +python3 scripts/lab6-observe.py \ + --prometheus-url "$prometheus_url" \ + --phase baseline \ + --output "$output_dir/baseline-observability.json" >/dev/null +python3 scripts/lab6-correlation.py \ + --tempo-url "$tempo_url" \ + --loki-url "$loki_url" \ + --output "$output_dir/baseline-correlation.json" \ + --require >/dev/null + +if [ "$profile" = "full" ]; then + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh +fi + +arguments= +if [ "$profile" = "full" ]; then + arguments="--require-incidents" + if [ "${REQUIRE_ALERT_TRANSITIONS:-1}" = "1" ]; then + arguments="$arguments --require-alerts" + fi +fi +# shellcheck disable=SC2086 +python3 scripts/validate-lab6-evidence.py "$output_dir" $arguments +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/validate-lab6-evidence.py b/scripts/validate-lab6-evidence.py new file mode 100755 index 0000000..733ad02 --- /dev/null +++ b/scripts/validate-lab6-evidence.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path + + +def load(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def check(condition, message, errors): + if not condition: + errors.append(message) + + +def main(): + parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence") + parser.add_argument("directory") + parser.add_argument("--require-incidents", action="store_true") + parser.add_argument("--require-alerts", action="store_true") + args = parser.parse_args() + + root = Path(args.directory) + errors = [] + baseline = root / "baseline-traffic.json" + check(baseline.exists(), "baseline-traffic.json is missing", errors) + if baseline.exists(): + report = load(baseline) + check(report.get("requests", 0) > 0, "baseline sent no requests", errors) + check(report.get("successful_requests", 0) > 0, "baseline had no successful requests", errors) + + correlation = root / "baseline-correlation.json" + check(correlation.exists(), "baseline-correlation.json is missing", errors) + if correlation.exists(): + check(load(correlation).get("complete") is True, "metrics/log/trace correlation is incomplete", errors) + + observability = root / "baseline-observability.json" + check(observability.exists(), "baseline-observability.json is missing", errors) + if observability.exists(): + firing = [item.get("name") for item in load(observability).get("alerts", {}).get("items", []) if item.get("state") == "firing"] + check(not firing, "baseline has firing alerts: " + ", ".join(firing), errors) + + summaries = list(root.rglob("summary.json")) + if args.require_incidents: + names = {load(path).get("incident") for path in summaries} + for required in ("worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"): + check(required in names, "incident evidence is missing: " + required, errors) + + for summary_path in summaries: + summary = load(summary_path) + incident = summary.get("incident") + traffic_path = summary_path.parent / "traffic.json" + check(traffic_path.exists(), str(traffic_path) + " is missing", errors) + if not traffic_path.exists(): + continue + traffic = load(traffic_path) + check(traffic.get("requests", 0) > 0, incident + " sent no traffic", errors) + if incident == "worker_unavailable": + terminal = traffic.get("attachment_terminal_states") or {} + check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors) + check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not drain every accepted attachment", errors) + check(terminal.get("failed", 0) == 0, incident + " produced failed attachments", errors) + check(terminal.get("timeout", 0) == 0, incident + " left attachment jobs non-terminal", errors) + elif incident == "postgres_unavailable": + statuses = traffic.get("status_counts", {}) + failures = sum(count for status, count in statuses.items() if not status.startswith("2")) + check(failures > 0, incident + " produced no observable failed requests", errors) + correlation_path = summary_path.parent / "database-correlation.json" + check(correlation_path.exists(), str(correlation_path) + " is missing", errors) + if correlation_path.exists(): + check(load(correlation_path).get("complete") is True, "database error trace/log correlation is incomplete", errors) + elif incident == "telemetry_backends_unavailable": + terminal = traffic.get("attachment_terminal_states") or {} + check(traffic.get("successful_requests", 0) > 0, incident + " had no successful business requests", errors) + check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors) + check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors) + check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors) + check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors) + + if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"): + for name in ("alert-firing.json", "alert-resolved.json"): + path = summary_path.parent / name + check(path.exists(), str(path) + " is missing", errors) + if path.exists(): + check(load(path).get("observed") is True, str(path) + " did not observe the transition", errors) + if incident == "worker_unavailable" and summary.get("profile") == "long": + for name in ("burn-fast-firing.json", "burn-slow-firing.json"): + path = summary_path.parent / name + check(path.exists(), str(path) + " is missing", errors) + if path.exists(): + check(load(path).get("observed") is True, str(path) + " did not observe attachment burn", errors) + + result = {"valid": not errors, "errors": errors, "incident_summaries": len(summaries)} + print(json.dumps(result, ensure_ascii=False, indent=2)) + if errors: + raise SystemExit(1) + + +if __name__ == "__main__": + main() -- 2.54.0 From 0b237618c07a4912ffcc763f44fcff3fe7a597cc Mon Sep 17 00:00:00 2001 From: dizi Date: Sun, 1 Nov 2026 17:06:39 +0300 Subject: [PATCH 4/8] docs: document Lab 6 results --- REPORT.md | 260 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 240 insertions(+), 20 deletions(-) diff --git a/REPORT.md b/REPORT.md index b97ae42..38511cb 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,257 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Пластинин Денис, БВТ2303 +- Целевая оценка: 5 +- Выбранный трек: 2 — SLO и обнаружение инцидентов ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Desktop с Compose v2, Go из `go.mod`, Python 3.9+ и `curl`. Все образы закреплены в Compose или Dockerfile. Основной observability stack: OpenTelemetry Collector `0.160.0`, Prometheus `3.13.3`, Loki `3.7.9`, Tempo `2.10.10`, Grafana `13.2.4`. -## Архитектура и решения +```bash +cp lab6.env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180 +make test +make test-lab5-acceptance +make test-lab5 +make test-lab6-smoke +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Интерфейсы локального стенда: + +- API: `http://127.0.0.1:8080`; +- Swagger: `http://127.0.0.1:8081`; +- Grafana: `http://127.0.0.1:3000`; +- Prometheus: `http://127.0.0.1:9090`; +- Loki API: `http://127.0.0.1:3100`; +- Tempo API: `http://127.0.0.1:3200`. + +Безопасная остановка сохраняет volumes: + +```bash +docker compose --env-file .env down +``` + +Намеренное удаление всех учебных данных выполняется отдельно: + +```bash +docker compose --env-file .env down -v +``` + +## Архитектура + +Возможности Lab 5 сохранены: две API-реплики за Nginx, PostgreSQL как источник бизнес-состояния, Valkey для сессий, MinIO/S3 для изображений, transactional outbox, Redis Streams и отдельный worker. Lab 6 добавляет единый telemetry pipeline. + +```text +Client → Nginx → API replicas → PostgreSQL / Valkey / MinIO + │ +PostgreSQL outbox → Dispatcher → Redis Streams → Worker → PostgreSQL / MinIO + │ │ + └──────── OTLP ──────────┤ +API ────────────────────────────── OTLP ──────────┤ + ▼ + OpenTelemetry Collector + ├─ metrics → Prometheus + ├─ logs → Loki + └─ traces → Tempo + │ + ▼ + Grafana +``` + +Приложения экспортируют OTLP только в Collector. Они не знают адреса Loki и Tempo. Collector выполняет `memory_limiter` до `batch`, ограничивает очереди exporter и прекращает retry через 30 секунд. Недоступность telemetry backend поэтому не блокирует бизнес-запросы и не вызывает неограниченный рост памяти. + +Prometheus, Loki, Tempo и Grafana используют отдельные volumes. Retention локальных logs/traces — 7 суток; Prometheus хранит 7 суток. Collector является транспортом и процессором, а не долговременным хранилищем. + +### Resource attributes и среды + +Каждый процесс объявляет: + +```text +service.name = messenger-api | messenger-dispatcher | messenger-worker +service.version = значение SERVICE_VERSION +deployment.environment.name = lab +service.instance.id = стабильный ID процесса +``` + +В учебном Compose одна среда `lab`. Атрибут среды всё равно передаётся: один и тот же dashboard и TraceQL остаются пригодны для `test`/`production`, если стенд будет развёрнут там позднее. Среда и service name имеют низкую кардинальность. + +### Middleware и сквозной контекст + +HTTP middleware создаёт server span, нормализует route template, считает request counter/histogram и добавляет в JSON-лог `request_id`, `trace_id` и `span_id`. Реальный UUID из URL не становится label. + +При upload текущий W3C context сохраняется в outbox envelope как `traceparent`/`tracestate`. Dispatcher создаёт producer span. Worker извлекает context и создаёт consumer span. Поэтому один trace показывает цепочку: + +```text +HTTP upload → staging S3 → PostgreSQL/outbox → Redis publish +→ Redis consume → image transform → S3 variants → PostgreSQL ready +``` + +Идентификаторы пользователя, чата, сообщения, вложения, filename, raw URL и request ID запрещены в metric labels. Ограниченные технические ID допустимы в logs/spans синтетического стенда для расследования. `Authorization`, session token, пароль, тело сообщения, байты изображения, S3 credentials и signed URL не экспортируются. + +## Хранилища и назначение сигналов + +| Сигнал | Хранилище | На какой вопрос отвечает | +| --- | --- | --- | +| Метрика | Prometheus TSDB | Есть ли массовая деградация, когда началась и каков масштаб | +| Лог | Loki | Какое событие и ошибка произошли в конкретном процессе | +| Trace | Tempo | На каком участке сквозного запроса потрачено время или возникла ошибка | +| Dashboard / Explore | Grafana | Единый поиск, корреляция и переходы между тремя сигналами | + +Grafana provisioning создаёт Prometheus, Loki и Tempo без ручной настройки. Tempo datasource включает поиск TraceQL, node graph, service map, переходы trace → logs и trace → metrics. Loki derived field открывает trace по `trace_id`; Prometheus exemplars ведут из histogram в Tempo. + +## Карта сигналов + +| Контрактный сигнал | Реализация и единица | +| --- | --- | +| HTTP requests | `messenger_http_requests_total{method,route,status_class}`, requests | +| HTTP duration | `messenger_http_request_duration_seconds_bucket`, seconds | +| Jobs processed / failed | `messenger_attachment_worker_jobs_total{job_type,result}`, jobs | +| Job duration | `messenger_attachment_worker_job_duration_seconds_bucket`, seconds | +| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket`, seconds | +| Queue states | `messenger_attachment_jobs_count{status}`, jobs | +| Oldest job | `messenger_attachment_jobs_oldest_age_seconds{status}`, seconds | +| Worker saturation | `messenger_attachment_worker_jobs_in_flight` / `messenger_attachment_worker_capacity`, jobs | +| Outbox | `messenger_outbox_pending`, events | +| Logs | OTLP log records: timestamp, severity, service, event, trace/span IDs | +| Traces | HTTP server, producer, consumer, PostgreSQL, Redis and S3 spans | + +p50/p95/p99 вычисляются `histogram_quantile` из суммы bucket всех реплик. Перцентили реплик не усредняются. Queue depth и oldest age показываются отдельно: количество элементов не измеряет длительность ожидания. + +## SLO + +### Доступность API + +SLI — доля бизнес-запросов без `5xx`. Health и `/metrics` исключены. Ожидаемые клиентские `4xx` не считаются отказом сервиса. + +```text +bad = business responses 5xx +total = all business responses +SLI = 1 - bad / total +SLO = 99.5% за 30 дней +error budget = 0.5% +``` + +Цель 99,5% выбрана для учебного single-host стенда без multi-zone HA: она требует реагировать на деградацию, но не заявляет недостижимую для локального Compose доступность. + +### Время готовности вложений + +SLI — доля завершённых attachment jobs, достигших `ready` не позднее 30 секунд. `failed` и ready позже 30 секунд являются bad events. + +```text +bad = failed + ready later than 30s +total = ready + failed +SLI = 1 - bad / total +SLO = 99% за 30 дней +error budget = 1% +``` + +30 секунд отделяют нормальную асинхронную обработку от видимой пользователю задержки и существенно превышают обычное время небольшого синтетического изображения. + +### Multi-window burn-rate alerts + +Для обоих SLO используются две пары окон: + +| Alert | Burn rate | Короткое окно | Длинное окно | Назначение | +| --- | ---: | ---: | ---: | --- | +| Fast | 14,4× | 5 минут | 1 час | быстро обнаружить сильный инцидент | +| Slow | 6× | 30 минут | 6 часов | обнаружить устойчивую деградацию | + +Условие должно выполняться в обоих окнах пары. Короткое окно быстро реагирует, длинное отсекает одиночный шум. Отдельные operational alerts контролируют отсутствие worker, рост oldest queued age и Redis Stream lag. + +## Dashboard и поиск + +Dashboard **Messenger Lab 6 overview** содержит: + +- API RPS, error ratio, p50/p95/p99; +- состояния API/worker/dispatcher и DB pool; +- accepted/ready/failed jobs, processing duration и end-to-end ready time; +- queue depth, oldest age, Redis lag/PEL, outbox pending; +- worker busy/capacity; +- SLO и firing alerts; +- Collector export failures и переходы к Explore. + +Поиск trace выполняется в Grafana Explore → Tempo по `service.name`, span name, route, status и duration. Из trace доступен переход в Loki по тому же `trace_id`. Пошаговые процедуры: + +- `monitoring/runbooks/api-availability.md`; +- `monitoring/runbooks/attachment-pipeline.md`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Проверяемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Offline contract | `make check` | структура и обязательные файлы не ослаблены | stdout CI | +| OpenAPI | `make validate` | OpenAPI 3.1 проходит полную проверку | stdout CI | +| Публичный smoke | `make test` | бизнес-API предыдущих лабораторных работает | stdout CI | +| Lab 5 regression | `make test-lab5-acceptance && make test-lab5` | durable jobs, retry, replay и idempotency сохранены | stdout CI | +| Три telemetry signal | `make test-lab6-smoke` | нагрузка создаёт metrics/logs/traces, attachments доходят до terminal state | `evidence/runtime/.../baseline-*.json` | +| Worker incident | `scripts/incident-worker.sh short` | alert firing, после запуска worker backlog дренирован и alert resolved | `worker-short-*/` | +| DB incident | `scripts/incident-database.sh` | видны 5xx и DB error spans, burn-rate firing, API восстанавливается, оперативный alert resolved | `database-*/` | +| Длительная деградация | `scripts/incident-worker.sh long` | растут oldest age/lag, срабатывает backlog alert, все jobs затем ready | `worker-long-*/` | +| Telemetry outage | `scripts/incident-telemetry.sh` | API readiness остаётся 200, Collector queue/retry ограничены, backend восстанавливается | `telemetry-*/` | +| Полный трек 2 | `make test-lab6` | выполнены short/long incidents, firing/resolved и schema validation | `evidence/runtime/lab6-*/` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Runtime evidence создаётся локально и игнорируется Git до проверки содержимого. В нём нет session tokens, cookies, passwords, payload envelope и пользовательских текстов. Для публикации выбираются только небольшие агрегированные JSON. -## Эксперименты и отказы +## Методика экспериментов -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +`scripts/lab6-traffic.py` создаёт изолированных синтетических пользователей и чат, затем генерирует сообщения и PNG-вложения. Нагрузка closed-loop, concurrency и duration фиксируются в JSON; HTTP status и p50/p95/p99 рассчитываются клиентом только как вспомогательные данные. Серверные SLO используют Prometheus histograms. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Полный сценарий: -## Ограничения и следующие шаги +```bash +make test-lab6 +``` -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Он выполняет: + +1. baseline mixed load; +2. короткую остановку worker; +3. краткую недоступность PostgreSQL под message traffic; +4. длительную остановку worker; +5. остановку Loki и Tempo под mixed traffic; +6. проверку evidence и переходов alert `firing → resolved`. + +Каждый incident script устанавливает trap и восстанавливает затронутые сервисы. `docker compose down -v` не используется. Baseline/degraded/recovered snapshots содержат одни и те же PromQL, поэтому сравнение не зависит от ручного выбора панели. + +Гипотезы трека: + +- короткий сильный отказ расходует бюджет быстрее 14,4× и обнаруживается fast alert; +- длительная деградация выявляется ростом oldest queued age и multi-window SLO; +- после восстановления worker backlog уменьшается, все принятые jobs достигают terminal state, alerts переходят в resolved; +- отказ Loki/Tempo не влияет на бизнес-ответы, а ограниченные очереди Collector отражают потери telemetry без неограниченного роста RAM. + +Фактические времена обнаружения и восстановления берутся из `started_at`, `alert-firing.json`, `recovery_started_at` и `alert-resolved.json`. Отсутствие перехода считается падением сценария, а не успешным экспериментом. Ложное срабатывание проверяется baseline snapshot: до fault operational и burn-rate alerts не должны находиться в `firing`. + +## Результаты прогона + +Полный прогон `2026-11-01` сохранён локально в `evidence/runtime/lab6-20261101T135321Z` и прошёл schema validation без ошибок. Обезличенная сводка включена в `evidence/lab6-results.json`. Время обнаружения является верхней границей: сценарий опрашивает alert после выдержки заданной длительности отказа. + +| Инцидент | Наблюдение | Обнаружение | Восстановление | +| --- | --- | ---: | ---: | +| Worker, короткий | `AttachmentWorkerUnavailable` firing, очередь затем обработана | не более 76 с | alert resolved через 15 с, полное наблюдение 36 с | +| PostgreSQL | operational 5xx и fast burn-rate firing, 4 DB error traces связаны с логами | не более 76 с | operational alert resolved через 127 с, API стабилен через 148 с | +| Worker, длительный | oldest queued age рос; backlog, fast и slow burn alerts firing | не более 180 с | alert resolved через 15 с, backlog обработан за 65 с | +| Loki и Tempo | Collector использовал ограниченные retry/queue, бизнес-путь продолжил работу | 30 с отказа | `/health/ready` вернул 200, backends healthy через 31 с | + +Перед первым fault список firing alerts был пуст. Burn-rate alerts после инцидента закономерно остаются активными дольше оперативных alerts: их длинные окна сохраняют уже потраченный error budget. Это не считается ложным срабатыванием следующей фазы. + +## Ограничения + +- Стенд single-host и не проверяет потерю всего Docker host или volume. +- Loki и Tempo работают в single-binary режиме без репликации. +- 30-дневный SLO определяется production-подобными окнами, но учебный эксперимент создаёт искусственно высокий error ratio и не заменяет месячное наблюдение. +- Closed-loop generator подвержен coordinated omission; capacity conclusions по его RPS не делаются. +- Остановка telemetry backend проверяет ограниченное поведение exporter, но не гарантирует отсутствие потерь: после заполнения bounded queue часть telemetry намеренно отбрасывается. +- Внешняя доставка alerts не настраивается; проверяются локальные состояния Prometheus `firing` и `resolved`. + +## Источники + +ИИ-ассистент использовался при подготовке кода, конфигурации и текста; результат проверен тестами, CI-сценариями и воспроизводимым fault-прогоном. + +- OpenTelemetry Collector configuration и Go SDK; +- W3C Trace Context; +- Prometheus histograms и multi-window burn-rate alerts; +- Grafana provisioning, Loki OTLP ingestion и Tempo TraceQL; +- документация Redis Streams, PostgreSQL, MinIO и Docker Compose. -- 2.54.0 From f23855f78605d30865ced3bb6265e4c95b0692c6 Mon Sep 17 00:00:00 2001 From: dizi Date: Wed, 4 Nov 2026 20:56:35 +0300 Subject: [PATCH 5/8] fix: persist attachment SLO outcomes --- .../messenger/attachment/pipeline/metrics.go | 13 +- .../messenger/attachment/worker/worker.go | 5 +- internal/postgres/attachment_jobs.go | 38 ++-- internal/postgres/attachment_recovery.go | 11 +- internal/postgres/store_integration_test.go | 214 ++++++++++++++++++ .../0008_attachment_terminal_events.sql | 33 +++ monitoring/postgres-queries.yml | 58 +++++ monitoring/rules/messenger.yml | 32 +-- monitoring/runbooks/attachment-pipeline.md | 5 +- 9 files changed, 357 insertions(+), 52 deletions(-) create mode 100644 migrations/0008_attachment_terminal_events.sql diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go index 07a91a4..3bc4ef7 100644 --- a/internal/messenger/attachment/pipeline/metrics.go +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -17,7 +17,6 @@ type Metrics struct { workerInFlight metric.Int64UpDownCounter workerJobs metric.Int64Counter workerDuration metric.Float64Histogram - workerTimeToReady metric.Float64Histogram workerRetries metric.Int64Counter workerReclaimed metric.Int64Counter outboxPublished metric.Int64Counter @@ -39,10 +38,6 @@ func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *M metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(workerDurationBuckets...), ) - workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready", - metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(workerDurationBuckets...), - ) workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt.")) workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout.")) outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams.")) @@ -59,8 +54,8 @@ func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *M } return &Metrics{ meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs, - workerDuration: workerDuration, workerTimeToReady: workerTimeToReady, - workerRetries: workerRetries, workerReclaimed: workerReclaimed, + workerDuration: workerDuration, + workerRetries: workerRetries, workerReclaimed: workerReclaimed, outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure, outboxRecovered: outboxRecovered, handler: handler, } @@ -81,10 +76,6 @@ func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration t m.workerDuration.Record(ctx, duration.Seconds(), attributes) } -func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) { - m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process"))) -} - func (m *Metrics) ObserveRetry(ctx context.Context) { m.workerRetries.Add(ctx, 1) } diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go index 45ff600..8b88c84 100644 --- a/internal/messenger/attachment/worker/worker.go +++ b/internal/messenger/attachment/worker/worker.go @@ -58,7 +58,6 @@ type ImageProcessor interface { type Metrics interface { ObserveStarted(context.Context) ObserveFinished(context.Context, string, time.Duration) - ObserveTimeToReady(context.Context, time.Duration) ObserveRetry(context.Context) ObserveReclaimed(context.Context, int) } @@ -290,7 +289,6 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return err } result = "ready" - w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt)) return nil } @@ -350,7 +348,7 @@ func (w *Worker) resolveClaim( if job.ErrorCode != nil { code = *job.ErrorCode } - return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + return "duplicate_failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) default: return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) } @@ -538,6 +536,5 @@ type nopMetrics struct{} func (nopMetrics) ObserveStarted(context.Context) {} func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {} -func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {} func (nopMetrics) ObserveRetry(context.Context) {} func (nopMetrics) ObserveReclaimed(context.Context, int) {} diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go index 6c959de..0442e4d 100644 --- a/internal/postgres/attachment_jobs.go +++ b/internal/postgres/attachment_jobs.go @@ -37,9 +37,6 @@ func (s *Store) QueueAttachmentWithJobAndOutbox( return model.Attachment{}, model.ErrInvalidInput } now := time.Now().UTC() - if job.CreatedAt.IsZero() { - job.CreatedAt = now - } if job.AvailableAt.IsZero() { job.AvailableAt = now } @@ -67,8 +64,7 @@ func (s *Store) QueueAttachmentWithJobAndOutbox( job.AttachmentID = attachmentID job.Status = model.AttachmentJobQueued - job.UpdatedAt = job.CreatedAt - if err := insertAttachmentJob(ctx, tx, job); err != nil { + if err := insertAttachmentJob(ctx, tx, &job); err != nil { return model.Attachment{}, err } if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { @@ -92,7 +88,6 @@ func (s *Store) QueueAttachmentPipelineVersion( } job.Attempt = 1 now := time.Now().UTC() - job.CreatedAt = now job.AvailableAt = now tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) @@ -114,7 +109,6 @@ func (s *Store) QueueAttachmentPipelineVersion( } job.AttachmentID = attachment.ID job.Status = model.AttachmentJobQueued - job.UpdatedAt = now const queueAttachment = ` UPDATE attachments SET status = 'queued', pipeline_version = $2, staging_key = object_key, @@ -124,7 +118,7 @@ func (s *Store) QueueAttachmentPipelineVersion( if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil { return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err) } - if err := insertAttachmentJob(ctx, tx, job); err != nil { + if err := insertAttachmentJob(ctx, tx, &job); err != nil { return model.Attachment{}, err } if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { @@ -163,15 +157,16 @@ func (s *Store) alreadyQueuedAttachment( return publicAttachment(attachment), nil } -func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error { +func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job *model.AttachmentJob) error { const query = ` INSERT INTO attachment_jobs ( id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at ) - VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)` - if _, err := tx.Exec( - ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt, - ); err != nil { + VALUES ($1, $2, $3, $4, 'queued', $5, statement_timestamp(), statement_timestamp()) + RETURNING created_at, updated_at` + if err := tx.QueryRow( + ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, + ).Scan(&job.CreatedAt, &job.UpdatedAt); err != nil { return fmt.Errorf("insert attachment job: %w", err) } return nil @@ -417,6 +412,9 @@ func (s *Store) CompleteAttachmentJob( if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil { return "", fmt.Errorf("complete attachment: %w", err) } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobReady); err != nil { + return "", err + } if err := tx.Commit(ctx); err != nil { return "", fmt.Errorf("commit completed attachment job: %w", err) } @@ -527,12 +525,26 @@ func (s *Store) FailAttachmentJob( if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil { return "", fmt.Errorf("fail attachment: %w", err) } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil { + return "", err + } if err := tx.Commit(ctx); err != nil { return "", fmt.Errorf("commit failed attachment job: %w", err) } return model.JobMutationApplied, nil } +func insertAttachmentTerminalEvent(ctx context.Context, tx pgx.Tx, job model.AttachmentJob, status string) error { + const query = ` + INSERT INTO attachment_terminal_events (job_id, status, accepted_at) + VALUES ($1, $2, $3) + ON CONFLICT (job_id) DO NOTHING` + if _, err := tx.Exec(ctx, query, job.ID, status, job.CreatedAt); err != nil { + return fmt.Errorf("insert attachment terminal event: %w", err) + } + return nil +} + func lockJobMutation( ctx context.Context, tx pgx.Tx, diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go index e305a79..388dafc 100644 --- a/internal/postgres/attachment_recovery.go +++ b/internal/postgres/attachment_recovery.go @@ -89,9 +89,7 @@ func (s *Store) RecoverExpiredAttachmentJobs( if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil { return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err) } - eventJob := job - eventJob.CreatedAt = time.Now().UTC() - if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil { return model.RecoveryResult{}, err } recovered.Requeued++ @@ -118,6 +116,9 @@ func markExpiredJobFailed( if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil { return fmt.Errorf("fail expired attachment job: %w", err) } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil { + return err + } if !failAttachment { return nil } @@ -198,9 +199,7 @@ func (s *Store) ReplayFailedAttachment( ); err != nil { return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err) } - eventJob := job - eventJob.CreatedAt = time.Now().UTC() - if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil { return model.AttachmentJob{}, err } if err := tx.Commit(ctx); err != nil { diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index e4184db..e71ab6b 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/suite" @@ -19,6 +20,7 @@ import ( "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" ) type PostgresSuite struct { @@ -210,6 +212,7 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants) s.Require().NoError(err) s.Equal(model.JobMutationApplied, result) + s.assertTerminalEvent(job.ID, model.AttachmentJobReady, lease.Job.CreatedAt) result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) s.Require().NoError(err) s.Equal(model.JobMutationAlreadyComplete, result) @@ -252,6 +255,130 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { s.Equal(model.JobMutationStale, result) } +func (s *PostgresSuite) TestAttachmentJobCreationUsesDatabaseClock() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/original" + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: owner.ID, Filename: "future.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("f", 64), ObjectKey: stagingKey, + }) + s.Require().NoError(err) + future := time.Now().Add(24 * time.Hour) + job := model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, + Attempt: 1, CreatedAt: future, + } + _, err = s.store.QueueAttachmentWithJobAndOutbox(context.Background(), pending.ID, stagingKey, job, "", "") + s.Require().NoError(err) + stored, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.True(stored.CreatedAt.Before(future)) + + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachmentID, 1, 1, "worker-clock", time.Minute, + ) + s.Require().NoError(err) + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachmentID, 1)) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + s.assertTerminalEvent(job.ID, model.AttachmentJobReady, stored.CreatedAt) + s.assertOutboxCreationTime(job.ID, 1, stored.CreatedAt) +} + +func (s *PostgresSuite) TestTerminalLedgerMigrationBackfillsExistingJob() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + replayedAttachment, replayedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + skewedAttachment, skewedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + replayedAcceptedAt := time.Now().Add(-2 * time.Minute).UTC().Truncate(time.Microsecond) + firstFailureAt := replayedAcceptedAt.Add(20 * time.Second) + replayedReadyAt := firstFailureAt.Add(time.Minute) + skewedTerminalAt := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond) + skewedCreatedAt := skewedTerminalAt.Add(time.Minute) + tx, err := s.pool.Begin(context.Background()) + s.Require().NoError(err) + defer func() { s.Require().NoError(tx.Rollback(context.Background())) }() + _, err = tx.Exec( + context.Background(), + `UPDATE attachment_jobs SET status = 'ready', attempt = 2, error_code = NULL, created_at = $2, updated_at = $3 + WHERE id = $1`, + replayedJob.ID, + replayedAcceptedAt, + replayedReadyAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = $2 + WHERE id = $1`, + replayedAttachment.ID, + replayedReadyAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `INSERT INTO attachment_replay_audit ( + id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason, created_at + ) VALUES ($1, $2, $3, 1, 2, 'legacy-operator', 'legacy replay', $4)`, + uuid.NewString(), + replayedJob.ID, + replayedAttachment.ID, + firstFailureAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachment_jobs SET status = 'failed', error_code = 'legacy_failure', created_at = $2, updated_at = $3 + WHERE id = $1`, + skewedJob.ID, + skewedCreatedAt, + skewedTerminalAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachments SET status = 'failed', error_code = 'legacy_failure', ready_at = NULL + WHERE id = $1`, + skewedAttachment.ID, + ) + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), `ALTER INDEX attachment_terminal_events_status_terminal_at_idx RENAME TO attachment_terminal_events_status_terminal_at_idx_old`) + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), `ALTER TABLE attachment_terminal_events RENAME TO attachment_terminal_events_old`) + s.Require().NoError(err) + migration, err := migrations.FS.ReadFile("0008_attachment_terminal_events.sql") + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), string(migration), pgx.QueryExecModeSimpleProtocol) + s.Require().NoError(err) + + s.assertTerminalEventInTx(tx, replayedJob.ID, model.AttachmentJobFailed, replayedAcceptedAt, firstFailureAt) + s.assertTerminalEventInTx(tx, skewedJob.ID, model.AttachmentJobFailed, skewedTerminalAt, skewedTerminalAt) +} + +func (s *PostgresSuite) assertTerminalEventInTx( + tx pgx.Tx, + jobID string, + status string, + acceptedAt time.Time, + terminalAt time.Time, +) { + s.T().Helper() + var ( + actualStatus string + actualAcceptedAt time.Time + actualTerminalAt time.Time + ) + err := tx.QueryRow( + context.Background(), + `SELECT status, accepted_at, terminal_at FROM attachment_terminal_events WHERE job_id = $1`, + jobID, + ).Scan(&actualStatus, &actualAcceptedAt, &actualTerminalAt) + s.Require().NoError(err) + s.Equal(status, actualStatus) + s.True(acceptedAt.Equal(actualAcceptedAt)) + s.True(terminalAt.Equal(actualTerminalAt)) +} + func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { owner, chatID := createOwnerAndChat(s.T(), s.store) _, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) @@ -295,6 +422,8 @@ func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { owner, chatID := createOwnerAndChat(s.T(), s.store) attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) lease, err := s.store.ClaimAttachmentJob( context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, ) @@ -302,6 +431,7 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image") s.Require().NoError(err) s.Equal(model.JobMutationApplied, result) + firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt) _, _, err = s.store.CreateMessage( context.Background(), chatID, owner.ID, "failed attachment", @@ -320,6 +450,18 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { s.Require().NoError(err) s.Equal(2, replayed.Attempt) s.Equal(model.AttachmentJobQueued, replayed.Status) + s.Equal(original.CreatedAt, replayed.CreatedAt) + s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt) + replayedLease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, replayed.PipelineVersion, replayed.Attempt, "worker-2", time.Minute, + ) + s.Require().NoError(err) + result, err = s.store.CompleteAttachmentJob( + context.Background(), replayedLease, completedVariants(attachment.ID, replayed.PipelineVersion), + ) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)) var auditCount int err = s.pool.QueryRow( @@ -336,6 +478,8 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, ) s.Require().NoError(err) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) _, err = s.pool.Exec( context.Background(), `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, @@ -351,12 +495,82 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { s.Require().NoError(err) s.Equal(2, current.Attempt) s.Equal(model.AttachmentJobQueued, current.Status) + s.Equal(original.CreatedAt, current.CreatedAt) + s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt) result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1)) s.Require().NoError(err) s.Equal(model.JobMutationStale, result) } +func (s *PostgresSuite) TestExpiredLeaseRecoveryFailsExhaustedJobOnce() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + _, err = s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + _, err = s.pool.Exec( + context.Background(), + `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, + job.ID, + ) + s.Require().NoError(err) + + recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now()) + s.Require().NoError(err) + s.Zero(recovered.Requeued) + s.Equal(1, recovered.Failed) + firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt) + + recovered, err = s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now()) + s.Require().NoError(err) + s.Zero(recovered.Requeued) + s.Zero(recovered.Failed) + current, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentJobFailed, current.Status) + s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)) +} + +func (s *PostgresSuite) assertTerminalEvent(jobID string, status string, acceptedAt time.Time) time.Time { + s.T().Helper() + var ( + actualStatus string + actualAcceptedAt time.Time + terminalAt time.Time + count int + ) + err := s.pool.QueryRow( + context.Background(), + `SELECT min(status), min(accepted_at), min(terminal_at), count(*) + FROM attachment_terminal_events WHERE job_id = $1`, + jobID, + ).Scan(&actualStatus, &actualAcceptedAt, &terminalAt, &count) + s.Require().NoError(err) + s.Equal(1, count) + s.Equal(status, actualStatus) + s.True(acceptedAt.Equal(actualAcceptedAt), "expected %s, got %s", acceptedAt, actualAcceptedAt) + return terminalAt +} + +func (s *PostgresSuite) assertOutboxCreationTime(jobID string, attempt int, expected time.Time) { + s.T().Helper() + var payload []byte + err := s.pool.QueryRow( + context.Background(), + `SELECT payload FROM outbox_events WHERE job_id = $1 AND attempt = $2`, + jobID, + attempt, + ).Scan(&payload) + s.Require().NoError(err) + var message model.AttachmentJobMessage + s.Require().NoError(json.Unmarshal(payload, &message)) + s.True(expected.Equal(message.CreatedAt), "expected %s, got %s", expected, message.CreatedAt) +} + func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() { owner, chatID := createOwnerAndChat(s.T(), s.store) attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) diff --git a/migrations/0008_attachment_terminal_events.sql b/migrations/0008_attachment_terminal_events.sql new file mode 100644 index 0000000..53f17ce --- /dev/null +++ b/migrations/0008_attachment_terminal_events.sql @@ -0,0 +1,33 @@ +CREATE TABLE attachment_terminal_events ( + job_id UUID PRIMARY KEY, + status TEXT NOT NULL, + accepted_at TIMESTAMPTZ NOT NULL, + terminal_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_terminal_events_status CHECK (status IN ('ready', 'failed')), + CONSTRAINT attachment_terminal_events_time CHECK (terminal_at >= accepted_at) +); + +CREATE INDEX attachment_terminal_events_status_terminal_at_idx + ON attachment_terminal_events (status, terminal_at); + +WITH first_replay AS ( + SELECT job_id, MIN(created_at) AS created_at + FROM attachment_replay_audit + GROUP BY job_id +) +INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at) +SELECT jobs.id, 'failed', LEAST(jobs.created_at, first_replay.created_at), first_replay.created_at +FROM attachment_jobs jobs +JOIN first_replay ON first_replay.job_id = jobs.id +ON CONFLICT (job_id) DO NOTHING; + +INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at) +SELECT jobs.id, jobs.status, LEAST(jobs.created_at, jobs.updated_at), jobs.updated_at +FROM attachment_jobs jobs +WHERE jobs.status IN ('ready', 'failed') + AND NOT EXISTS ( + SELECT 1 + FROM attachment_replay_audit replay + WHERE replay.job_id = jobs.id + ) +ON CONFLICT (job_id) DO NOTHING; diff --git a/monitoring/postgres-queries.yml b/monitoring/postgres-queries.yml index 7666281..52fcfe1 100644 --- a/monitoring/postgres-queries.yml +++ b/monitoring/postgres-queries.yml @@ -37,3 +37,61 @@ messenger_outbox: - oldest_pending_age_seconds: usage: GAUGE description: Age of the oldest unpublished outbox event. + +messenger_attachment_terminal_transitions: + query: | + WITH statuses(status) AS ( + VALUES ('ready'), ('failed') + ) + SELECT + statuses.status, + COUNT(events.job_id)::bigint AS total + FROM statuses + LEFT JOIN attachment_terminal_events events ON events.status = statuses.status + GROUP BY statuses.status + metrics: + - status: + usage: LABEL + description: First terminal status recorded for a durable attachment job. + - total: + usage: COUNTER + description: Durable attachment jobs whose first terminal outcome has this status. + +messenger_attachment_time_to_ready_seconds: + query: | + WITH bounds(le, upper_bound) AS ( + VALUES + ('0.05', 0.05::double precision), + ('0.1', 0.1::double precision), + ('0.25', 0.25::double precision), + ('0.5', 0.5::double precision), + ('1', 1::double precision), + ('2.5', 2.5::double precision), + ('5', 5::double precision), + ('10', 10::double precision), + ('15', 15::double precision), + ('20', 20::double precision), + ('30', 30::double precision), + ('60', 60::double precision), + ('+Inf', NULL::double precision) + ) + SELECT + bounds.le, + COUNT(events.job_id) FILTER ( + WHERE events.status = 'ready' + AND ( + bounds.upper_bound IS NULL + OR EXTRACT(EPOCH FROM (events.terminal_at - events.accepted_at)) <= bounds.upper_bound + ) + )::bigint AS bucket + FROM bounds + LEFT JOIN attachment_terminal_events events ON events.status = 'ready' + GROUP BY bounds.le, bounds.upper_bound + ORDER BY bounds.upper_bound NULLS LAST + metrics: + - le: + usage: LABEL + description: Upper bound of the durable time-to-ready bucket in seconds. + - bucket: + usage: COUNTER + description: Durable attachment jobs reaching ready within the bucket bound. diff --git a/monitoring/rules/messenger.yml b/monitoring/rules/messenger.yml index 4999ddb..78a86dd 100644 --- a/monitoring/rules/messenger.yml +++ b/monitoring/rules/messenger.yml @@ -68,29 +68,29 @@ groups: expr: | ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)), 0.000001 ) > 0.144 and ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)), 0.000001 ) > 0.144 for: 30s @@ -107,29 +107,29 @@ groups: expr: | ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)), 0.000001 ) > 0.06 and ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)), 0.000001 ) > 0.06 for: 1m diff --git a/monitoring/runbooks/attachment-pipeline.md b/monitoring/runbooks/attachment-pipeline.md index c2ae007..4c7155f 100644 --- a/monitoring/runbooks/attachment-pipeline.md +++ b/monitoring/runbooks/attachment-pipeline.md @@ -2,11 +2,12 @@ ## Цель и сигналы -Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget. +Объект контроля — первый terminal outcome каждой уникальной durable job. Цель — не менее 99% jobs впервые завершаются `ready` не позднее 30 секунд после acceptance за скользящие 30 дней. Первый `failed` и первое завершение `ready` позже 30 секунд расходуют error budget; replay не переписывает уже записанный outcome. Основные сигналы: -- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker; +- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы доставок worker; +- `messenger_attachment_terminal_transitions_total{status}` — первые terminal outcomes из append-only PostgreSQL ledger; - `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки; - `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время; - `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL; -- 2.54.0 From c9155474491d0307227085ccf062092c44ef68fb Mon Sep 17 00:00:00 2001 From: dizi Date: Wed, 4 Nov 2026 20:56:35 +0300 Subject: [PATCH 6/8] fix: validate telemetry outage bounds --- infra/otel/collector.yaml | 7 ++ .../grafana/dashboards/messenger-lab6.json | 12 +-- scripts/lab6-observe.py | 7 +- scripts/validate-lab6-evidence.py | 101 ++++++++++++++++++ 4 files changed, 120 insertions(+), 7 deletions(-) diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml index e912195..b22fa0e 100644 --- a/infra/otel/collector.yaml +++ b/infra/otel/collector.yaml @@ -58,6 +58,13 @@ service: telemetry: logs: level: info + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 pipelines: metrics: receivers: [otlp] diff --git a/monitoring/grafana/dashboards/messenger-lab6.json b/monitoring/grafana/dashboards/messenger-lab6.json index f20cabe..d3b39d9 100644 --- a/monitoring/grafana/dashboards/messenger-lab6.json +++ b/monitoring/grafana/dashboards/messenger-lab6.json @@ -1010,7 +1010,7 @@ }, "targets": [ { - "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)", + "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001)", "legendFormat": "ready in 30s", "refId": "A" } @@ -1045,17 +1045,17 @@ }, "targets": [ { - "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", "legendFormat": "5m burn", "refId": "A" }, { - "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", "legendFormat": "1h burn", "refId": "B" }, { - "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", "legendFormat": "6h burn", "refId": "C" } @@ -1410,12 +1410,12 @@ }, "targets": [ { - "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m]))", "legendFormat": "{{exporter}} spans failed/s", "refId": "A" }, { - "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m]))", "legendFormat": "{{exporter}} logs failed/s", "refId": "B" }, diff --git a/scripts/lab6-observe.py b/scripts/lab6-observe.py index 4614176..6ff2489 100644 --- a/scripts/lab6-observe.py +++ b/scripts/lab6-observe.py @@ -18,7 +18,12 @@ QUERIES = { "worker_capacity": 'sum(messenger_attachment_worker_capacity)', "pending_outbox": 'sum(messenger_outbox_pending)', "worker_up": 'min(up{job="attachment-worker"})', - "collector_export_failures_5m": 'sum(increase(otelcol_exporter_send_failed_spans_total[5m])) + sum(increase(otelcol_exporter_send_failed_log_records_total[5m]))', + "collector_queue_size": 'max by (exporter) (otelcol_exporter_queue_size)', + "collector_queue_capacity": 'max by (exporter) (otelcol_exporter_queue_capacity)', + "collector_export_failures_total": 'sum by (exporter) (otelcol_exporter_send_failed_spans) or sum by (exporter) (otelcol_exporter_send_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))', + "collector_dropped_records_total": 'sum by (exporter) (otelcol_exporter_enqueue_failed_spans) or sum by (exporter) (otelcol_exporter_enqueue_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))', + "collector_memory_bytes": 'max(otelcol_process_memory_rss) or max(process_resident_memory_bytes{job="otel-collector"})', + "collector_memory_peak_bytes": 'max(max_over_time(otelcol_process_memory_rss[2m])) or max(max_over_time(process_resident_memory_bytes{job="otel-collector"}[2m]))', } diff --git a/scripts/validate-lab6-evidence.py b/scripts/validate-lab6-evidence.py index 733ad02..643a6fb 100755 --- a/scripts/validate-lab6-evidence.py +++ b/scripts/validate-lab6-evidence.py @@ -2,6 +2,7 @@ import argparse import json +import math from pathlib import Path @@ -14,6 +15,105 @@ def check(condition, message, errors): errors.append(message) +def query_scalar(snapshot, name, source, errors): + query = snapshot.get("queries", {}).get(name, {}) + check("error" not in query, source + " query failed: " + name, errors) + series = query.get("series", []) + check(len(series) == 1, source + " query must return one series: " + name, errors) + if len(series) != 1: + return None + try: + value = float(series[0]["value"]) + except (KeyError, TypeError, ValueError): + check(False, source + " query returned a non-numeric value: " + name, errors) + return None + check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors) + return value if math.isfinite(value) else None + + +def query_exporters(snapshot, name, source, errors): + expected = {"otlp_grpc/tempo", "otlp_http/loki"} + query = snapshot.get("queries", {}).get(name, {}) + check("error" not in query, source + " query failed: " + name, errors) + values = {} + for series in query.get("series", []): + exporter = series.get("labels", {}).get("exporter") + check(exporter in expected, source + " query returned unexpected exporter: " + name, errors) + check(exporter not in values, source + " query returned duplicate exporter: " + name, errors) + try: + value = float(series["value"]) + except (KeyError, TypeError, ValueError): + check(False, source + " query returned a non-numeric value: " + name, errors) + continue + check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors) + if exporter in expected and exporter not in values and math.isfinite(value): + values[exporter] = value + check(set(values) == expected, source + " query must return the exact exporter set: " + name, errors) + return values if set(values) == expected else None + + +def validate_telemetry_outage(directory, errors): + phase_names = ("baseline", "degraded", "recovered") + exporter_metric_names = ( + "collector_queue_size", + "collector_queue_capacity", + "collector_export_failures_total", + "collector_dropped_records_total", + ) + scalar_metric_names = ( + "collector_memory_bytes", + "collector_memory_peak_bytes", + ) + phases = {} + for phase in phase_names: + path = directory / (phase + ".json") + check(path.exists(), str(path) + " is missing", errors) + if not path.exists(): + return + snapshot = load(path) + phases[phase] = { + **{name: query_exporters(snapshot, name, path.name, errors) for name in exporter_metric_names}, + **{name: query_scalar(snapshot, name, path.name, errors) for name in scalar_metric_names}, + } + if any(value is None for values in phases.values() for value in values.values()): + return + + for phase, values in phases.items(): + for exporter in ("otlp_grpc/tempo", "otlp_http/loki"): + capacity = values["collector_queue_capacity"][exporter] + size = values["collector_queue_size"][exporter] + check(capacity > 0, phase + " collector queue capacity is not positive: " + exporter, errors) + check(0 <= size <= capacity, phase + " collector queue size is outside capacity: " + exporter, errors) + check(values["collector_memory_bytes"] > 0, phase + " collector memory is not positive", errors) + check(values["collector_memory_peak_bytes"] > 0, phase + " collector peak memory is not positive", errors) + + baseline = phases["baseline"] + degraded = phases["degraded"] + recovered = phases["recovered"] + for exporter in ("otlp_grpc/tempo", "otlp_http/loki"): + for name in ("collector_export_failures_total", "collector_dropped_records_total"): + check(degraded[name][exporter] >= baseline[name][exporter], name + " decreased during outage: " + exporter, errors) + check(recovered[name][exporter] >= degraded[name][exporter], name + " decreased after recovery: " + exporter, errors) + outage_observed = ( + degraded["collector_queue_size"][exporter] > baseline["collector_queue_size"][exporter] + or recovered["collector_export_failures_total"][exporter] > baseline["collector_export_failures_total"][exporter] + or recovered["collector_dropped_records_total"][exporter] > baseline["collector_dropped_records_total"][exporter] + ) + check(outage_observed, "telemetry outage was not observed for exporter: " + exporter, errors) + drained_limit = max( + baseline["collector_queue_size"][exporter], + recovered["collector_queue_capacity"][exporter] * 0.05, + ) + check( + recovered["collector_queue_size"][exporter] <= drained_limit, + "collector queue did not drain after recovery: " + exporter, + errors, + ) + memory_limit = max(baseline["collector_memory_peak_bytes"] * 1.5, baseline["collector_memory_peak_bytes"] + 64 * 1024 * 1024) + check(degraded["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory grew beyond the outage bound", errors) + check(recovered["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory remained above the recovery bound", errors) + + def main(): parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence") parser.add_argument("directory") @@ -77,6 +177,7 @@ def main(): check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors) check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors) check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors) + validate_telemetry_outage(summary_path.parent, errors) if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"): for name in ("alert-firing.json", "alert-resolved.json"): -- 2.54.0 From 323d1e9294b2035eeb5ef3f002d34c73a1d5f131 Mon Sep 17 00:00:00 2001 From: dizi Date: Wed, 4 Nov 2026 20:56:35 +0300 Subject: [PATCH 7/8] fix: repair trace diagnostics --- internal/messenger/httpapi/attachments.go | 22 +++++-------------- .../datasources/observability.yml | 4 +++- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go index 2e832d6..ad8d1df 100644 --- a/internal/messenger/httpapi/attachments.go +++ b/internal/messenger/httpapi/attachments.go @@ -6,7 +6,6 @@ import ( "mime" "net/http" "net/url" - "regexp" "strconv" "strings" "time" @@ -14,7 +13,9 @@ import ( "unicode/utf8" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" @@ -133,6 +134,9 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, content.Body); err != nil { + span := trace.SpanFromContext(r.Context()) + span.RecordError(err) + span.SetStatus(codes.Error, "response body write failed") s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err) } return nil @@ -153,22 +157,6 @@ func attachmentVariant(query url.Values) (string, bool) { } } -var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) - -func traceparent(value string) string { - if traceparentPattern.MatchString(value) { - return value - } - return "" -} - -func tracestate(value string) string { - if len(value) <= 512 { - return value - } - return "" -} - func attachmentIDFromRequest(r *http.Request) (string, *apiError) { parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") attachmentID, valid := canonicalID(parts[3]) diff --git a/monitoring/grafana/provisioning/datasources/observability.yml b/monitoring/grafana/provisioning/datasources/observability.yml index 99af68b..741e6d2 100644 --- a/monitoring/grafana/provisioning/datasources/observability.yml +++ b/monitoring/grafana/provisioning/datasources/observability.yml @@ -32,9 +32,11 @@ datasources: maxLines: 1000 derivedFields: - name: TraceID - matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})' + matcherType: label + matcherRegex: trace_id datasourceUid: tempo url: '$${__value.raw}' + urlDisplayLabel: View trace - name: Tempo type: tempo -- 2.54.0 From 8c8a0302c91408b44cb35461fd1a8e948569b9d8 Mon Sep 17 00:00:00 2001 From: dizi Date: Wed, 4 Nov 2026 20:56:35 +0300 Subject: [PATCH 8/8] fix: refresh Lab 6 evidence --- REPORT.md | 19 ++++++----- evidence/lab6-results.json | 65 +++++++++++++++++++++++++------------- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/REPORT.md b/REPORT.md index 38511cb..c2cdea2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -60,7 +60,7 @@ API ───────────────────────── Grafana ``` -Приложения экспортируют OTLP только в Collector. Они не знают адреса Loki и Tempo. Collector выполняет `memory_limiter` до `batch`, ограничивает очереди exporter и прекращает retry через 30 секунд. Недоступность telemetry backend поэтому не блокирует бизнес-запросы и не вызывает неограниченный рост памяти. +Приложения экспортируют OTLP только в Collector. Они не знают адреса Loki и Tempo. Collector выполняет `memory_limiter` до `batch`, ограничивает очереди exporter и прекращает retry через 30 секунд. Evidence проверяет queue size/capacity, failures и drops отдельно для `otlp_grpc/tempo` и `otlp_http/loki`, а память — по пиковому RSS за двухминутное окно. Недоступность telemetry backend поэтому не блокирует бизнес-запросы и не вызывает неограниченный рост памяти. Prometheus, Loki, Tempo и Grafana используют отдельные volumes. Retention локальных logs/traces — 7 суток; Prometheus хранит 7 суток. Collector является транспортом и процессором, а не долговременным хранилищем. @@ -107,7 +107,8 @@ Grafana provisioning создаёт Prometheus, Loki и Tempo без ручно | --- | --- | | HTTP requests | `messenger_http_requests_total{method,route,status_class}`, requests | | HTTP duration | `messenger_http_request_duration_seconds_bucket`, seconds | -| Jobs processed / failed | `messenger_attachment_worker_jobs_total{job_type,result}`, jobs | +| Delivery outcomes | `messenger_attachment_worker_jobs_total{job_type,result}`, deliveries | +| Terminal job transitions | `messenger_attachment_terminal_transitions_total{status}`, first durable outcome per job | | Job duration | `messenger_attachment_worker_job_duration_seconds_bucket`, seconds | | Time to ready | `messenger_attachment_time_to_ready_seconds_bucket`, seconds | | Queue states | `messenger_attachment_jobs_count{status}`, jobs | @@ -137,7 +138,7 @@ error budget = 0.5% ### Время готовности вложений -SLI — доля завершённых attachment jobs, достигших `ready` не позднее 30 секунд. `failed` и ready позже 30 секунд являются bad events. +SLI — доля уникальных durable attachment jobs, чей первый terminal outcome равен `ready` и наступил не позднее 30 секунд после acceptance. Время acceptance назначает PostgreSQL при создании job, поэтому начало и конец интервала принадлежат одному clock domain. PostgreSQL записывает первый outcome в append-only ledger в той же транзакции, что и переход job. Миграция переносит уже завершённые jobs с `accepted_at = created_at` и `terminal_at = updated_at`. Повторная delivery и ручной replay не меняют уже учтённый результат: job, впервые завершившаяся `failed`, остаётся bad event даже после успешного replay. ```text bad = failed + ready later than 30s @@ -147,6 +148,8 @@ SLO = 99% за 30 дней error budget = 1% ``` +`messenger_attachment_terminal_transitions_total` и cumulative buckets `messenger_attachment_time_to_ready_seconds_bucket` экспортируются из ledger. Bucket `le="+Inf"` является durable количеством первых outcomes `ready` и используется как знаменатель ready jobs. + 30 секунд отделяют нормальную асинхронную обработку от видимой пользователю задержки и существенно превышают обычное время небольшого синтетического изображения. ### Multi-window burn-rate alerts @@ -226,14 +229,14 @@ make test-lab6 ## Результаты прогона -Полный прогон `2026-11-01` сохранён локально в `evidence/runtime/lab6-20261101T135321Z` и прошёл schema validation без ошибок. Обезличенная сводка включена в `evidence/lab6-results.json`. Время обнаружения является верхней границей: сценарий опрашивает alert после выдержки заданной длительности отказа. +Полный прогон `2026-11-04` сохранён локально в `evidence/runtime/lab6-20261104T174049Z` и прошёл schema validation без ошибок. Обезличенная сводка включена в `evidence/lab6-results.json`. Время обнаружения является верхней границей: сценарий опрашивает alert после выдержки заданной длительности отказа. | Инцидент | Наблюдение | Обнаружение | Восстановление | | --- | --- | ---: | ---: | -| Worker, короткий | `AttachmentWorkerUnavailable` firing, очередь затем обработана | не более 76 с | alert resolved через 15 с, полное наблюдение 36 с | -| PostgreSQL | operational 5xx и fast burn-rate firing, 4 DB error traces связаны с логами | не более 76 с | operational alert resolved через 127 с, API стабилен через 148 с | -| Worker, длительный | oldest queued age рос; backlog, fast и slow burn alerts firing | не более 180 с | alert resolved через 15 с, backlog обработан за 65 с | -| Loki и Tempo | Collector использовал ограниченные retry/queue, бизнес-путь продолжил работу | 30 с отказа | `/health/ready` вернул 200, backends healthy через 31 с | +| Worker, короткий | `AttachmentWorkerUnavailable` firing, очередь затем обработана | не более 76 с | alert resolved через 13 с, полное наблюдение 33 с | +| PostgreSQL | operational 5xx и fast burn-rate firing; сохранены 3 DB error traces, один связан с API-логом | не более 78 с | operational alert resolved через 134 с, API стабилен через 154 с | +| Worker, длительный | oldest queued age рос; backlog, fast и slow burn alerts firing | не более 181 с | alert resolved через 20 с, backlog обработан за 91 с | +| Loki и Tempo | очереди выросли до 15/512 и 16/512, затем дренировались до нуля; peak RSS 226.7 MB | 30 с отказа | `/health/ready` вернул 200, backends healthy через 31 с | Перед первым fault список firing alerts был пуст. Burn-rate alerts после инцидента закономерно остаются активными дольше оперативных alerts: их длинные окна сохраняют уже потраченный error budget. Это не считается ложным срабатыванием следующей фазы. diff --git a/evidence/lab6-results.json b/evidence/lab6-results.json index 3ae353c..c09c6b9 100644 --- a/evidence/lab6-results.json +++ b/evidence/lab6-results.json @@ -1,46 +1,46 @@ { "schema_version": 1, - "captured_at": "2026-11-01T14:05:43Z", + "captured_at": "2026-11-04T17:53:52Z", "baseline": { - "api_request_rate_per_second": 1.1485206460733426, + "api_request_rate_per_second": 1.014325, "firing_alerts": 0, "traces_by_service": { - "messenger-api": 39, - "messenger-worker": 23, - "messenger-dispatcher": 22 + "messenger-api": 100, + "messenger-worker": 72, + "messenger-dispatcher": 64 }, "logs_by_service": { - "messenger-api": 162, - "messenger-worker": 40, + "messenger-api": 207, + "messenger-worker": 41, "messenger-dispatcher": 1 }, - "api_worker_shared_traces": 3, - "traces_with_correlated_logs": 37 + "api_worker_shared_traces": 22, + "traces_with_correlated_logs": 101 }, "incidents": [ { "name": "worker_unavailable_short", "injected_duration_seconds": 76, "alert_detection_upper_bound_seconds": 76, - "alert_resolution_seconds": 15, - "recovery_observation_seconds": 36 + "alert_resolution_seconds": 13, + "recovery_observation_seconds": 33 }, { "name": "postgres_unavailable", - "injected_duration_seconds": 76, - "alert_detection_upper_bound_seconds": 76, - "operational_alert_resolution_seconds": 127, - "recovery_observation_seconds": 148, + "injected_duration_seconds": 78, + "alert_detection_upper_bound_seconds": 78, + "operational_alert_resolution_seconds": 134, + "recovery_observation_seconds": 154, "fast_burn_alert_observed": true, - "database_error_traces": 4, - "database_error_traces_with_logs": 4 + "database_error_traces": 3, + "database_error_traces_with_logs": 1 }, { "name": "worker_unavailable_long", - "injected_duration_seconds": 180, - "alert_detection_upper_bound_seconds": 180, - "alert_resolution_seconds": 15, - "recovery_observation_seconds": 65, + "injected_duration_seconds": 181, + "alert_detection_upper_bound_seconds": 181, + "alert_resolution_seconds": 20, + "recovery_observation_seconds": 91, "fast_burn_alert_observed": true, "slow_burn_alert_observed": true }, @@ -48,7 +48,28 @@ "name": "telemetry_backends_unavailable", "injected_duration_seconds": 30, "api_ready_http_status": 200, - "recovery_observation_seconds": 31 + "recovery_observation_seconds": 31, + "exporters": { + "otlp_grpc/tempo": { + "capacity": 512, + "degraded_queue_size": 15, + "recovered_queue_size": 0, + "export_failures": 16, + "dropped_records": 0 + }, + "otlp_http/loki": { + "capacity": 512, + "degraded_queue_size": 16, + "recovered_queue_size": 0, + "export_failures": 8, + "dropped_records": 0 + } + }, + "collector_peak_rss_bytes": { + "baseline": 225923072, + "degraded": 226549760, + "recovered": 226746368 + } } ], "validation": { -- 2.54.0