From 934dfefee78179b52e8440af5d397e6c7d0270ef Mon Sep 17 00:00:00 2001 From: dizi Date: Sun, 1 Nov 2026 17:06:32 +0300 Subject: [PATCH] feat: add OpenTelemetry instrumentation --- cmd/lab5experiment/api_scenario.go | 2 +- cmd/lab5experiment/main.go | 18 +- cmd/messenger/main.go | 4 +- go.mod | 29 +- go.sum | 65 ++++- internal/bootstrap/migration.go | 4 +- internal/bootstrap/pipeline.go | 51 ++-- internal/bootstrap/reconciliation.go | 10 +- internal/bootstrap/replay.go | 6 +- internal/bootstrap/server.go | 26 +- internal/bootstrap/telemetry.go | 23 ++ internal/broker/redisstream/broker.go | 20 +- internal/database/database.go | 3 +- internal/database/telemetry.go | 28 ++ .../messenger/attachment/pipeline/metrics.go | 140 +++++----- internal/messenger/attachment/service.go | 20 +- internal/messenger/attachment/service_test.go | 2 +- .../messenger/attachment/worker/worker.go | 83 ++++-- .../attachment/worker/worker_test.go | 6 +- internal/messenger/httpapi/attachments.go | 19 +- internal/messenger/httpapi/auth.go | 2 +- internal/messenger/httpapi/handlers.go | 4 +- .../messenger/httpapi/messenger_interface.go | 4 +- internal/messenger/httpapi/metrics.go | 111 +++----- internal/messenger/httpapi/model.go | 2 +- internal/messenger/httpapi/server.go | 37 ++- internal/messenger/httpapi/server_test.go | 4 +- internal/objectstore/store.go | 53 +++- internal/outbox/dispatcher.go | 56 +++- internal/outbox/dispatcher_test.go | 2 +- internal/postgres/attachment_jobs.go | 8 +- internal/postgres/attachment_lease.go | 2 +- internal/postgres/attachment_recovery.go | 2 +- internal/postgres/attachments.go | 2 +- internal/postgres/errors.go | 2 +- internal/postgres/idempotency.go | 2 +- internal/postgres/outbox.go | 2 +- internal/postgres/store.go | 2 +- internal/postgres/store_integration_test.go | 8 +- internal/reconcile/reconcile.go | 2 +- internal/reconcile/reconcile_test.go | 2 +- internal/session/session.go | 2 +- internal/session/session_integration_test.go | 2 +- internal/telemetry/database.go | 39 +++ internal/telemetry/logging.go | 74 +++++ internal/telemetry/propagation.go | 70 +++++ internal/telemetry/propagation_test.go | 33 +++ internal/telemetry/runtime.go | 252 ++++++++++++++++++ internal/telemetry/runtime_test.go | 59 ++++ 49 files changed, 1113 insertions(+), 286 deletions(-) create mode 100644 internal/bootstrap/telemetry.go create mode 100644 internal/database/telemetry.go create mode 100644 internal/telemetry/database.go create mode 100644 internal/telemetry/logging.go create mode 100644 internal/telemetry/propagation.go create mode 100644 internal/telemetry/propagation_test.go create mode 100644 internal/telemetry/runtime.go create mode 100644 internal/telemetry/runtime_test.go diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go index 5dc3072..048a121 100644 --- a/cmd/lab5experiment/api_scenario.go +++ b/cmd/lab5experiment/api_scenario.go @@ -17,7 +17,7 @@ import ( "github.com/google/uuid" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type apiState struct { diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go index 3749235..35c5241 100644 --- a/cmd/lab5experiment/main.go +++ b/cmd/lab5experiment/main.go @@ -21,15 +21,15 @@ import ( "github.com/google/uuid" "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" ) const ( diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 80230b8..c2c6c96 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -5,8 +5,8 @@ import ( "log/slog" "os" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/bootstrap" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/logger" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger" ) func main() { diff --git a/go.mod b/go.mod index 563b37a..37c0a6e 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gitea.privcloud.tech/dizi/mtusi2026lab5 +module gitea.privcloud.tech/dizi/mtusi2026lab6 go 1.26.0 @@ -12,19 +12,36 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/valkey-io/valkey-go v1.0.78 + go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 + go.opentelemetry.io/otel/exporters/prometheus v0.60.0 + go.opentelemetry.io/otel/metric v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 golang.org/x/crypto v0.57.0 golang.org/x/sync v0.23.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect @@ -39,7 +56,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/otlptranslator v0.0.2 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect @@ -48,11 +66,18 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tinylib/msgp v1.6.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/log v0.14.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index edfd0ad..63eb5b2 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -21,12 +23,23 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -70,12 +83,14 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= +github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= @@ -101,6 +116,40 @@ github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8= github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= @@ -117,6 +166,14 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/bootstrap/migration.go b/internal/bootstrap/migration.go index 4cf7712..0e7dd49 100644 --- a/internal/bootstrap/migration.go +++ b/internal/bootstrap/migration.go @@ -3,8 +3,8 @@ package bootstrap import ( "context" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" ) func RunMigrations() error { diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go index 97ce3ec..1539107 100644 --- a/internal/bootstrap/pipeline.go +++ b/internal/bootstrap/pipeline.go @@ -14,15 +14,16 @@ import ( "github.com/google/uuid" "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) func RunDispatcher(logger *slog.Logger) error { @@ -30,11 +31,23 @@ func RunDispatcher(logger *slog.Logger) error { if err != nil { return err } + instanceID, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-dispatcher", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings)) if err != nil { return err } defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil { + return err + } client, err := newRedisClient(settings.BrokerURL) if err != nil { return err @@ -51,7 +64,7 @@ func RunDispatcher(logger *slog.Logger) error { if err := broker.Ensure(context.Background()); err != nil { return err } - metrics := pipeline.NewMetrics() + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0) dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, @@ -70,11 +83,23 @@ func RunWorker(logger *slog.Logger) error { if err != nil { return err } + hostname, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-worker", hostname) + if err != nil { + return err + } + defer shutdownTelemetry(observability) pool, err := database.Open(context.Background(), workerDatabaseConfig(settings)) if err != nil { return err } defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil { + return err + } objects, err := objectstore.New(objectstore.Config{ Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, @@ -88,10 +113,6 @@ func RunWorker(logger *slog.Logger) error { return err } defer func() { _ = client.Close() }() - hostname, err := os.Hostname() - if err != nil { - return err - } consumer := hostname + "-" + uuid.NewString() broker, err := redisstream.New(client, redisstream.Config{ Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, @@ -108,7 +129,7 @@ func RunWorker(logger *slog.Logger) error { if err != nil { return err } - metrics := pipeline.NewMetrics() + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency) processorWorker, err := worker.New( logger, postgres.New(pool), broker, objects, processor, metrics, worker.Config{ diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go index 447dd23..3f09a3f 100644 --- a/internal/bootstrap/reconciliation.go +++ b/internal/bootstrap/reconciliation.go @@ -10,11 +10,11 @@ import ( "syscall" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" ) type reconciliationOutput struct { diff --git a/internal/bootstrap/replay.go b/internal/bootstrap/replay.go index 56b9f2e..70cc3e8 100644 --- a/internal/bootstrap/replay.go +++ b/internal/bootstrap/replay.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" ) func RunAttachmentReplay(arguments []string, output io.Writer) error { diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go index 121e328..473c0b7 100644 --- a/internal/bootstrap/server.go +++ b/internal/bootstrap/server.go @@ -9,14 +9,15 @@ import ( "os/signal" "syscall" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/httpapi" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/password" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) func RunServer(logger *slog.Logger) error { @@ -28,11 +29,19 @@ func RunServer(logger *slog.Logger) error { if err != nil { return err } + observability, logger, err := startTelemetry("messenger-api", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) pool, err := database.Open(context.Background(), databaseConfig(settings)) if err != nil { return err } defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil { + return err + } sessions, err := session.New(settings.CacheURL) if err != nil { return err @@ -62,6 +71,7 @@ func RunServer(logger *slog.Logger) error { app := httpapi.NewServer(logger, httpapi.Dependencies{ Users: repository, Chats: repository, Messages: repository, Attachments: attachments, Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects, + Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(), }, httpapi.Config{ SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL, RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort, diff --git a/internal/bootstrap/telemetry.go b/internal/bootstrap/telemetry.go new file mode 100644 index 0000000..bb1dcd9 --- /dev/null +++ b/internal/bootstrap/telemetry.go @@ -0,0 +1,23 @@ +package bootstrap + +import ( + "context" + "log/slog" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) { + runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID)) + if err != nil { + return nil, nil, err + } + return runtime, runtime.Logger(), nil +} + +func shutdownTelemetry(runtime *telemetry.Runtime) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = runtime.Shutdown(ctx) +} diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go index 35d46f0..0788997 100644 --- a/internal/broker/redisstream/broker.go +++ b/internal/broker/redisstream/broker.go @@ -11,7 +11,12 @@ import ( "github.com/redis/go-redis/v9" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" ) const ( @@ -103,6 +108,10 @@ func isBusyGroup(err error) bool { } func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() envelope, err := DecodeEnvelope(payload) if err != nil { return "", err @@ -117,12 +126,19 @@ func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, Values: map[string]any{payloadField: string(payload)}, }).Result() if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XADD failed") return "", fmt.Errorf("publish redis stream job: %w", err) } + span.SetAttributes(attribute.String("messaging.message.id", id)) return id, nil } func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: b.config.Group, Consumer: b.config.Consumer, @@ -134,6 +150,8 @@ func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { return nil, nil } if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XREADGROUP failed") return nil, fmt.Errorf("read redis stream jobs: %w", err) } return b.deliveries(streams) diff --git a/internal/database/database.go b/internal/database/database.go index ff1f8bd..c631b06 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab5/migrations" + "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" ) var ( @@ -40,6 +40,7 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) { if err != nil { return nil, fmt.Errorf("parse database configuration: %w", err) } + poolConfig.ConnConfig.Tracer = queryTracer{} poolConfig.MinConns = config.MinConnections poolConfig.MaxConns = config.MaxConnections poolConfig.MaxConnLifetime = config.MaxConnectionAge diff --git a/internal/database/telemetry.go b/internal/database/telemetry.go new file mode 100644 index 0000000..e2c34f8 --- /dev/null +++ b/internal/database/telemetry.go @@ -0,0 +1,28 @@ +package database + +import ( + "context" + + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +type queryTracer struct{} + +func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "postgresql"))) + return ctx +} + +func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + span := trace.SpanFromContext(ctx) + if data.Err != nil { + span.RecordError(data.Err) + span.SetStatus(codes.Error, "query failed") + } + span.End() +} diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go index 8e8d432..07a91a4 100644 --- a/internal/messenger/attachment/pipeline/metrics.go +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -1,104 +1,106 @@ package pipeline import ( + "context" "net/http" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} type Metrics struct { - workerInFlight prometheus.Gauge - workerJobs *prometheus.CounterVec - workerDuration *prometheus.HistogramVec - workerRetries prometheus.Counter - workerReclaimed prometheus.Counter - outboxPublished prometheus.Counter - outboxPublishFailure prometheus.Counter - outboxRecovered prometheus.Counter + meter metric.Meter + workerInFlight metric.Int64UpDownCounter + workerJobs metric.Int64Counter + workerDuration metric.Float64Histogram + workerTimeToReady metric.Float64Histogram + workerRetries metric.Int64Counter + workerReclaimed metric.Int64Counter + outboxPublished metric.Int64Counter + outboxPublishFailure metric.Int64Counter + outboxRecovered metric.Int64Counter handler http.Handler } -func NewMetrics() *Metrics { - registry := prometheus.NewRegistry() - metrics := &Metrics{ - workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight", - Help: "Attachment jobs currently processed by this worker.", - }), - workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total", - Help: "Completed attachment job deliveries by result.", - }, []string{"result"}), - workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds", - Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets, - }, []string{"result"}), - workerRetries: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total", - Help: "Attachment jobs scheduled for another attempt.", - }), - workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total", - Help: "Redis Stream deliveries reclaimed after their idle timeout.", - }), - outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "outbox", Name: "published_total", - Help: "Outbox events confirmed by Redis Streams.", - }), - outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total", - Help: "Outbox events whose Redis Streams publish failed.", - }), - outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total", - Help: "Expired outbox leases returned for publication.", - }), +func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics { + if meter == nil { + meter = otel.Meter("messenger-pipeline") } - registry.MustRegister( - metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration, - metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished, - metrics.outboxPublishFailure, metrics.outboxRecovered, - collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + if handler == nil { + handler = http.NotFoundHandler() + } + workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker.")) + workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result.")) + workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration", + metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(workerDurationBuckets...), ) - metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) - return metrics + workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready", + metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(workerDurationBuckets...), + ) + workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt.")) + workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout.")) + outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams.")) + outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed.")) + outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication.")) + if workerCapacity > 0 { + _, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity", + metric.WithDescription("Maximum concurrent attachment jobs for this worker."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(int64(workerCapacity)) + return nil + }), + ) + } + return &Metrics{ + meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs, + workerDuration: workerDuration, workerTimeToReady: workerTimeToReady, + workerRetries: workerRetries, workerReclaimed: workerReclaimed, + outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure, + outboxRecovered: outboxRecovered, handler: handler, + } } func (m *Metrics) Handler() http.Handler { return m.handler } -func (m *Metrics) ObserveStarted() { - m.workerInFlight.Inc() +func (m *Metrics) ObserveStarted(ctx context.Context) { + m.workerInFlight.Add(ctx, 1) } -func (m *Metrics) ObserveFinished(result string, duration time.Duration) { - m.workerInFlight.Dec() - m.workerJobs.WithLabelValues(result).Inc() - m.workerDuration.WithLabelValues(result).Observe(duration.Seconds()) +func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) { + attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result)) + m.workerInFlight.Add(ctx, -1) + m.workerJobs.Add(ctx, 1, attributes) + m.workerDuration.Record(ctx, duration.Seconds(), attributes) } -func (m *Metrics) ObserveRetry() { - m.workerRetries.Inc() +func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) { + m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process"))) } -func (m *Metrics) ObserveReclaimed(count int) { - m.workerReclaimed.Add(float64(count)) +func (m *Metrics) ObserveRetry(ctx context.Context) { + m.workerRetries.Add(ctx, 1) } -func (m *Metrics) ObservePublished() { - m.outboxPublished.Inc() +func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) { + m.workerReclaimed.Add(ctx, int64(count)) } -func (m *Metrics) ObservePublishFailure() { - m.outboxPublishFailure.Inc() +func (m *Metrics) ObservePublished(ctx context.Context) { + m.outboxPublished.Add(ctx, 1) } -func (m *Metrics) ObserveRecovered(count int) { - m.outboxRecovered.Add(float64(count)) +func (m *Metrics) ObservePublishFailure(ctx context.Context) { + m.outboxPublishFailure.Add(ctx, 1) +} + +func (m *Metrics) ObserveRecovered(ctx context.Context, count int) { + m.outboxRecovered.Add(ctx, int64(count)) } diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go index 7e0bbf8..38e5bdc 100644 --- a/internal/messenger/attachment/service.go +++ b/internal/messenger/attachment/service.go @@ -19,7 +19,8 @@ import ( "github.com/google/uuid" "golang.org/x/sync/semaphore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) type Repository interface { @@ -53,8 +54,7 @@ type Upload struct { ContentType string ContentLength int64 Body io.Reader - Traceparent string - Tracestate string + TraceContext telemetry.MessageCarrier } type Content struct { @@ -112,27 +112,27 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e return model.Attachment{}, mapDependencyError(err) } if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil { - s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err) - s.deletePending(pending.ID, stagingKey) + s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err) + s.deletePending(ctx, pending.ID, stagingKey) return model.Attachment{}, model.ErrUnavailable } createdAt := time.Now().UTC() queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{ ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion, Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt, - }, input.Traceparent, input.Tracestate) + }, input.TraceContext.Traceparent, input.TraceContext.Tracestate) if err != nil { - s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err) + s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err) return model.Attachment{}, mapDependencyError(err) } return queued, nil } -func (s *Service) deletePending(attachmentID, stagingKey string) { - cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout) +func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout) defer cancel() if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil { - s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) + s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) } } diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go index da14eb0..296f283 100644 --- a/internal/messenger/attachment/service_test.go +++ b/internal/messenger/attachment/service_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func TestUploadLifecycle(t *testing.T) { diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go index f94f525..45ff600 100644 --- a/internal/messenger/attachment/worker/worker.go +++ b/internal/messenger/attachment/worker/worker.go @@ -18,10 +18,16 @@ import ( "golang.org/x/sync/semaphore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) type Repository interface { @@ -29,7 +35,7 @@ type Repository interface { ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) - RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error) + RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error) FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error) } @@ -50,10 +56,11 @@ type ImageProcessor interface { } type Metrics interface { - ObserveStarted() - ObserveFinished(string, time.Duration) - ObserveRetry() - ObserveReclaimed(int) + ObserveStarted(context.Context) + ObserveFinished(context.Context, string, time.Duration) + ObserveTimeToReady(context.Context, time.Duration) + ObserveRetry(context.Context) + ObserveReclaimed(context.Context, int) } type Config struct { @@ -166,7 +173,7 @@ func (w *Worker) Run(ctx context.Context) error { } return err } - w.metrics.ObserveReclaimed(len(deliveries)) + w.metrics.ObserveReclaimed(ctx, len(deliveries)) w.processBatch(ctx, deliveries) reclaimCursor = next if reclaimCursor == "" { @@ -189,25 +196,48 @@ func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Deli go func(delivery redisstream.Delivery) { defer group.Done() defer func() { <-limit }() - if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil { - w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err) - } + _ = w.ProcessDelivery(ctx, delivery, FaultNone) }(delivery) } group.Wait() } -func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error { +func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) { started := w.now() - w.metrics.ObserveStarted() result := "transient_error" - defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }() - if delivery.DecodeError != nil { + w.metrics.ObserveStarted(ctx) + defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }() result = "invalid_job" return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) } envelope := delivery.Envelope + ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{ + Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate, + }) + ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer), + trace.WithAttributes( + attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()), + attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt), + attribute.Int("pipeline.version", envelope.PipelineVersion), + )) + w.metrics.ObserveStarted(ctx) + defer func() { + w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) + span.SetAttributes(attribute.String("job.outcome", result)) + if resultErr != nil { + span.RecordError(resultErr) + span.SetStatus(codes.Error, result) + if ctx.Err() == nil { + w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr) + } + } else { + w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result) + } + span.End() + }() + lease, err := w.repository.ClaimAttachmentJob( ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt, w.config.WorkerID, w.config.LeaseDuration, @@ -246,7 +276,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv } if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") { if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil { - w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) + w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) } } if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale { @@ -260,6 +290,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return err } result = "ready" + w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt)) return nil } @@ -444,7 +475,10 @@ func (w *Worker) resolveProcessingFailure( return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) } nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt)) - mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt) + traceContext := telemetry.ContextCarrier(ctx) + mutation, err := w.repository.RetryAttachmentJob( + ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate, + ) if err != nil { return "transient_error", fmt.Errorf("retry attachment job: %w", err) } @@ -454,7 +488,7 @@ func (w *Worker) resolveProcessingFailure( if mutation != model.JobMutationApplied { return "transient_error", fmt.Errorf("unexpected retry result %q", mutation) } - w.metrics.ObserveRetry() + w.metrics.ObserveRetry(ctx) return "retry", delivery.Ack(ctx) } @@ -502,7 +536,8 @@ func (w *Worker) retryDelay(jobID string, attempt int) time.Duration { type nopMetrics struct{} -func (nopMetrics) ObserveStarted() {} -func (nopMetrics) ObserveFinished(string, time.Duration) {} -func (nopMetrics) ObserveRetry() {} -func (nopMetrics) ObserveReclaimed(int) {} +func (nopMetrics) ObserveStarted(context.Context) {} +func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {} +func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {} +func (nopMetrics) ObserveRetry(context.Context) {} +func (nopMetrics) ObserveReclaimed(context.Context, int) {} diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go index 7a573f1..c2ca384 100644 --- a/internal/messenger/attachment/worker/worker_test.go +++ b/internal/messenger/attachment/worker/worker_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" ) func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) { diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go index f9250c8..2e832d6 100644 --- a/internal/messenger/httpapi/attachments.go +++ b/internal/messenger/httpapi/attachments.go @@ -13,16 +13,21 @@ import ( "unicode" "unicode/utf8" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { started := time.Now() result := "client_error" defer func() { - s.metrics.attachmentUploads.WithLabelValues(result).Inc() - s.metrics.attachmentDuration.Observe(time.Since(started).Seconds()) + attributes := metric.WithAttributes(attribute.String("result", result)) + s.metrics.attachmentUploads.Add(r.Context(), 1, attributes) + s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes) }() if r.Method != http.MethodPost { @@ -47,7 +52,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ ChatID: chatID, OwnerID: user.ID, Filename: filename, ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, - Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")), + TraceContext: telemetry.ContextCarrier(r.Context()), }) if err != nil { if errors.Is(err, model.ErrCapacity) { @@ -59,7 +64,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr return attachmentError(err) } result = "success" - s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes)) + s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes) writeJSON(w, http.StatusAccepted, uploaded) return nil } @@ -128,7 +133,7 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, content.Body); err != nil { - s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err) + s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err) } return nil } diff --git a/internal/messenger/httpapi/auth.go b/internal/messenger/httpapi/auth.go index 5baa066..29419a1 100644 --- a/internal/messenger/httpapi/auth.go +++ b/internal/messenger/httpapi/auth.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go index 1b41574..9084d13 100644 --- a/internal/messenger/httpapi/handlers.go +++ b/internal/messenger/httpapi/handlers.go @@ -9,7 +9,7 @@ import ( "time" "unicode/utf8" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) @@ -229,7 +229,7 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { return unavailable() } if replay { - s.metrics.replays.Inc() + s.metrics.replays.Add(r.Context(), 1) } writeJSON(w, http.StatusCreated, message) return nil diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go index 055b80b..9844c25 100644 --- a/internal/messenger/httpapi/messenger_interface.go +++ b/internal/messenger/httpapi/messenger_interface.go @@ -4,8 +4,8 @@ import ( "context" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type UserRepository interface { diff --git a/internal/messenger/httpapi/metrics.go b/internal/messenger/httpapi/metrics.go index f1b058e..5047ebd 100644 --- a/internal/messenger/httpapi/metrics.go +++ b/internal/messenger/httpapi/metrics.go @@ -1,14 +1,15 @@ package httpapi import ( + "context" "net/http" "strconv" "strings" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) var ( @@ -17,80 +18,50 @@ var ( ) type metrics struct { - requests *prometheus.CounterVec - duration *prometheus.HistogramVec - inFlight prometheus.Gauge - replays prometheus.Counter - attachmentUploads *prometheus.CounterVec - attachmentDuration prometheus.Histogram - attachmentBytes prometheus.Counter + requests metric.Int64Counter + duration metric.Float64Histogram + inFlight metric.Int64UpDownCounter + replays metric.Int64Counter + attachmentUploads metric.Int64Counter + attachmentDuration metric.Float64Histogram + attachmentBytes metric.Int64Counter handler http.Handler } -func newMetrics() *metrics { - registry := prometheus.NewRegistry() - result := &metrics{ - requests: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "requests_total", - Help: "Completed HTTP requests.", - }, []string{"method", "route", "status_class"}), - duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "request_duration_seconds", - Help: "HTTP request duration in seconds.", - Buckets: requestDurationBuckets, - }, []string{"method", "route"}), - inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "requests_in_flight", - Help: "HTTP requests currently being served.", - }), - replays: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", - Subsystem: "http", - Name: "idempotency_replays_total", - Help: "Message requests served from an idempotency record.", - }), - attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "messenger", - Name: "attachment_uploads_total", - Help: "Completed attachment uploads by result.", - }, []string{"result"}), - attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: "messenger", - Name: "attachment_upload_duration_seconds", - Help: "Attachment upload duration in seconds.", - Buckets: uploadDurationBuckets, - }), - attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "messenger", - Name: "attachment_upload_bytes_total", - Help: "Bytes accepted by successful attachment uploads.", - }), +func newMetrics(meter metric.Meter, handler http.Handler) *metrics { + if meter == nil { + meter = otel.Meter("messenger-api") } - registry.MustRegister( - result.requests, - result.duration, - result.inFlight, - result.replays, - result.attachmentUploads, - result.attachmentDuration, - result.attachmentBytes, - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + if handler == nil { + handler = http.NotFoundHandler() + } + requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests.")) + duration, _ := meter.Float64Histogram("messenger_http_request_duration", + metric.WithDescription("HTTP request duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(requestDurationBuckets...), ) - result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) - return result + inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served.")) + replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record.")) + attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result.")) + attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration", + metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(uploadDurationBuckets...), + ) + attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By")) + return &metrics{ + requests: requests, duration: duration, inFlight: inFlight, replays: replays, + attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration, + attachmentBytes: attachmentBytes, handler: handler, + } } -func (m *metrics) observe(r *http.Request, status int, started time.Time) { - route := routeTemplate(r.URL.Path) - m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc() - m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds()) +func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) { + attributes := metric.WithAttributes( + attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)), + attribute.String("status_class", strconv.Itoa(status/100)+"xx"), + ) + m.requests.Add(ctx, 1, attributes) + m.duration.Record(ctx, time.Since(started).Seconds(), attributes) } func routeTemplate(path string) string { diff --git a/internal/messenger/httpapi/model.go b/internal/messenger/httpapi/model.go index 9ecc48d..7f3f6dc 100644 --- a/internal/messenger/httpapi/model.go +++ b/internal/messenger/httpapi/model.go @@ -1,7 +1,7 @@ package httpapi import ( - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" "github.com/google/uuid" ) diff --git a/internal/messenger/httpapi/server.go b/internal/messenger/httpapi/server.go index 9af19cf..fcc18bf 100644 --- a/internal/messenger/httpapi/server.go +++ b/internal/messenger/httpapi/server.go @@ -6,6 +6,16 @@ import ( "net/http" "sync/atomic" "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) type Dependencies struct { @@ -17,6 +27,8 @@ type Dependencies struct { Passwords PasswordService Database HealthChecker Objects HealthChecker + Meter metric.Meter + Metrics http.Handler } type Config struct { @@ -67,7 +79,8 @@ func NewServer( attachments: dependencies.Attachments, sessions: dependencies.Sessions, passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects, sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL, - instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(), + instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, + metrics: newMetrics(dependencies.Meter, dependencies.Metrics), } s.ready.Store(true) return s @@ -82,27 +95,39 @@ func (s *Server) Handler() http.Handler { } func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header)) + tracer := otel.Tracer("messenger-api/http") + route := routeTemplate(r.URL.Path) + requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route))) + defer span.End() + r = r.WithContext(requestContext) w.Header().Set("X-Instance-Id", s.instanceID) requestID, err := newID() if err != nil { http.Error(w, "server unavailable", http.StatusServiceUnavailable) return } + span.SetAttributes(attribute.String("request.id", requestID)) w.Header().Set("X-Request-Id", requestID) w.Header().Set("Cache-Control", "no-store") wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK} started := time.Now() - s.metrics.inFlight.Inc() + s.metrics.inFlight.Add(r.Context(), 1) defer func() { - s.metrics.inFlight.Dec() + s.metrics.inFlight.Add(r.Context(), -1) if recovered := recover(); recovered != nil { - s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path) + s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route) if wrapped.status == http.StatusOK { writeError(wrapped, requestID, internalError()) } } - s.metrics.observe(r, wrapped.status, started) - s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) + s.metrics.observe(r.Context(), r, wrapped.status, started) + span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status)) + if wrapped.status >= http.StatusInternalServerError { + span.SetStatus(codes.Error, http.StatusText(wrapped.status)) + } + s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) }() if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" { wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go index d2e0a21..c1bf91d 100644 --- a/internal/messenger/httpapi/server_test.go +++ b/internal/messenger/httpapi/server_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) var errDependency = errors.New("dependency unavailable") diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go index ac41f65..e2d6ded 100644 --- a/internal/objectstore/store.go +++ b/internal/objectstore/store.go @@ -13,7 +13,12 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) var ErrNotFound = errors.New("object not found") @@ -85,30 +90,38 @@ func (s *Store) Ping(ctx context.Context) error { } func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error { + ctx, span := s.startSpan(ctx, "PUT") + defer span.End() ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() _, err := s.client.PutObject( ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType}, ) if err != nil { + recordSpanError(span, err) return fmt.Errorf("put S3 object: %w", err) } return nil } func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { + ctx, span := s.startSpan(ctx, "GET") ctx, cancel := context.WithTimeout(ctx, s.timeout) object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) if err != nil { cancel() + recordSpanError(span, err) + span.End() return nil, objectError("get S3 object", err) } if _, err := object.Stat(); err != nil { cancel() _ = object.Close() + recordSpanError(span, err) + span.End() return nil, objectError("stat S3 object", err) } - return &timeoutReader{ReadCloser: object, cancel: cancel}, nil + return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil } func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) { @@ -122,9 +135,12 @@ func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error } func (s *Store) Delete(ctx context.Context, key string) error { + ctx, span := s.startSpan(ctx, "DELETE") + defer span.End() ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { + recordSpanError(span, err) return objectError("delete S3 object", err) } return nil @@ -161,6 +177,28 @@ type timeoutReader struct { cancel context.CancelFunc } +type tracedReader struct { + io.ReadCloser + span trace.Span +} + +func (r *tracedReader) Read(buffer []byte) (int, error) { + read, err := r.ReadCloser.Read(buffer) + if err != nil && !errors.Is(err, io.EOF) { + recordSpanError(r.span, err) + } + return read, err +} + +func (r *tracedReader) Close() error { + err := r.ReadCloser.Close() + if err != nil { + recordSpanError(r.span, err) + } + r.span.End() + return err +} + func (r *timeoutReader) Close() error { err := r.ReadCloser.Close() r.cancel() @@ -174,3 +212,14 @@ func objectError(action string, err error) error { } return fmt.Errorf("%s: %w", action, err) } + +func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) { + return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation), + attribute.String("s3.bucket.name", s.bucket))) +} + +func recordSpanError(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, "S3 operation failed") +} diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go index 22aa75d..ea74f61 100644 --- a/internal/outbox/dispatcher.go +++ b/internal/outbox/dispatcher.go @@ -6,7 +6,13 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" ) var errLeaseLost = errors.New("outbox lease lost") @@ -24,9 +30,9 @@ type Publisher interface { } type Metrics interface { - ObservePublished() - ObservePublishFailure() - ObserveRecovered(int) + ObservePublished(context.Context) + ObservePublishFailure(context.Context) + ObserveRecovered(context.Context, int) } type Config struct { @@ -156,15 +162,32 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { if err := ctx.Err(); err != nil { return published, nil } - _, err := d.publisher.PublishConfirmed(ctx, event.Payload) + publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload) + publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish", + trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt), + )) + payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload) + if injectErr != nil { + span.RecordError(injectErr) + span.SetStatus(codes.Error, "inject trace context") + span.End() + return published, fmt.Errorf("inject outbox trace context: %w", injectErr) + } + _, err := d.publisher.PublishConfirmed(publishContext, payload) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "publish failed") + span.End() if ctx.Err() != nil { return published, nil } - d.metrics.ObservePublishFailure() + d.metrics.ObservePublishFailure(publishContext) nextAttemptAt := d.now().Add(d.config.RetryDelay) released, releaseErr := d.store.ReleaseOutboxEvent( - ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, + publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, ) if releaseErr != nil { return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr) @@ -174,14 +197,21 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { } continue } - marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken) + marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "mark published") + span.End() return published, fmt.Errorf("mark outbox event published: %w", err) } if !marked { + span.SetStatus(codes.Error, "outbox lease lost") + span.End() return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost) } - d.metrics.ObservePublished() + span.SetAttributes(attribute.String("messaging.operation.type", "publish")) + span.End() + d.metrics.ObservePublished(publishContext) published++ } return published, nil @@ -192,12 +222,12 @@ func (d *Dispatcher) RecoverExpired(ctx context.Context) error { if err != nil { return fmt.Errorf("recover expired outbox leases: %w", err) } - d.metrics.ObserveRecovered(recovered) + d.metrics.ObserveRecovered(ctx, recovered) return nil } type nopMetrics struct{} -func (nopMetrics) ObservePublished() {} -func (nopMetrics) ObservePublishFailure() {} -func (nopMetrics) ObserveRecovered(int) {} +func (nopMetrics) ObservePublished(context.Context) {} +func (nopMetrics) ObservePublishFailure(context.Context) {} +func (nopMetrics) ObserveRecovered(context.Context, int) {} diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go index 206790d..b9d792c 100644 --- a/internal/outbox/dispatcher_test.go +++ b/internal/outbox/dispatcher_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" "github.com/stretchr/testify/require" ) diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go index 3bcfb39..6c959de 100644 --- a/internal/postgres/attachment_jobs.go +++ b/internal/postgres/attachment_jobs.go @@ -10,7 +10,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const attachmentJobTopic = "attachment.jobs" @@ -447,6 +447,8 @@ func (s *Store) RetryAttachmentJob( lease model.AttachmentJobLease, errorCode string, nextAttemptAt time.Time, + traceparent string, + tracestate string, ) (model.JobMutationResult, error) { if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { return "", model.ErrInvalidInput @@ -483,9 +485,7 @@ func (s *Store) RetryAttachmentJob( if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil { return "", fmt.Errorf("requeue attachment: %w", err) } - eventJob := job - eventJob.CreatedAt = time.Now().UTC() - if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { return "", err } if err := tx.Commit(ctx); err != nil { diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go index b6883eb..9e631a9 100644 --- a/internal/postgres/attachment_lease.go +++ b/internal/postgres/attachment_lease.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Store) RenewAttachmentJobLease( diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go index d7d84ca..e305a79 100644 --- a/internal/postgres/attachment_recovery.go +++ b/internal/postgres/attachment_recovery.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Store) RecoverExpiredAttachmentJobs( diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index e7c2531..2765eda 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const attachmentColumns = ` diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go index e4daccb..39509ad 100644 --- a/internal/postgres/errors.go +++ b/internal/postgres/errors.go @@ -1,6 +1,6 @@ package postgres -import "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" var ( ErrConflict = model.ErrConflict diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go index a8168f5..4bbb95b 100644 --- a/internal/postgres/idempotency.go +++ b/internal/postgres/idempotency.go @@ -12,7 +12,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const expiredIdempotencyCleanupBatch = 100 diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go index 9eda865..0817f7c 100644 --- a/internal/postgres/outbox.go +++ b/internal/postgres/outbox.go @@ -8,7 +8,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func (s *Store) ClaimOutboxEvents( diff --git a/internal/postgres/store.go b/internal/postgres/store.go index 5413f28..44980b7 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -10,7 +10,7 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type Store struct { diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 4867523..e4184db 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -15,10 +15,10 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" ) type PostgresSuite struct { diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 9c4ee8c..15607f8 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) type MetadataStore interface { diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 8cbef1f..36a2955 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) func TestReconcilerLifecycleAndRepeat(t *testing.T) { diff --git a/internal/session/session.go b/internal/session/session.go index b10261b..a7df4b9 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -14,7 +14,7 @@ import ( "github.com/valkey-io/valkey-go" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" ) const ( diff --git a/internal/session/session_integration_test.go b/internal/session/session_integration_test.go index c7d874e..a82e5eb 100644 --- a/internal/session/session_integration_test.go +++ b/internal/session/session_integration_test.go @@ -11,7 +11,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/suite" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" ) type SessionSuite struct { diff --git a/internal/telemetry/database.go b/internal/telemetry/database.go new file mode 100644 index 0000000..6390cd3 --- /dev/null +++ b/internal/telemetry/database.go @@ -0,0 +1,39 @@ +package telemetry + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + connectionStateUsed = attribute.String("state", "used") + connectionStateIdle = attribute.String("state", "idle") + connectionStateConnecting = attribute.String("state", "connecting") +) + +func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error { + _, err := meter.Int64ObservableGauge("messenger_db_pool_connections", + metric.WithDescription("PostgreSQL pool connections by state."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + stats := pool.Stat() + observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed)) + observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle)) + observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting)) + return nil + }), + ) + if err != nil { + return err + } + _, err = meter.Int64ObservableCounter("messenger_db_pool_waited", + metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(pool.Stat().EmptyAcquireCount()) + return nil + }), + ) + return err +} diff --git a/internal/telemetry/logging.go b/internal/telemetry/logging.go new file mode 100644 index 0000000..880d349 --- /dev/null +++ b/internal/telemetry/logging.go @@ -0,0 +1,74 @@ +package telemetry + +import ( + "context" + "log/slog" + "strings" + + "go.opentelemetry.io/otel/trace" +) + +type correlationHandler struct { + stdout slog.Handler + otel slog.Handler +} + +func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler { + return correlationHandler{stdout: stdout, otel: otel} +} + +func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level) +} + +func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error { + span := trace.SpanContextFromContext(ctx) + if span.IsValid() { + record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String())) + } + record = sanitizedRecord(record) + if err := h.stdout.Handle(ctx, record); err != nil { + return err + } + return h.otel.Handle(ctx, record) +} + +func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + attributes = sanitizedAttributes(attributes) + return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)} +} + +func (h correlationHandler) WithGroup(name string) slog.Handler { + return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)} +} + +func sanitizedRecord(record slog.Record) slog.Record { + copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC) + record.Attrs(func(attribute slog.Attr) bool { + if !sensitiveKey(attribute.Key) { + copy.AddAttrs(attribute) + } + return true + }) + return copy +} + +func sanitizedAttributes(attributes []slog.Attr) []slog.Attr { + result := make([]slog.Attr, 0, len(attributes)) + for _, attribute := range attributes { + if !sensitiveKey(attribute.Key) { + result = append(result, attribute) + } + } + return result +} + +func sensitiveKey(key string) bool { + key = strings.ToLower(key) + for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} { + if strings.Contains(key, marker) { + return true + } + } + return false +} diff --git a/internal/telemetry/propagation.go b/internal/telemetry/propagation.go new file mode 100644 index 0000000..92c3dac --- /dev/null +++ b/internal/telemetry/propagation.go @@ -0,0 +1,70 @@ +package telemetry + +import ( + "context" + "encoding/json" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +type MessageCarrier struct { + Traceparent string + Tracestate string +} + +func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, carrier) +} + +func Inject(ctx context.Context, carrier propagation.TextMapCarrier) { + otel.GetTextMapPropagator().Inject(ctx, carrier) +} + +func ContextCarrier(ctx context.Context) MessageCarrier { + carrier := propagation.MapCarrier{} + Inject(ctx, carrier) + return MessageCarrier{ + Traceparent: carrier.Get("traceparent"), + Tracestate: carrier.Get("tracestate"), + } +} + +func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context { + headers := propagation.MapCarrier{ + "traceparent": carrier.Traceparent, + "tracestate": carrier.Tracestate, + } + return Extract(ctx, headers) +} + +func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context { + var carrier struct { + Traceparent string `json:"traceparent"` + Tracestate string `json:"tracestate"` + } + if json.Unmarshal(payload, &carrier) != nil { + return ctx + } + return ExtractMessage(ctx, MessageCarrier(carrier)) +} + +func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) { + var value map[string]json.RawMessage + if err := json.Unmarshal(payload, &value); err != nil { + return nil, err + } + carrier := ContextCarrier(ctx) + setJSONField(value, "traceparent", carrier.Traceparent) + setJSONField(value, "tracestate", carrier.Tracestate) + return json.Marshal(value) +} + +func setJSONField(value map[string]json.RawMessage, key, field string) { + if field == "" { + delete(value, key) + return + } + encoded, _ := json.Marshal(field) + value[key] = encoded +} diff --git a/internal/telemetry/propagation_test.go b/internal/telemetry/propagation_test.go new file mode 100644 index 0000000..31895a5 --- /dev/null +++ b/internal/telemetry/propagation_test.go @@ -0,0 +1,33 @@ +package telemetry + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) { + otel.SetTextMapPropagator(propagation.TraceContext{}) + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + + payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`)) + require.NoError(t, err) + var value map[string]any + require.NoError(t, json.Unmarshal(payload, &value)) + require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"]) + require.NotContains(t, value, "baggage") + + extracted := ExtractJSONMessage(context.Background(), payload) + require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID()) +} diff --git a/internal/telemetry/runtime.go b/internal/telemetry/runtime.go new file mode 100644 index 0000000..90267e6 --- /dev/null +++ b/internal/telemetry/runtime.go @@ -0,0 +1,252 @@ +package telemetry + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/bridges/otelslog" + otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + otelprometheus "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" +) + +const ( + defaultEnvironment = "lab" + defaultNamespace = "mtusi-highload" +) + +type Config struct { + ServiceName string + Version string + InstanceID string + Endpoint string + Environment string + Namespace string + Insecure bool +} + +type Runtime struct { + traces *trace.TracerProvider + metrics *metric.MeterProvider + logs *log.LoggerProvider + logger *slog.Logger + handler http.Handler +} + +func ConfigFromEnvironment(serviceName, instanceID string) Config { + resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES")) + return Config{ + ServiceName: serviceName, + Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"), + InstanceID: instanceID, + Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")), + Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment), + Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace), + Insecure: true, + } +} + +func parseResourceAttributes(raw string) map[string]string { + result := make(map[string]string) + for _, pair := range strings.Split(raw, ",") { + key, value, found := strings.Cut(pair, "=") + if found && strings.TrimSpace(key) != "" { + result[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + } + return result +} + +func firstValue(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func New(ctx context.Context, config Config) (*Runtime, error) { + if err := config.validate(); err != nil { + return nil, err + } + res, err := newResource(ctx, config) + if err != nil { + return nil, err + } + registry := prometheus.NewRegistry() + prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry)) + if err != nil { + return nil, fmt.Errorf("create Prometheus reader: %w", err) + } + + traceOptions := []trace.TracerProviderOption{trace.WithResource(res)} + metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)} + logOptions := []log.LoggerProviderOption{log.WithResource(res)} + if config.Endpoint != "" { + traceExporter, err := newTraceExporter(ctx, config) + if err != nil { + return nil, err + } + metricExporter, err := newMetricExporter(ctx, config) + if err != nil { + return nil, err + } + logExporter, err := newLogExporter(ctx, config) + if err != nil { + return nil, err + } + traceOptions = append(traceOptions, trace.WithBatcher(traceExporter, + trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512), + trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second), + )) + metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader( + metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second), + ))) + logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter, + log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512), + log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second), + ))) + } + + traces := trace.NewTracerProvider(traceOptions...) + metrics := metric.NewMeterProvider(metricOptions...) + logs := log.NewLoggerProvider(logOptions...) + propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}) + otel.SetTracerProvider(traces) + otel.SetMeterProvider(metrics) + otel.SetTextMapPropagator(propagator) + if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil { + return nil, fmt.Errorf("start Go runtime metrics: %w", err) + } + + stdout := slog.NewJSONHandler(os.Stdout, nil) + otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs)) + logger := slog.New(newCorrelationHandler(stdout, otelHandler)) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + return &Runtime{ + traces: traces, metrics: metrics, logs: logs, logger: logger, + handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), + }, nil +} + +func (c Config) validate() error { + switch { + case strings.TrimSpace(c.ServiceName) == "": + return errors.New("telemetry service name is required") + case strings.TrimSpace(c.InstanceID) == "": + return errors.New("telemetry instance ID is required") + case strings.TrimSpace(c.Environment) == "": + return errors.New("telemetry environment is required") + case strings.TrimSpace(c.Namespace) == "": + return errors.New("telemetry service namespace is required") + default: + return nil + } +} + +func newResource(ctx context.Context, config Config) (*resource.Resource, error) { + res, err := resource.New(ctx, + resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(), + resource.WithAttributes( + semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version), + semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace), + semconv.DeploymentEnvironmentName(config.Environment), + ), + ) + if err != nil { + return nil, fmt.Errorf("create telemetry resource: %w", err) + } + return res, nil +} + +func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) { + var options []otlptracegrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlptracegrpc.WithInsecure()) + } + exporter, err := otlptracegrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP trace exporter: %w", err) + } + return exporter, nil +} + +func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) { + var options []otlpmetricgrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlpmetricgrpc.WithInsecure()) + } + exporter, err := otlpmetricgrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP metric exporter: %w", err) + } + return exporter, nil +} + +func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) { + var options []otlploggrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlploggrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlploggrpc.WithInsecure()) + } + exporter, err := otlploggrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP log exporter: %w", err) + } + return exporter, nil +} + +func (r *Runtime) Logger() *slog.Logger { + return r.logger +} + +func (r *Runtime) MeterProvider() *metric.MeterProvider { + return r.metrics +} + +func (r *Runtime) MetricsHandler() http.Handler { + return r.handler +} + +func (r *Runtime) Shutdown(ctx context.Context) error { + var result error + for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} { + if err := shutdown(ctx); err != nil { + result = errors.Join(result, err) + } + } + return result +} diff --git a/internal/telemetry/runtime_test.go b/internal/telemetry/runtime_test.go new file mode 100644 index 0000000..e8b41b1 --- /dev/null +++ b/internal/telemetry/runtime_test.go @@ -0,0 +1,59 @@ +package telemetry + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" +) + +func TestConfigValidation(t *testing.T) { + valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"} + require.NoError(t, valid.validate()) + + invalid := valid + invalid.ServiceName = "" + require.EqualError(t, invalid.validate(), "telemetry service name is required") +} + +func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) { + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + first := &recordHandler{} + second := &recordHandler{} + handler := newCorrelationHandler(first, second) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0) + record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret")) + + require.NoError(t, handler.Handle(ctx, record)) + for _, captured := range []*recordHandler{first, second} { + require.Equal(t, traceID.String(), captured.attributes["trace_id"]) + require.Equal(t, spanID.String(), captured.attributes["span_id"]) + require.Equal(t, "http_request_completed", captured.attributes["event"]) + require.NotContains(t, captured.attributes, "authorization") + } +} + +type recordHandler struct { + attributes map[string]string +} + +func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h *recordHandler) Handle(_ context.Context, record slog.Record) error { + h.attributes = make(map[string]string) + record.Attrs(func(attribute slog.Attr) bool { + h.attributes[attribute.Key] = attribute.Value.String() + return true + }) + return nil +} +func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *recordHandler) WithGroup(string) slog.Handler { return h }