feat: add OpenTelemetry instrumentation

This commit is contained in:
2026-11-01 17:06:32 +03:00
parent cea50c2cd6
commit 934dfefee7
49 changed files with 1113 additions and 286 deletions
+1 -1
View File
@@ -17,7 +17,7 @@ import (
"github.com/google/uuid"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type apiState struct {
+9 -9
View File
@@ -21,15 +21,15 @@ import (
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
)
const (
+2 -2
View File
@@ -5,8 +5,8 @@ import (
"log/slog"
"os"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/bootstrap"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/logger"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger"
)
func main() {
+27 -2
View File
@@ -1,4 +1,4 @@
module gitea.privcloud.tech/dizi/mtusi2026lab5
module gitea.privcloud.tech/dizi/mtusi2026lab6
go 1.26.0
@@ -12,19 +12,36 @@ require (
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/valkey-io/valkey-go v1.0.78
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
go.opentelemetry.io/otel/exporters/prometheus v0.60.0
go.opentelemetry.io/otel/metric v1.38.0
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/sdk/log v0.14.0
go.opentelemetry.io/otel/sdk/metric v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.23.0
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
@@ -39,7 +56,8 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/prometheus/otlptranslator v0.0.2 // indirect
github.com/prometheus/procfs v0.17.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
@@ -48,11 +66,18 @@ require (
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/log v0.14.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+61 -4
View File
@@ -4,6 +4,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -21,12 +23,23 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -70,12 +83,14 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ=
github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI=
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
@@ -101,6 +116,40 @@ github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s=
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4=
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE=
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo=
go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk=
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
@@ -117,6 +166,14 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+2 -2
View File
@@ -3,8 +3,8 @@ package bootstrap
import (
"context"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
)
func RunMigrations() error {
+36 -15
View File
@@ -14,15 +14,16 @@ import (
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func RunDispatcher(logger *slog.Logger) error {
@@ -30,11 +31,23 @@ func RunDispatcher(logger *slog.Logger) error {
if err != nil {
return err
}
instanceID, err := os.Hostname()
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-dispatcher", instanceID)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil {
return err
}
client, err := newRedisClient(settings.BrokerURL)
if err != nil {
return err
@@ -51,7 +64,7 @@ func RunDispatcher(logger *slog.Logger) error {
if err := broker.Ensure(context.Background()); err != nil {
return err
}
metrics := pipeline.NewMetrics()
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0)
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
@@ -70,11 +83,23 @@ func RunWorker(logger *slog.Logger) error {
if err != nil {
return err
}
hostname, err := os.Hostname()
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-worker", hostname)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil {
return err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
@@ -88,10 +113,6 @@ func RunWorker(logger *slog.Logger) error {
return err
}
defer func() { _ = client.Close() }()
hostname, err := os.Hostname()
if err != nil {
return err
}
consumer := hostname + "-" + uuid.NewString()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
@@ -108,7 +129,7 @@ func RunWorker(logger *slog.Logger) error {
if err != nil {
return err
}
metrics := pipeline.NewMetrics()
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency)
processorWorker, err := worker.New(
logger, postgres.New(pool), broker, objects, processor, metrics,
worker.Config{
+5 -5
View File
@@ -10,11 +10,11 @@ import (
"syscall"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
)
type reconciliationOutput struct {
+3 -3
View File
@@ -6,9 +6,9 @@ import (
"fmt"
"io"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
)
func RunAttachmentReplay(arguments []string, output io.Writer) error {
+18 -8
View File
@@ -9,14 +9,15 @@ import (
"os/signal"
"syscall"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/httpapi"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func RunServer(logger *slog.Logger) error {
@@ -28,11 +29,19 @@ func RunServer(logger *slog.Logger) error {
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-api", instanceID)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), databaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil {
return err
}
sessions, err := session.New(settings.CacheURL)
if err != nil {
return err
@@ -62,6 +71,7 @@ func RunServer(logger *slog.Logger) error {
app := httpapi.NewServer(logger, httpapi.Dependencies{
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(),
}, httpapi.Config{
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
+23
View File
@@ -0,0 +1,23 @@
package bootstrap
import (
"context"
"log/slog"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) {
runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID))
if err != nil {
return nil, nil, err
}
return runtime, runtime.Logger(), nil
}
func shutdownTelemetry(runtime *telemetry.Runtime) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = runtime.Shutdown(ctx)
}
+19 -1
View File
@@ -11,7 +11,12 @@ import (
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
)
const (
@@ -103,6 +108,10 @@ func isBusyGroup(err error) bool {
}
func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) {
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"),
attribute.String("messaging.destination.name", b.config.Stream)))
defer span.End()
envelope, err := DecodeEnvelope(payload)
if err != nil {
return "", err
@@ -117,12 +126,19 @@ func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string,
Values: map[string]any{payloadField: string(payload)},
}).Result()
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "XADD failed")
return "", fmt.Errorf("publish redis stream job: %w", err)
}
span.SetAttributes(attribute.String("messaging.message.id", id))
return id, nil
}
func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"),
attribute.String("messaging.destination.name", b.config.Stream)))
defer span.End()
streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: b.config.Group,
Consumer: b.config.Consumer,
@@ -134,6 +150,8 @@ func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
return nil, nil
}
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "XREADGROUP failed")
return nil, fmt.Errorf("read redis stream jobs: %w", err)
}
return b.deliveries(streams)
+2 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab5/migrations"
"gitea.privcloud.tech/dizi/mtusi2026lab6/migrations"
)
var (
@@ -40,6 +40,7 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
if err != nil {
return nil, fmt.Errorf("parse database configuration: %w", err)
}
poolConfig.ConnConfig.Tracer = queryTracer{}
poolConfig.MinConns = config.MinConnections
poolConfig.MaxConns = config.MaxConnections
poolConfig.MaxConnLifetime = config.MaxConnectionAge
+28
View File
@@ -0,0 +1,28 @@
package database
import (
"context"
"github.com/jackc/pgx/v5"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
type queryTracer struct{}
func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system", "postgresql")))
return ctx
}
func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
span := trace.SpanFromContext(ctx)
if data.Err != nil {
span.RecordError(data.Err)
span.SetStatus(codes.Error, "query failed")
}
span.End()
}
@@ -1,104 +1,106 @@
package pipeline
import (
"context"
"net/http"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60}
type Metrics struct {
workerInFlight prometheus.Gauge
workerJobs *prometheus.CounterVec
workerDuration *prometheus.HistogramVec
workerRetries prometheus.Counter
workerReclaimed prometheus.Counter
outboxPublished prometheus.Counter
outboxPublishFailure prometheus.Counter
outboxRecovered prometheus.Counter
meter metric.Meter
workerInFlight metric.Int64UpDownCounter
workerJobs metric.Int64Counter
workerDuration metric.Float64Histogram
workerTimeToReady metric.Float64Histogram
workerRetries metric.Int64Counter
workerReclaimed metric.Int64Counter
outboxPublished metric.Int64Counter
outboxPublishFailure metric.Int64Counter
outboxRecovered metric.Int64Counter
handler http.Handler
}
func NewMetrics() *Metrics {
registry := prometheus.NewRegistry()
metrics := &Metrics{
workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight",
Help: "Attachment jobs currently processed by this worker.",
}),
workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total",
Help: "Completed attachment job deliveries by result.",
}, []string{"result"}),
workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds",
Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets,
}, []string{"result"}),
workerRetries: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total",
Help: "Attachment jobs scheduled for another attempt.",
}),
workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total",
Help: "Redis Stream deliveries reclaimed after their idle timeout.",
}),
outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "published_total",
Help: "Outbox events confirmed by Redis Streams.",
}),
outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total",
Help: "Outbox events whose Redis Streams publish failed.",
}),
outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total",
Help: "Expired outbox leases returned for publication.",
}),
func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics {
if meter == nil {
meter = otel.Meter("messenger-pipeline")
}
registry.MustRegister(
metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration,
metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished,
metrics.outboxPublishFailure, metrics.outboxRecovered,
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
if handler == nil {
handler = http.NotFoundHandler()
}
workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker."))
workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result."))
workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration",
metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
)
metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
return metrics
workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready",
metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
)
workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt."))
workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout."))
outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams."))
outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed."))
outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication."))
if workerCapacity > 0 {
_, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity",
metric.WithDescription("Maximum concurrent attachment jobs for this worker."),
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
observer.Observe(int64(workerCapacity))
return nil
}),
)
}
return &Metrics{
meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs,
workerDuration: workerDuration, workerTimeToReady: workerTimeToReady,
workerRetries: workerRetries, workerReclaimed: workerReclaimed,
outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure,
outboxRecovered: outboxRecovered, handler: handler,
}
}
func (m *Metrics) Handler() http.Handler {
return m.handler
}
func (m *Metrics) ObserveStarted() {
m.workerInFlight.Inc()
func (m *Metrics) ObserveStarted(ctx context.Context) {
m.workerInFlight.Add(ctx, 1)
}
func (m *Metrics) ObserveFinished(result string, duration time.Duration) {
m.workerInFlight.Dec()
m.workerJobs.WithLabelValues(result).Inc()
m.workerDuration.WithLabelValues(result).Observe(duration.Seconds())
func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) {
attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result))
m.workerInFlight.Add(ctx, -1)
m.workerJobs.Add(ctx, 1, attributes)
m.workerDuration.Record(ctx, duration.Seconds(), attributes)
}
func (m *Metrics) ObserveRetry() {
m.workerRetries.Inc()
func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) {
m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process")))
}
func (m *Metrics) ObserveReclaimed(count int) {
m.workerReclaimed.Add(float64(count))
func (m *Metrics) ObserveRetry(ctx context.Context) {
m.workerRetries.Add(ctx, 1)
}
func (m *Metrics) ObservePublished() {
m.outboxPublished.Inc()
func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) {
m.workerReclaimed.Add(ctx, int64(count))
}
func (m *Metrics) ObservePublishFailure() {
m.outboxPublishFailure.Inc()
func (m *Metrics) ObservePublished(ctx context.Context) {
m.outboxPublished.Add(ctx, 1)
}
func (m *Metrics) ObserveRecovered(count int) {
m.outboxRecovered.Add(float64(count))
func (m *Metrics) ObservePublishFailure(ctx context.Context) {
m.outboxPublishFailure.Add(ctx, 1)
}
func (m *Metrics) ObserveRecovered(ctx context.Context, count int) {
m.outboxRecovered.Add(ctx, int64(count))
}
+10 -10
View File
@@ -19,7 +19,8 @@ import (
"github.com/google/uuid"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
type Repository interface {
@@ -53,8 +54,7 @@ type Upload struct {
ContentType string
ContentLength int64
Body io.Reader
Traceparent string
Tracestate string
TraceContext telemetry.MessageCarrier
}
type Content struct {
@@ -112,27 +112,27 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e
return model.Attachment{}, mapDependencyError(err)
}
if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil {
s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err)
s.deletePending(pending.ID, stagingKey)
s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err)
s.deletePending(ctx, pending.ID, stagingKey)
return model.Attachment{}, model.ErrUnavailable
}
createdAt := time.Now().UTC()
queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{
ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion,
Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt,
}, input.Traceparent, input.Tracestate)
}, input.TraceContext.Traceparent, input.TraceContext.Tracestate)
if err != nil {
s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
return model.Attachment{}, mapDependencyError(err)
}
return queued, nil
}
func (s *Service) deletePending(attachmentID, stagingKey string) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) {
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout)
defer cancel()
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
}
}
@@ -15,7 +15,7 @@ import (
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func TestUploadLifecycle(t *testing.T) {
+59 -24
View File
@@ -18,10 +18,16 @@ import (
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
type Repository interface {
@@ -29,7 +35,7 @@ type Repository interface {
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error)
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
}
@@ -50,10 +56,11 @@ type ImageProcessor interface {
}
type Metrics interface {
ObserveStarted()
ObserveFinished(string, time.Duration)
ObserveRetry()
ObserveReclaimed(int)
ObserveStarted(context.Context)
ObserveFinished(context.Context, string, time.Duration)
ObserveTimeToReady(context.Context, time.Duration)
ObserveRetry(context.Context)
ObserveReclaimed(context.Context, int)
}
type Config struct {
@@ -166,7 +173,7 @@ func (w *Worker) Run(ctx context.Context) error {
}
return err
}
w.metrics.ObserveReclaimed(len(deliveries))
w.metrics.ObserveReclaimed(ctx, len(deliveries))
w.processBatch(ctx, deliveries)
reclaimCursor = next
if reclaimCursor == "" {
@@ -189,25 +196,48 @@ func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Deli
go func(delivery redisstream.Delivery) {
defer group.Done()
defer func() { <-limit }()
if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil {
w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err)
}
_ = w.ProcessDelivery(ctx, delivery, FaultNone)
}(delivery)
}
group.Wait()
}
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error {
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) {
started := w.now()
w.metrics.ObserveStarted()
result := "transient_error"
defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }()
if delivery.DecodeError != nil {
w.metrics.ObserveStarted(ctx)
defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }()
result = "invalid_job"
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
}
envelope := delivery.Envelope
ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{
Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate,
})
ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(
attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"),
attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()),
attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt),
attribute.Int("pipeline.version", envelope.PipelineVersion),
))
w.metrics.ObserveStarted(ctx)
defer func() {
w.metrics.ObserveFinished(ctx, result, w.now().Sub(started))
span.SetAttributes(attribute.String("job.outcome", result))
if resultErr != nil {
span.RecordError(resultErr)
span.SetStatus(codes.Error, result)
if ctx.Err() == nil {
w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr)
}
} else {
w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result)
}
span.End()
}()
lease, err := w.repository.ClaimAttachmentJob(
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
w.config.WorkerID, w.config.LeaseDuration,
@@ -246,7 +276,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
}
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
}
}
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
@@ -260,6 +290,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
return err
}
result = "ready"
w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt))
return nil
}
@@ -444,7 +475,10 @@ func (w *Worker) resolveProcessingFailure(
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
}
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt)
traceContext := telemetry.ContextCarrier(ctx)
mutation, err := w.repository.RetryAttachmentJob(
ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate,
)
if err != nil {
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
}
@@ -454,7 +488,7 @@ func (w *Worker) resolveProcessingFailure(
if mutation != model.JobMutationApplied {
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
}
w.metrics.ObserveRetry()
w.metrics.ObserveRetry(ctx)
return "retry", delivery.Ack(ctx)
}
@@ -502,7 +536,8 @@ func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
type nopMetrics struct{}
func (nopMetrics) ObserveStarted() {}
func (nopMetrics) ObserveFinished(string, time.Duration) {}
func (nopMetrics) ObserveRetry() {}
func (nopMetrics) ObserveReclaimed(int) {}
func (nopMetrics) ObserveStarted(context.Context) {}
func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {}
func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {}
func (nopMetrics) ObserveRetry(context.Context) {}
func (nopMetrics) ObserveReclaimed(context.Context, int) {}
@@ -9,9 +9,9 @@ import (
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
)
func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) {
+12 -7
View File
@@ -13,16 +13,21 @@ import (
"unicode"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
started := time.Now()
result := "client_error"
defer func() {
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
attributes := metric.WithAttributes(attribute.String("result", result))
s.metrics.attachmentUploads.Add(r.Context(), 1, attributes)
s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes)
}()
if r.Method != http.MethodPost {
@@ -47,7 +52,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
ChatID: chatID, OwnerID: user.ID, Filename: filename,
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")),
TraceContext: telemetry.ContextCarrier(r.Context()),
})
if err != nil {
if errors.Is(err, model.ErrCapacity) {
@@ -59,7 +64,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
return attachmentError(err)
}
result = "success"
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes)
writeJSON(w, http.StatusAccepted, uploaded)
return nil
}
@@ -128,7 +133,7 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, content.Body); err != nil {
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err)
}
return nil
}
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"net/http"
"strings"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"time"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
@@ -229,7 +229,7 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
return unavailable()
}
if replay {
s.metrics.replays.Inc()
s.metrics.replays.Add(r.Context(), 1)
}
writeJSON(w, http.StatusCreated, message)
return nil
@@ -4,8 +4,8 @@ import (
"context"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type UserRepository interface {
+41 -70
View File
@@ -1,14 +1,15 @@
package httpapi
import (
"context"
"net/http"
"strconv"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
var (
@@ -17,80 +18,50 @@ var (
)
type metrics struct {
requests *prometheus.CounterVec
duration *prometheus.HistogramVec
inFlight prometheus.Gauge
replays prometheus.Counter
attachmentUploads *prometheus.CounterVec
attachmentDuration prometheus.Histogram
attachmentBytes prometheus.Counter
requests metric.Int64Counter
duration metric.Float64Histogram
inFlight metric.Int64UpDownCounter
replays metric.Int64Counter
attachmentUploads metric.Int64Counter
attachmentDuration metric.Float64Histogram
attachmentBytes metric.Int64Counter
handler http.Handler
}
func newMetrics() *metrics {
registry := prometheus.NewRegistry()
result := &metrics{
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "requests_total",
Help: "Completed HTTP requests.",
}, []string{"method", "route", "status_class"}),
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "request_duration_seconds",
Help: "HTTP request duration in seconds.",
Buckets: requestDurationBuckets,
}, []string{"method", "route"}),
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "requests_in_flight",
Help: "HTTP requests currently being served.",
}),
replays: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "idempotency_replays_total",
Help: "Message requests served from an idempotency record.",
}),
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_uploads_total",
Help: "Completed attachment uploads by result.",
}, []string{"result"}),
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "messenger",
Name: "attachment_upload_duration_seconds",
Help: "Attachment upload duration in seconds.",
Buckets: uploadDurationBuckets,
}),
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_upload_bytes_total",
Help: "Bytes accepted by successful attachment uploads.",
}),
func newMetrics(meter metric.Meter, handler http.Handler) *metrics {
if meter == nil {
meter = otel.Meter("messenger-api")
}
registry.MustRegister(
result.requests,
result.duration,
result.inFlight,
result.replays,
result.attachmentUploads,
result.attachmentDuration,
result.attachmentBytes,
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
if handler == nil {
handler = http.NotFoundHandler()
}
requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests."))
duration, _ := meter.Float64Histogram("messenger_http_request_duration",
metric.WithDescription("HTTP request duration."), metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(requestDurationBuckets...),
)
result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
return result
inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served."))
replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record."))
attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result."))
attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration",
metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(uploadDurationBuckets...),
)
attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By"))
return &metrics{
requests: requests, duration: duration, inFlight: inFlight, replays: replays,
attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration,
attachmentBytes: attachmentBytes, handler: handler,
}
}
func (m *metrics) observe(r *http.Request, status int, started time.Time) {
route := routeTemplate(r.URL.Path)
m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc()
m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds())
func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) {
attributes := metric.WithAttributes(
attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)),
attribute.String("status_class", strconv.Itoa(status/100)+"xx"),
)
m.requests.Add(ctx, 1, attributes)
m.duration.Record(ctx, time.Since(started).Seconds(), attributes)
}
func routeTemplate(path string) string {
+1 -1
View File
@@ -1,7 +1,7 @@
package httpapi
import (
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"github.com/google/uuid"
)
+31 -6
View File
@@ -6,6 +6,16 @@ import (
"net/http"
"sync/atomic"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
type Dependencies struct {
@@ -17,6 +27,8 @@ type Dependencies struct {
Passwords PasswordService
Database HealthChecker
Objects HealthChecker
Meter metric.Meter
Metrics http.Handler
}
type Config struct {
@@ -67,7 +79,8 @@ func NewServer(
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort,
metrics: newMetrics(dependencies.Meter, dependencies.Metrics),
}
s.ready.Store(true)
return s
@@ -82,27 +95,39 @@ func (s *Server) Handler() http.Handler {
}
func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
tracer := otel.Tracer("messenger-api/http")
route := routeTemplate(r.URL.Path)
requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer),
trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route)))
defer span.End()
r = r.WithContext(requestContext)
w.Header().Set("X-Instance-Id", s.instanceID)
requestID, err := newID()
if err != nil {
http.Error(w, "server unavailable", http.StatusServiceUnavailable)
return
}
span.SetAttributes(attribute.String("request.id", requestID))
w.Header().Set("X-Request-Id", requestID)
w.Header().Set("Cache-Control", "no-store")
wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK}
started := time.Now()
s.metrics.inFlight.Inc()
s.metrics.inFlight.Add(r.Context(), 1)
defer func() {
s.metrics.inFlight.Dec()
s.metrics.inFlight.Add(r.Context(), -1)
if recovered := recover(); recovered != nil {
s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path)
s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route)
if wrapped.status == http.StatusOK {
writeError(wrapped, requestID, internalError())
}
}
s.metrics.observe(r, wrapped.status, started)
s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
s.metrics.observe(r.Context(), r, wrapped.status, started)
span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status))
if wrapped.status >= http.StatusInternalServerError {
span.SetStatus(codes.Error, http.StatusText(wrapped.status))
}
s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
}()
if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
+2 -2
View File
@@ -14,8 +14,8 @@ import (
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
var errDependency = errors.New("dependency unavailable")
+51 -2
View File
@@ -13,7 +13,12 @@ import (
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
var ErrNotFound = errors.New("object not found")
@@ -85,30 +90,38 @@ func (s *Store) Ping(ctx context.Context) error {
}
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
ctx, span := s.startSpan(ctx, "PUT")
defer span.End()
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
_, err := s.client.PutObject(
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
)
if err != nil {
recordSpanError(span, err)
return fmt.Errorf("put S3 object: %w", err)
}
return nil
}
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
ctx, span := s.startSpan(ctx, "GET")
ctx, cancel := context.WithTimeout(ctx, s.timeout)
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
cancel()
recordSpanError(span, err)
span.End()
return nil, objectError("get S3 object", err)
}
if _, err := object.Stat(); err != nil {
cancel()
_ = object.Close()
recordSpanError(span, err)
span.End()
return nil, objectError("stat S3 object", err)
}
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil
}
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
@@ -122,9 +135,12 @@ func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error
}
func (s *Store) Delete(ctx context.Context, key string) error {
ctx, span := s.startSpan(ctx, "DELETE")
defer span.End()
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
recordSpanError(span, err)
return objectError("delete S3 object", err)
}
return nil
@@ -161,6 +177,28 @@ type timeoutReader struct {
cancel context.CancelFunc
}
type tracedReader struct {
io.ReadCloser
span trace.Span
}
func (r *tracedReader) Read(buffer []byte) (int, error) {
read, err := r.ReadCloser.Read(buffer)
if err != nil && !errors.Is(err, io.EOF) {
recordSpanError(r.span, err)
}
return read, err
}
func (r *tracedReader) Close() error {
err := r.ReadCloser.Close()
if err != nil {
recordSpanError(r.span, err)
}
r.span.End()
return err
}
func (r *timeoutReader) Close() error {
err := r.ReadCloser.Close()
r.cancel()
@@ -174,3 +212,14 @@ func objectError(action string, err error) error {
}
return fmt.Errorf("%s: %w", action, err)
}
func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) {
return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation),
attribute.String("s3.bucket.name", s.bucket)))
}
func recordSpanError(span trace.Span, err error) {
span.RecordError(err)
span.SetStatus(codes.Error, "S3 operation failed")
}
+43 -13
View File
@@ -6,7 +6,13 @@ import (
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
var errLeaseLost = errors.New("outbox lease lost")
@@ -24,9 +30,9 @@ type Publisher interface {
}
type Metrics interface {
ObservePublished()
ObservePublishFailure()
ObserveRecovered(int)
ObservePublished(context.Context)
ObservePublishFailure(context.Context)
ObserveRecovered(context.Context, int)
}
type Config struct {
@@ -156,15 +162,32 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
if err := ctx.Err(); err != nil {
return published, nil
}
_, err := d.publisher.PublishConfirmed(ctx, event.Payload)
publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload)
publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish",
trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes(
attribute.String("messaging.system", "redis"),
attribute.String("messaging.destination.name", "attachment-jobs"),
attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt),
))
payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload)
if injectErr != nil {
span.RecordError(injectErr)
span.SetStatus(codes.Error, "inject trace context")
span.End()
return published, fmt.Errorf("inject outbox trace context: %w", injectErr)
}
_, err := d.publisher.PublishConfirmed(publishContext, payload)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "publish failed")
span.End()
if ctx.Err() != nil {
return published, nil
}
d.metrics.ObservePublishFailure()
d.metrics.ObservePublishFailure(publishContext)
nextAttemptAt := d.now().Add(d.config.RetryDelay)
released, releaseErr := d.store.ReleaseOutboxEvent(
ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
)
if releaseErr != nil {
return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr)
@@ -174,14 +197,21 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
}
continue
}
marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken)
marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "mark published")
span.End()
return published, fmt.Errorf("mark outbox event published: %w", err)
}
if !marked {
span.SetStatus(codes.Error, "outbox lease lost")
span.End()
return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost)
}
d.metrics.ObservePublished()
span.SetAttributes(attribute.String("messaging.operation.type", "publish"))
span.End()
d.metrics.ObservePublished(publishContext)
published++
}
return published, nil
@@ -192,12 +222,12 @@ func (d *Dispatcher) RecoverExpired(ctx context.Context) error {
if err != nil {
return fmt.Errorf("recover expired outbox leases: %w", err)
}
d.metrics.ObserveRecovered(recovered)
d.metrics.ObserveRecovered(ctx, recovered)
return nil
}
type nopMetrics struct{}
func (nopMetrics) ObservePublished() {}
func (nopMetrics) ObservePublishFailure() {}
func (nopMetrics) ObserveRecovered(int) {}
func (nopMetrics) ObservePublished(context.Context) {}
func (nopMetrics) ObservePublishFailure(context.Context) {}
func (nopMetrics) ObserveRecovered(context.Context, int) {}
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"testing"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"github.com/stretchr/testify/require"
)
+4 -4
View File
@@ -10,7 +10,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const attachmentJobTopic = "attachment.jobs"
@@ -447,6 +447,8 @@ func (s *Store) RetryAttachmentJob(
lease model.AttachmentJobLease,
errorCode string,
nextAttemptAt time.Time,
traceparent string,
tracestate string,
) (model.JobMutationResult, error) {
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
return "", model.ErrInvalidInput
@@ -483,9 +485,7 @@ func (s *Store) RetryAttachmentJob(
if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil {
return "", fmt.Errorf("requeue attachment: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
return "", err
}
if err := tx.Commit(ctx); err != nil {
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Store) RenewAttachmentJobLease(
+1 -1
View File
@@ -9,7 +9,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Store) RecoverExpiredAttachmentJobs(
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const attachmentColumns = `
+1 -1
View File
@@ -1,6 +1,6 @@
package postgres
import "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
var (
ErrConflict = model.ErrConflict
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const expiredIdempotencyCleanupBatch = 100
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Store) ClaimOutboxEvents(
+1 -1
View File
@@ -10,7 +10,7 @@ import (
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type Store struct {
+4 -4
View File
@@ -15,10 +15,10 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
)
type PostgresSuite struct {
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type MetadataStore interface {
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"github.com/valkey-io/valkey-go"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const (
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
)
type SessionSuite struct {
+39
View File
@@ -0,0 +1,39 @@
package telemetry
import (
"context"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
var (
connectionStateUsed = attribute.String("state", "used")
connectionStateIdle = attribute.String("state", "idle")
connectionStateConnecting = attribute.String("state", "connecting")
)
func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error {
_, err := meter.Int64ObservableGauge("messenger_db_pool_connections",
metric.WithDescription("PostgreSQL pool connections by state."),
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
stats := pool.Stat()
observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed))
observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle))
observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting))
return nil
}),
)
if err != nil {
return err
}
_, err = meter.Int64ObservableCounter("messenger_db_pool_waited",
metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."),
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
observer.Observe(pool.Stat().EmptyAcquireCount())
return nil
}),
)
return err
}
+74
View File
@@ -0,0 +1,74 @@
package telemetry
import (
"context"
"log/slog"
"strings"
"go.opentelemetry.io/otel/trace"
)
type correlationHandler struct {
stdout slog.Handler
otel slog.Handler
}
func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler {
return correlationHandler{stdout: stdout, otel: otel}
}
func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool {
return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level)
}
func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error {
span := trace.SpanContextFromContext(ctx)
if span.IsValid() {
record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String()))
}
record = sanitizedRecord(record)
if err := h.stdout.Handle(ctx, record); err != nil {
return err
}
return h.otel.Handle(ctx, record)
}
func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler {
attributes = sanitizedAttributes(attributes)
return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)}
}
func (h correlationHandler) WithGroup(name string) slog.Handler {
return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)}
}
func sanitizedRecord(record slog.Record) slog.Record {
copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC)
record.Attrs(func(attribute slog.Attr) bool {
if !sensitiveKey(attribute.Key) {
copy.AddAttrs(attribute)
}
return true
})
return copy
}
func sanitizedAttributes(attributes []slog.Attr) []slog.Attr {
result := make([]slog.Attr, 0, len(attributes))
for _, attribute := range attributes {
if !sensitiveKey(attribute.Key) {
result = append(result, attribute)
}
}
return result
}
func sensitiveKey(key string) bool {
key = strings.ToLower(key)
for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} {
if strings.Contains(key, marker) {
return true
}
}
return false
}
+70
View File
@@ -0,0 +1,70 @@
package telemetry
import (
"context"
"encoding/json"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
)
type MessageCarrier struct {
Traceparent string
Tracestate string
}
func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context {
return otel.GetTextMapPropagator().Extract(ctx, carrier)
}
func Inject(ctx context.Context, carrier propagation.TextMapCarrier) {
otel.GetTextMapPropagator().Inject(ctx, carrier)
}
func ContextCarrier(ctx context.Context) MessageCarrier {
carrier := propagation.MapCarrier{}
Inject(ctx, carrier)
return MessageCarrier{
Traceparent: carrier.Get("traceparent"),
Tracestate: carrier.Get("tracestate"),
}
}
func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context {
headers := propagation.MapCarrier{
"traceparent": carrier.Traceparent,
"tracestate": carrier.Tracestate,
}
return Extract(ctx, headers)
}
func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context {
var carrier struct {
Traceparent string `json:"traceparent"`
Tracestate string `json:"tracestate"`
}
if json.Unmarshal(payload, &carrier) != nil {
return ctx
}
return ExtractMessage(ctx, MessageCarrier(carrier))
}
func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) {
var value map[string]json.RawMessage
if err := json.Unmarshal(payload, &value); err != nil {
return nil, err
}
carrier := ContextCarrier(ctx)
setJSONField(value, "traceparent", carrier.Traceparent)
setJSONField(value, "tracestate", carrier.Tracestate)
return json.Marshal(value)
}
func setJSONField(value map[string]json.RawMessage, key, field string) {
if field == "" {
delete(value, key)
return
}
encoded, _ := json.Marshal(field)
value[key] = encoded
}
+33
View File
@@ -0,0 +1,33 @@
package telemetry
import (
"context"
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) {
otel.SetTextMapPropagator(propagation.TraceContext{})
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
require.NoError(t, err)
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
require.NoError(t, err)
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
}))
payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`))
require.NoError(t, err)
var value map[string]any
require.NoError(t, json.Unmarshal(payload, &value))
require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"])
require.NotContains(t, value, "baggage")
extracted := ExtractJSONMessage(context.Background(), payload)
require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID())
}
+252
View File
@@ -0,0 +1,252 @@
package telemetry
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/contrib/bridges/otelslog"
otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
otelprometheus "go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/log"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
)
const (
defaultEnvironment = "lab"
defaultNamespace = "mtusi-highload"
)
type Config struct {
ServiceName string
Version string
InstanceID string
Endpoint string
Environment string
Namespace string
Insecure bool
}
type Runtime struct {
traces *trace.TracerProvider
metrics *metric.MeterProvider
logs *log.LoggerProvider
logger *slog.Logger
handler http.Handler
}
func ConfigFromEnvironment(serviceName, instanceID string) Config {
resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES"))
return Config{
ServiceName: serviceName,
Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"),
InstanceID: instanceID,
Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")),
Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment),
Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace),
Insecure: true,
}
}
func parseResourceAttributes(raw string) map[string]string {
result := make(map[string]string)
for _, pair := range strings.Split(raw, ",") {
key, value, found := strings.Cut(pair, "=")
if found && strings.TrimSpace(key) != "" {
result[strings.TrimSpace(key)] = strings.TrimSpace(value)
}
}
return result
}
func firstValue(values ...string) string {
for _, value := range values {
if value = strings.TrimSpace(value); value != "" {
return value
}
}
return ""
}
func New(ctx context.Context, config Config) (*Runtime, error) {
if err := config.validate(); err != nil {
return nil, err
}
res, err := newResource(ctx, config)
if err != nil {
return nil, err
}
registry := prometheus.NewRegistry()
prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry))
if err != nil {
return nil, fmt.Errorf("create Prometheus reader: %w", err)
}
traceOptions := []trace.TracerProviderOption{trace.WithResource(res)}
metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)}
logOptions := []log.LoggerProviderOption{log.WithResource(res)}
if config.Endpoint != "" {
traceExporter, err := newTraceExporter(ctx, config)
if err != nil {
return nil, err
}
metricExporter, err := newMetricExporter(ctx, config)
if err != nil {
return nil, err
}
logExporter, err := newLogExporter(ctx, config)
if err != nil {
return nil, err
}
traceOptions = append(traceOptions, trace.WithBatcher(traceExporter,
trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512),
trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second),
))
metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader(
metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second),
)))
logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter,
log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512),
log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second),
)))
}
traces := trace.NewTracerProvider(traceOptions...)
metrics := metric.NewMeterProvider(metricOptions...)
logs := log.NewLoggerProvider(logOptions...)
propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{})
otel.SetTracerProvider(traces)
otel.SetMeterProvider(metrics)
otel.SetTextMapPropagator(propagator)
if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil {
return nil, fmt.Errorf("start Go runtime metrics: %w", err)
}
stdout := slog.NewJSONHandler(os.Stdout, nil)
otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs))
logger := slog.New(newCorrelationHandler(stdout, otelHandler))
registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
return &Runtime{
traces: traces, metrics: metrics, logs: logs, logger: logger,
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
}, nil
}
func (c Config) validate() error {
switch {
case strings.TrimSpace(c.ServiceName) == "":
return errors.New("telemetry service name is required")
case strings.TrimSpace(c.InstanceID) == "":
return errors.New("telemetry instance ID is required")
case strings.TrimSpace(c.Environment) == "":
return errors.New("telemetry environment is required")
case strings.TrimSpace(c.Namespace) == "":
return errors.New("telemetry service namespace is required")
default:
return nil
}
}
func newResource(ctx context.Context, config Config) (*resource.Resource, error) {
res, err := resource.New(ctx,
resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(),
resource.WithAttributes(
semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version),
semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace),
semconv.DeploymentEnvironmentName(config.Environment),
),
)
if err != nil {
return nil, fmt.Errorf("create telemetry resource: %w", err)
}
return res, nil
}
func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) {
var options []otlptracegrpc.Option
if strings.Contains(config.Endpoint, "://") {
options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint))
} else {
options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint))
}
if config.Insecure {
options = append(options, otlptracegrpc.WithInsecure())
}
exporter, err := otlptracegrpc.New(ctx, options...)
if err != nil {
return nil, fmt.Errorf("create OTLP trace exporter: %w", err)
}
return exporter, nil
}
func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) {
var options []otlpmetricgrpc.Option
if strings.Contains(config.Endpoint, "://") {
options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint))
} else {
options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint))
}
if config.Insecure {
options = append(options, otlpmetricgrpc.WithInsecure())
}
exporter, err := otlpmetricgrpc.New(ctx, options...)
if err != nil {
return nil, fmt.Errorf("create OTLP metric exporter: %w", err)
}
return exporter, nil
}
func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) {
var options []otlploggrpc.Option
if strings.Contains(config.Endpoint, "://") {
options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint))
} else {
options = append(options, otlploggrpc.WithEndpoint(config.Endpoint))
}
if config.Insecure {
options = append(options, otlploggrpc.WithInsecure())
}
exporter, err := otlploggrpc.New(ctx, options...)
if err != nil {
return nil, fmt.Errorf("create OTLP log exporter: %w", err)
}
return exporter, nil
}
func (r *Runtime) Logger() *slog.Logger {
return r.logger
}
func (r *Runtime) MeterProvider() *metric.MeterProvider {
return r.metrics
}
func (r *Runtime) MetricsHandler() http.Handler {
return r.handler
}
func (r *Runtime) Shutdown(ctx context.Context) error {
var result error
for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} {
if err := shutdown(ctx); err != nil {
result = errors.Join(result, err)
}
}
return result
}
+59
View File
@@ -0,0 +1,59 @@
package telemetry
import (
"context"
"log/slog"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace"
)
func TestConfigValidation(t *testing.T) {
valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"}
require.NoError(t, valid.validate())
invalid := valid
invalid.ServiceName = ""
require.EqualError(t, invalid.validate(), "telemetry service name is required")
}
func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) {
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
require.NoError(t, err)
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
require.NoError(t, err)
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
}))
first := &recordHandler{}
second := &recordHandler{}
handler := newCorrelationHandler(first, second)
record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0)
record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret"))
require.NoError(t, handler.Handle(ctx, record))
for _, captured := range []*recordHandler{first, second} {
require.Equal(t, traceID.String(), captured.attributes["trace_id"])
require.Equal(t, spanID.String(), captured.attributes["span_id"])
require.Equal(t, "http_request_completed", captured.attributes["event"])
require.NotContains(t, captured.attributes, "authorization")
}
}
type recordHandler struct {
attributes map[string]string
}
func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true }
func (h *recordHandler) Handle(_ context.Context, record slog.Record) error {
h.attributes = make(map[string]string)
record.Attrs(func(attribute slog.Attr) bool {
h.attributes[attribute.Key] = attribute.Value.String()
return true
})
return nil
}
func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
func (h *recordHandler) WithGroup(string) slog.Handler { return h }