feat: add OpenTelemetry instrumentation
This commit is contained in:
@@ -17,7 +17,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type apiState struct {
|
||||
|
||||
@@ -21,15 +21,15 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/bootstrap"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/logger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
module gitea.privcloud.tech/dizi/mtusi2026lab5
|
||||
module gitea.privcloud.tech/dizi/mtusi2026lab6
|
||||
|
||||
go 1.26.0
|
||||
|
||||
@@ -12,19 +12,36 @@ require (
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/valkey-io/valkey-go v1.0.78
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.60.0
|
||||
go.opentelemetry.io/otel/metric v1.38.0
|
||||
go.opentelemetry.io/otel/sdk v1.38.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0
|
||||
go.opentelemetry.io/otel/trace v1.38.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/sync v0.23.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -39,7 +56,8 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/prometheus/otlptranslator v0.0.2 // indirect
|
||||
github.com/prometheus/procfs v0.17.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
@@ -48,11 +66,18 @@ require (
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/log v0.14.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -4,6 +4,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -21,12 +23,23 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
|
||||
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -70,12 +83,14 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ=
|
||||
github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI=
|
||||
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
|
||||
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
@@ -101,6 +116,40 @@ github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
|
||||
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4=
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE=
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk=
|
||||
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
|
||||
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
@@ -117,6 +166,14 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
@@ -3,8 +3,8 @@ package bootstrap
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
)
|
||||
|
||||
func RunMigrations() error {
|
||||
|
||||
@@ -14,15 +14,16 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func RunDispatcher(logger *slog.Logger) error {
|
||||
@@ -30,11 +31,23 @@ func RunDispatcher(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instanceID, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
observability, logger, err := startTelemetry("messenger-dispatcher", instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer shutdownTelemetry(observability)
|
||||
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := newRedisClient(settings.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -51,7 +64,7 @@ func RunDispatcher(logger *slog.Logger) error {
|
||||
if err := broker.Ensure(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0)
|
||||
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
|
||||
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
|
||||
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
|
||||
@@ -70,11 +83,23 @@ func RunWorker(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
observability, logger, err := startTelemetry("messenger-worker", hostname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer shutdownTelemetry(observability)
|
||||
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil {
|
||||
return err
|
||||
}
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
@@ -88,10 +113,6 @@ func RunWorker(logger *slog.Logger) error {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
consumer := hostname + "-" + uuid.NewString()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
||||
@@ -108,7 +129,7 @@ func RunWorker(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency)
|
||||
processorWorker, err := worker.New(
|
||||
logger, postgres.New(pool), broker, objects, processor, metrics,
|
||||
worker.Config{
|
||||
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
|
||||
)
|
||||
|
||||
type reconciliationOutput struct {
|
||||
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
)
|
||||
|
||||
func RunAttachmentReplay(arguments []string, output io.Writer) error {
|
||||
|
||||
@@ -9,14 +9,15 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/httpapi"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func RunServer(logger *slog.Logger) error {
|
||||
@@ -28,11 +29,19 @@ func RunServer(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
observability, logger, err := startTelemetry("messenger-api", instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer shutdownTelemetry(observability)
|
||||
pool, err := database.Open(context.Background(), databaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil {
|
||||
return err
|
||||
}
|
||||
sessions, err := session.New(settings.CacheURL)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -62,6 +71,7 @@ func RunServer(logger *slog.Logger) error {
|
||||
app := httpapi.NewServer(logger, httpapi.Dependencies{
|
||||
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
|
||||
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
|
||||
Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(),
|
||||
}, httpapi.Config{
|
||||
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
|
||||
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) {
|
||||
runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return runtime, runtime.Logger(), nil
|
||||
}
|
||||
|
||||
func shutdownTelemetry(runtime *telemetry.Runtime) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = runtime.Shutdown(ctx)
|
||||
}
|
||||
@@ -11,7 +11,12 @@ import (
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -103,6 +108,10 @@ func isBusyGroup(err error) bool {
|
||||
}
|
||||
|
||||
func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) {
|
||||
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"),
|
||||
attribute.String("messaging.destination.name", b.config.Stream)))
|
||||
defer span.End()
|
||||
envelope, err := DecodeEnvelope(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -117,12 +126,19 @@ func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string,
|
||||
Values: map[string]any{payloadField: string(payload)},
|
||||
}).Result()
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "XADD failed")
|
||||
return "", fmt.Errorf("publish redis stream job: %w", err)
|
||||
}
|
||||
span.SetAttributes(attribute.String("messaging.message.id", id))
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
|
||||
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"),
|
||||
attribute.String("messaging.destination.name", b.config.Stream)))
|
||||
defer span.End()
|
||||
streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: b.config.Group,
|
||||
Consumer: b.config.Consumer,
|
||||
@@ -134,6 +150,8 @@ func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "XREADGROUP failed")
|
||||
return nil, fmt.Errorf("read redis stream jobs: %w", err)
|
||||
}
|
||||
return b.deliveries(streams)
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/migrations"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/migrations"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -40,6 +40,7 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse database configuration: %w", err)
|
||||
}
|
||||
poolConfig.ConnConfig.Tracer = queryTracer{}
|
||||
poolConfig.MinConns = config.MinConnections
|
||||
poolConfig.MaxConns = config.MaxConnections
|
||||
poolConfig.MaxConnLifetime = config.MaxConnectionAge
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type queryTracer struct{}
|
||||
|
||||
func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
|
||||
ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system", "postgresql")))
|
||||
return ctx
|
||||
}
|
||||
|
||||
func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
|
||||
span := trace.SpanFromContext(ctx)
|
||||
if data.Err != nil {
|
||||
span.RecordError(data.Err)
|
||||
span.SetStatus(codes.Error, "query failed")
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
@@ -1,104 +1,106 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60}
|
||||
|
||||
type Metrics struct {
|
||||
workerInFlight prometheus.Gauge
|
||||
workerJobs *prometheus.CounterVec
|
||||
workerDuration *prometheus.HistogramVec
|
||||
workerRetries prometheus.Counter
|
||||
workerReclaimed prometheus.Counter
|
||||
outboxPublished prometheus.Counter
|
||||
outboxPublishFailure prometheus.Counter
|
||||
outboxRecovered prometheus.Counter
|
||||
meter metric.Meter
|
||||
workerInFlight metric.Int64UpDownCounter
|
||||
workerJobs metric.Int64Counter
|
||||
workerDuration metric.Float64Histogram
|
||||
workerTimeToReady metric.Float64Histogram
|
||||
workerRetries metric.Int64Counter
|
||||
workerReclaimed metric.Int64Counter
|
||||
outboxPublished metric.Int64Counter
|
||||
outboxPublishFailure metric.Int64Counter
|
||||
outboxRecovered metric.Int64Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func NewMetrics() *Metrics {
|
||||
registry := prometheus.NewRegistry()
|
||||
metrics := &Metrics{
|
||||
workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight",
|
||||
Help: "Attachment jobs currently processed by this worker.",
|
||||
}),
|
||||
workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total",
|
||||
Help: "Completed attachment job deliveries by result.",
|
||||
}, []string{"result"}),
|
||||
workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds",
|
||||
Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets,
|
||||
}, []string{"result"}),
|
||||
workerRetries: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total",
|
||||
Help: "Attachment jobs scheduled for another attempt.",
|
||||
}),
|
||||
workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total",
|
||||
Help: "Redis Stream deliveries reclaimed after their idle timeout.",
|
||||
}),
|
||||
outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "published_total",
|
||||
Help: "Outbox events confirmed by Redis Streams.",
|
||||
}),
|
||||
outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total",
|
||||
Help: "Outbox events whose Redis Streams publish failed.",
|
||||
}),
|
||||
outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total",
|
||||
Help: "Expired outbox leases returned for publication.",
|
||||
}),
|
||||
func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics {
|
||||
if meter == nil {
|
||||
meter = otel.Meter("messenger-pipeline")
|
||||
}
|
||||
registry.MustRegister(
|
||||
metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration,
|
||||
metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished,
|
||||
metrics.outboxPublishFailure, metrics.outboxRecovered,
|
||||
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
if handler == nil {
|
||||
handler = http.NotFoundHandler()
|
||||
}
|
||||
workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker."))
|
||||
workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result."))
|
||||
workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration",
|
||||
metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
|
||||
)
|
||||
metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
|
||||
return metrics
|
||||
workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready",
|
||||
metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
|
||||
)
|
||||
workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt."))
|
||||
workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout."))
|
||||
outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams."))
|
||||
outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed."))
|
||||
outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication."))
|
||||
if workerCapacity > 0 {
|
||||
_, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity",
|
||||
metric.WithDescription("Maximum concurrent attachment jobs for this worker."),
|
||||
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
|
||||
observer.Observe(int64(workerCapacity))
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
}
|
||||
return &Metrics{
|
||||
meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs,
|
||||
workerDuration: workerDuration, workerTimeToReady: workerTimeToReady,
|
||||
workerRetries: workerRetries, workerReclaimed: workerReclaimed,
|
||||
outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure,
|
||||
outboxRecovered: outboxRecovered, handler: handler,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) Handler() http.Handler {
|
||||
return m.handler
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveStarted() {
|
||||
m.workerInFlight.Inc()
|
||||
func (m *Metrics) ObserveStarted(ctx context.Context) {
|
||||
m.workerInFlight.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveFinished(result string, duration time.Duration) {
|
||||
m.workerInFlight.Dec()
|
||||
m.workerJobs.WithLabelValues(result).Inc()
|
||||
m.workerDuration.WithLabelValues(result).Observe(duration.Seconds())
|
||||
func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) {
|
||||
attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result))
|
||||
m.workerInFlight.Add(ctx, -1)
|
||||
m.workerJobs.Add(ctx, 1, attributes)
|
||||
m.workerDuration.Record(ctx, duration.Seconds(), attributes)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRetry() {
|
||||
m.workerRetries.Inc()
|
||||
func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) {
|
||||
m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process")))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveReclaimed(count int) {
|
||||
m.workerReclaimed.Add(float64(count))
|
||||
func (m *Metrics) ObserveRetry(ctx context.Context) {
|
||||
m.workerRetries.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublished() {
|
||||
m.outboxPublished.Inc()
|
||||
func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) {
|
||||
m.workerReclaimed.Add(ctx, int64(count))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublishFailure() {
|
||||
m.outboxPublishFailure.Inc()
|
||||
func (m *Metrics) ObservePublished(ctx context.Context) {
|
||||
m.outboxPublished.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRecovered(count int) {
|
||||
m.outboxRecovered.Add(float64(count))
|
||||
func (m *Metrics) ObservePublishFailure(ctx context.Context) {
|
||||
m.outboxPublishFailure.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRecovered(ctx context.Context, count int) {
|
||||
m.outboxRecovered.Add(ctx, int64(count))
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
@@ -53,8 +54,7 @@ type Upload struct {
|
||||
ContentType string
|
||||
ContentLength int64
|
||||
Body io.Reader
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
TraceContext telemetry.MessageCarrier
|
||||
}
|
||||
|
||||
type Content struct {
|
||||
@@ -112,27 +112,27 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil {
|
||||
s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err)
|
||||
s.deletePending(pending.ID, stagingKey)
|
||||
s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err)
|
||||
s.deletePending(ctx, pending.ID, stagingKey)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
createdAt := time.Now().UTC()
|
||||
queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion,
|
||||
Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt,
|
||||
}, input.Traceparent, input.Tracestate)
|
||||
}, input.TraceContext.Traceparent, input.TraceContext.Tracestate)
|
||||
if err != nil {
|
||||
s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
|
||||
s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
return queued, nil
|
||||
}
|
||||
|
||||
func (s *Service) deletePending(attachmentID, stagingKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||
s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func TestUploadLifecycle(t *testing.T) {
|
||||
|
||||
@@ -18,10 +18,16 @@ import (
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
@@ -29,7 +35,7 @@ type Repository interface {
|
||||
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
|
||||
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
|
||||
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
|
||||
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
|
||||
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error)
|
||||
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
|
||||
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
|
||||
}
|
||||
@@ -50,10 +56,11 @@ type ImageProcessor interface {
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObserveStarted()
|
||||
ObserveFinished(string, time.Duration)
|
||||
ObserveRetry()
|
||||
ObserveReclaimed(int)
|
||||
ObserveStarted(context.Context)
|
||||
ObserveFinished(context.Context, string, time.Duration)
|
||||
ObserveTimeToReady(context.Context, time.Duration)
|
||||
ObserveRetry(context.Context)
|
||||
ObserveReclaimed(context.Context, int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -166,7 +173,7 @@ func (w *Worker) Run(ctx context.Context) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
w.metrics.ObserveReclaimed(len(deliveries))
|
||||
w.metrics.ObserveReclaimed(ctx, len(deliveries))
|
||||
w.processBatch(ctx, deliveries)
|
||||
reclaimCursor = next
|
||||
if reclaimCursor == "" {
|
||||
@@ -189,25 +196,48 @@ func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Deli
|
||||
go func(delivery redisstream.Delivery) {
|
||||
defer group.Done()
|
||||
defer func() { <-limit }()
|
||||
if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil {
|
||||
w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err)
|
||||
}
|
||||
_ = w.ProcessDelivery(ctx, delivery, FaultNone)
|
||||
}(delivery)
|
||||
}
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error {
|
||||
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) {
|
||||
started := w.now()
|
||||
w.metrics.ObserveStarted()
|
||||
result := "transient_error"
|
||||
defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }()
|
||||
|
||||
if delivery.DecodeError != nil {
|
||||
w.metrics.ObserveStarted(ctx)
|
||||
defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }()
|
||||
result = "invalid_job"
|
||||
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
||||
}
|
||||
envelope := delivery.Envelope
|
||||
ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{
|
||||
Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate,
|
||||
})
|
||||
ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"),
|
||||
attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()),
|
||||
attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt),
|
||||
attribute.Int("pipeline.version", envelope.PipelineVersion),
|
||||
))
|
||||
w.metrics.ObserveStarted(ctx)
|
||||
defer func() {
|
||||
w.metrics.ObserveFinished(ctx, result, w.now().Sub(started))
|
||||
span.SetAttributes(attribute.String("job.outcome", result))
|
||||
if resultErr != nil {
|
||||
span.RecordError(resultErr)
|
||||
span.SetStatus(codes.Error, result)
|
||||
if ctx.Err() == nil {
|
||||
w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr)
|
||||
}
|
||||
} else {
|
||||
w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result)
|
||||
}
|
||||
span.End()
|
||||
}()
|
||||
|
||||
lease, err := w.repository.ClaimAttachmentJob(
|
||||
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
|
||||
w.config.WorkerID, w.config.LeaseDuration,
|
||||
@@ -246,7 +276,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
|
||||
}
|
||||
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
|
||||
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
|
||||
w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
|
||||
w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
|
||||
}
|
||||
}
|
||||
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
|
||||
@@ -260,6 +290,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
|
||||
return err
|
||||
}
|
||||
result = "ready"
|
||||
w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -444,7 +475,10 @@ func (w *Worker) resolveProcessingFailure(
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
}
|
||||
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
|
||||
mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt)
|
||||
traceContext := telemetry.ContextCarrier(ctx)
|
||||
mutation, err := w.repository.RetryAttachmentJob(
|
||||
ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate,
|
||||
)
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
|
||||
}
|
||||
@@ -454,7 +488,7 @@ func (w *Worker) resolveProcessingFailure(
|
||||
if mutation != model.JobMutationApplied {
|
||||
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
|
||||
}
|
||||
w.metrics.ObserveRetry()
|
||||
w.metrics.ObserveRetry(ctx)
|
||||
return "retry", delivery.Ack(ctx)
|
||||
}
|
||||
|
||||
@@ -502,7 +536,8 @@ func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObserveStarted() {}
|
||||
func (nopMetrics) ObserveFinished(string, time.Duration) {}
|
||||
func (nopMetrics) ObserveRetry() {}
|
||||
func (nopMetrics) ObserveReclaimed(int) {}
|
||||
func (nopMetrics) ObserveStarted(context.Context) {}
|
||||
func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {}
|
||||
func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {}
|
||||
func (nopMetrics) ObserveRetry(context.Context) {}
|
||||
func (nopMetrics) ObserveReclaimed(context.Context, int) {}
|
||||
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
)
|
||||
|
||||
func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) {
|
||||
|
||||
@@ -13,16 +13,21 @@ import (
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
started := time.Now()
|
||||
result := "client_error"
|
||||
defer func() {
|
||||
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
|
||||
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
|
||||
attributes := metric.WithAttributes(attribute.String("result", result))
|
||||
s.metrics.attachmentUploads.Add(r.Context(), 1, attributes)
|
||||
s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes)
|
||||
}()
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
@@ -47,7 +52,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
|
||||
ChatID: chatID, OwnerID: user.ID, Filename: filename,
|
||||
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
|
||||
Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")),
|
||||
TraceContext: telemetry.ContextCarrier(r.Context()),
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, model.ErrCapacity) {
|
||||
@@ -59,7 +64,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
return attachmentError(err)
|
||||
}
|
||||
result = "success"
|
||||
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
|
||||
s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes)
|
||||
writeJSON(w, http.StatusAccepted, uploaded)
|
||||
return nil
|
||||
}
|
||||
@@ -128,7 +133,7 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, content.Body); err != nil {
|
||||
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
|
||||
s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
@@ -229,7 +229,7 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
return unavailable()
|
||||
}
|
||||
if replay {
|
||||
s.metrics.replays.Inc()
|
||||
s.metrics.replays.Add(r.Context(), 1)
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, message)
|
||||
return nil
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type UserRepository interface {
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -17,80 +18,50 @@ var (
|
||||
)
|
||||
|
||||
type metrics struct {
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
inFlight prometheus.Gauge
|
||||
replays prometheus.Counter
|
||||
attachmentUploads *prometheus.CounterVec
|
||||
attachmentDuration prometheus.Histogram
|
||||
attachmentBytes prometheus.Counter
|
||||
requests metric.Int64Counter
|
||||
duration metric.Float64Histogram
|
||||
inFlight metric.Int64UpDownCounter
|
||||
replays metric.Int64Counter
|
||||
attachmentUploads metric.Int64Counter
|
||||
attachmentDuration metric.Float64Histogram
|
||||
attachmentBytes metric.Int64Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func newMetrics() *metrics {
|
||||
registry := prometheus.NewRegistry()
|
||||
result := &metrics{
|
||||
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "requests_total",
|
||||
Help: "Completed HTTP requests.",
|
||||
}, []string{"method", "route", "status_class"}),
|
||||
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "request_duration_seconds",
|
||||
Help: "HTTP request duration in seconds.",
|
||||
Buckets: requestDurationBuckets,
|
||||
}, []string{"method", "route"}),
|
||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "requests_in_flight",
|
||||
Help: "HTTP requests currently being served.",
|
||||
}),
|
||||
replays: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "idempotency_replays_total",
|
||||
Help: "Message requests served from an idempotency record.",
|
||||
}),
|
||||
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_uploads_total",
|
||||
Help: "Completed attachment uploads by result.",
|
||||
}, []string{"result"}),
|
||||
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_duration_seconds",
|
||||
Help: "Attachment upload duration in seconds.",
|
||||
Buckets: uploadDurationBuckets,
|
||||
}),
|
||||
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_bytes_total",
|
||||
Help: "Bytes accepted by successful attachment uploads.",
|
||||
}),
|
||||
func newMetrics(meter metric.Meter, handler http.Handler) *metrics {
|
||||
if meter == nil {
|
||||
meter = otel.Meter("messenger-api")
|
||||
}
|
||||
registry.MustRegister(
|
||||
result.requests,
|
||||
result.duration,
|
||||
result.inFlight,
|
||||
result.replays,
|
||||
result.attachmentUploads,
|
||||
result.attachmentDuration,
|
||||
result.attachmentBytes,
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
if handler == nil {
|
||||
handler = http.NotFoundHandler()
|
||||
}
|
||||
requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests."))
|
||||
duration, _ := meter.Float64Histogram("messenger_http_request_duration",
|
||||
metric.WithDescription("HTTP request duration."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(requestDurationBuckets...),
|
||||
)
|
||||
result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
|
||||
return result
|
||||
inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served."))
|
||||
replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record."))
|
||||
attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result."))
|
||||
attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration",
|
||||
metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(uploadDurationBuckets...),
|
||||
)
|
||||
attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By"))
|
||||
return &metrics{
|
||||
requests: requests, duration: duration, inFlight: inFlight, replays: replays,
|
||||
attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration,
|
||||
attachmentBytes: attachmentBytes, handler: handler,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *metrics) observe(r *http.Request, status int, started time.Time) {
|
||||
route := routeTemplate(r.URL.Path)
|
||||
m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc()
|
||||
m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds())
|
||||
func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) {
|
||||
attributes := metric.WithAttributes(
|
||||
attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)),
|
||||
attribute.String("status_class", strconv.Itoa(status/100)+"xx"),
|
||||
)
|
||||
m.requests.Add(ctx, 1, attributes)
|
||||
m.duration.Record(ctx, time.Since(started).Seconds(), attributes)
|
||||
}
|
||||
|
||||
func routeTemplate(path string) string {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
|
||||
@@ -6,6 +6,16 @@ import (
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
type Dependencies struct {
|
||||
@@ -17,6 +27,8 @@ type Dependencies struct {
|
||||
Passwords PasswordService
|
||||
Database HealthChecker
|
||||
Objects HealthChecker
|
||||
Meter metric.Meter
|
||||
Metrics http.Handler
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -67,7 +79,8 @@ func NewServer(
|
||||
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
|
||||
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
|
||||
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
|
||||
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
|
||||
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort,
|
||||
metrics: newMetrics(dependencies.Meter, dependencies.Metrics),
|
||||
}
|
||||
s.ready.Store(true)
|
||||
return s
|
||||
@@ -82,27 +95,39 @@ func (s *Server) Handler() http.Handler {
|
||||
}
|
||||
|
||||
func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
|
||||
requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
|
||||
tracer := otel.Tracer("messenger-api/http")
|
||||
route := routeTemplate(r.URL.Path)
|
||||
requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer),
|
||||
trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route)))
|
||||
defer span.End()
|
||||
r = r.WithContext(requestContext)
|
||||
w.Header().Set("X-Instance-Id", s.instanceID)
|
||||
requestID, err := newID()
|
||||
if err != nil {
|
||||
http.Error(w, "server unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
span.SetAttributes(attribute.String("request.id", requestID))
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK}
|
||||
started := time.Now()
|
||||
s.metrics.inFlight.Inc()
|
||||
s.metrics.inFlight.Add(r.Context(), 1)
|
||||
defer func() {
|
||||
s.metrics.inFlight.Dec()
|
||||
s.metrics.inFlight.Add(r.Context(), -1)
|
||||
if recovered := recover(); recovered != nil {
|
||||
s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path)
|
||||
s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route)
|
||||
if wrapped.status == http.StatusOK {
|
||||
writeError(wrapped, requestID, internalError())
|
||||
}
|
||||
}
|
||||
s.metrics.observe(r, wrapped.status, started)
|
||||
s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
|
||||
s.metrics.observe(r.Context(), r, wrapped.status, started)
|
||||
span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status))
|
||||
if wrapped.status >= http.StatusInternalServerError {
|
||||
span.SetStatus(codes.Error, http.StatusText(wrapped.status))
|
||||
}
|
||||
s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
|
||||
}()
|
||||
if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
|
||||
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
|
||||
|
||||
@@ -14,8 +14,8 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
var errDependency = errors.New("dependency unavailable")
|
||||
|
||||
@@ -13,7 +13,12 @@ import (
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("object not found")
|
||||
@@ -85,30 +90,38 @@ func (s *Store) Ping(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
|
||||
ctx, span := s.startSpan(ctx, "PUT")
|
||||
defer span.End()
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
_, err := s.client.PutObject(
|
||||
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
|
||||
)
|
||||
if err != nil {
|
||||
recordSpanError(span, err)
|
||||
return fmt.Errorf("put S3 object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
ctx, span := s.startSpan(ctx, "GET")
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
cancel()
|
||||
recordSpanError(span, err)
|
||||
span.End()
|
||||
return nil, objectError("get S3 object", err)
|
||||
}
|
||||
if _, err := object.Stat(); err != nil {
|
||||
cancel()
|
||||
_ = object.Close()
|
||||
recordSpanError(span, err)
|
||||
span.End()
|
||||
return nil, objectError("stat S3 object", err)
|
||||
}
|
||||
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
|
||||
return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
|
||||
@@ -122,9 +135,12 @@ func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
ctx, span := s.startSpan(ctx, "DELETE")
|
||||
defer span.End()
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
recordSpanError(span, err)
|
||||
return objectError("delete S3 object", err)
|
||||
}
|
||||
return nil
|
||||
@@ -161,6 +177,28 @@ type timeoutReader struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type tracedReader struct {
|
||||
io.ReadCloser
|
||||
span trace.Span
|
||||
}
|
||||
|
||||
func (r *tracedReader) Read(buffer []byte) (int, error) {
|
||||
read, err := r.ReadCloser.Read(buffer)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
recordSpanError(r.span, err)
|
||||
}
|
||||
return read, err
|
||||
}
|
||||
|
||||
func (r *tracedReader) Close() error {
|
||||
err := r.ReadCloser.Close()
|
||||
if err != nil {
|
||||
recordSpanError(r.span, err)
|
||||
}
|
||||
r.span.End()
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *timeoutReader) Close() error {
|
||||
err := r.ReadCloser.Close()
|
||||
r.cancel()
|
||||
@@ -174,3 +212,14 @@ func objectError(action string, err error) error {
|
||||
}
|
||||
return fmt.Errorf("%s: %w", action, err)
|
||||
}
|
||||
|
||||
func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) {
|
||||
return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation),
|
||||
attribute.String("s3.bucket.name", s.bucket)))
|
||||
}
|
||||
|
||||
func recordSpanError(span trace.Span, err error) {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "S3 operation failed")
|
||||
}
|
||||
|
||||
@@ -6,7 +6,13 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
var errLeaseLost = errors.New("outbox lease lost")
|
||||
@@ -24,9 +30,9 @@ type Publisher interface {
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObservePublished()
|
||||
ObservePublishFailure()
|
||||
ObserveRecovered(int)
|
||||
ObservePublished(context.Context)
|
||||
ObservePublishFailure(context.Context)
|
||||
ObserveRecovered(context.Context, int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -156,15 +162,32 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return published, nil
|
||||
}
|
||||
_, err := d.publisher.PublishConfirmed(ctx, event.Payload)
|
||||
publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload)
|
||||
publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish",
|
||||
trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes(
|
||||
attribute.String("messaging.system", "redis"),
|
||||
attribute.String("messaging.destination.name", "attachment-jobs"),
|
||||
attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt),
|
||||
))
|
||||
payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload)
|
||||
if injectErr != nil {
|
||||
span.RecordError(injectErr)
|
||||
span.SetStatus(codes.Error, "inject trace context")
|
||||
span.End()
|
||||
return published, fmt.Errorf("inject outbox trace context: %w", injectErr)
|
||||
}
|
||||
_, err := d.publisher.PublishConfirmed(publishContext, payload)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "publish failed")
|
||||
span.End()
|
||||
if ctx.Err() != nil {
|
||||
return published, nil
|
||||
}
|
||||
d.metrics.ObservePublishFailure()
|
||||
d.metrics.ObservePublishFailure(publishContext)
|
||||
nextAttemptAt := d.now().Add(d.config.RetryDelay)
|
||||
released, releaseErr := d.store.ReleaseOutboxEvent(
|
||||
ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
|
||||
publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
|
||||
)
|
||||
if releaseErr != nil {
|
||||
return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr)
|
||||
@@ -174,14 +197,21 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken)
|
||||
marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "mark published")
|
||||
span.End()
|
||||
return published, fmt.Errorf("mark outbox event published: %w", err)
|
||||
}
|
||||
if !marked {
|
||||
span.SetStatus(codes.Error, "outbox lease lost")
|
||||
span.End()
|
||||
return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost)
|
||||
}
|
||||
d.metrics.ObservePublished()
|
||||
span.SetAttributes(attribute.String("messaging.operation.type", "publish"))
|
||||
span.End()
|
||||
d.metrics.ObservePublished(publishContext)
|
||||
published++
|
||||
}
|
||||
return published, nil
|
||||
@@ -192,12 +222,12 @@ func (d *Dispatcher) RecoverExpired(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("recover expired outbox leases: %w", err)
|
||||
}
|
||||
d.metrics.ObserveRecovered(recovered)
|
||||
d.metrics.ObserveRecovered(ctx, recovered)
|
||||
return nil
|
||||
}
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObservePublished() {}
|
||||
func (nopMetrics) ObservePublishFailure() {}
|
||||
func (nopMetrics) ObserveRecovered(int) {}
|
||||
func (nopMetrics) ObservePublished(context.Context) {}
|
||||
func (nopMetrics) ObservePublishFailure(context.Context) {}
|
||||
func (nopMetrics) ObserveRecovered(context.Context, int) {}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const attachmentJobTopic = "attachment.jobs"
|
||||
@@ -447,6 +447,8 @@ func (s *Store) RetryAttachmentJob(
|
||||
lease model.AttachmentJobLease,
|
||||
errorCode string,
|
||||
nextAttemptAt time.Time,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
|
||||
return "", model.ErrInvalidInput
|
||||
@@ -483,9 +485,7 @@ func (s *Store) RetryAttachmentJob(
|
||||
if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return "", fmt.Errorf("requeue attachment: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RenewAttachmentJobLease(
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RecoverExpiredAttachmentJobs(
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const attachmentColumns = `
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package postgres
|
||||
|
||||
import "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
|
||||
var (
|
||||
ErrConflict = model.ErrConflict
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const expiredIdempotencyCleanupBatch = 100
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) ClaimOutboxEvents(
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
|
||||
@@ -15,10 +15,10 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
|
||||
)
|
||||
|
||||
type PostgresSuite struct {
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type MetadataStore interface {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
"github.com/valkey-io/valkey-go"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
|
||||
)
|
||||
|
||||
type SessionSuite struct {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
connectionStateUsed = attribute.String("state", "used")
|
||||
connectionStateIdle = attribute.String("state", "idle")
|
||||
connectionStateConnecting = attribute.String("state", "connecting")
|
||||
)
|
||||
|
||||
func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error {
|
||||
_, err := meter.Int64ObservableGauge("messenger_db_pool_connections",
|
||||
metric.WithDescription("PostgreSQL pool connections by state."),
|
||||
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
|
||||
stats := pool.Stat()
|
||||
observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed))
|
||||
observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle))
|
||||
observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting))
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = meter.Int64ObservableCounter("messenger_db_pool_waited",
|
||||
metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."),
|
||||
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
|
||||
observer.Observe(pool.Stat().EmptyAcquireCount())
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type correlationHandler struct {
|
||||
stdout slog.Handler
|
||||
otel slog.Handler
|
||||
}
|
||||
|
||||
func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler {
|
||||
return correlationHandler{stdout: stdout, otel: otel}
|
||||
}
|
||||
|
||||
func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool {
|
||||
return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level)
|
||||
}
|
||||
|
||||
func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error {
|
||||
span := trace.SpanContextFromContext(ctx)
|
||||
if span.IsValid() {
|
||||
record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String()))
|
||||
}
|
||||
record = sanitizedRecord(record)
|
||||
if err := h.stdout.Handle(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
return h.otel.Handle(ctx, record)
|
||||
}
|
||||
|
||||
func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler {
|
||||
attributes = sanitizedAttributes(attributes)
|
||||
return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)}
|
||||
}
|
||||
|
||||
func (h correlationHandler) WithGroup(name string) slog.Handler {
|
||||
return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)}
|
||||
}
|
||||
|
||||
func sanitizedRecord(record slog.Record) slog.Record {
|
||||
copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC)
|
||||
record.Attrs(func(attribute slog.Attr) bool {
|
||||
if !sensitiveKey(attribute.Key) {
|
||||
copy.AddAttrs(attribute)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return copy
|
||||
}
|
||||
|
||||
func sanitizedAttributes(attributes []slog.Attr) []slog.Attr {
|
||||
result := make([]slog.Attr, 0, len(attributes))
|
||||
for _, attribute := range attributes {
|
||||
if !sensitiveKey(attribute.Key) {
|
||||
result = append(result, attribute)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func sensitiveKey(key string) bool {
|
||||
key = strings.ToLower(key)
|
||||
for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} {
|
||||
if strings.Contains(key, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
)
|
||||
|
||||
type MessageCarrier struct {
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
}
|
||||
|
||||
func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context {
|
||||
return otel.GetTextMapPropagator().Extract(ctx, carrier)
|
||||
}
|
||||
|
||||
func Inject(ctx context.Context, carrier propagation.TextMapCarrier) {
|
||||
otel.GetTextMapPropagator().Inject(ctx, carrier)
|
||||
}
|
||||
|
||||
func ContextCarrier(ctx context.Context) MessageCarrier {
|
||||
carrier := propagation.MapCarrier{}
|
||||
Inject(ctx, carrier)
|
||||
return MessageCarrier{
|
||||
Traceparent: carrier.Get("traceparent"),
|
||||
Tracestate: carrier.Get("tracestate"),
|
||||
}
|
||||
}
|
||||
|
||||
func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context {
|
||||
headers := propagation.MapCarrier{
|
||||
"traceparent": carrier.Traceparent,
|
||||
"tracestate": carrier.Tracestate,
|
||||
}
|
||||
return Extract(ctx, headers)
|
||||
}
|
||||
|
||||
func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context {
|
||||
var carrier struct {
|
||||
Traceparent string `json:"traceparent"`
|
||||
Tracestate string `json:"tracestate"`
|
||||
}
|
||||
if json.Unmarshal(payload, &carrier) != nil {
|
||||
return ctx
|
||||
}
|
||||
return ExtractMessage(ctx, MessageCarrier(carrier))
|
||||
}
|
||||
|
||||
func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) {
|
||||
var value map[string]json.RawMessage
|
||||
if err := json.Unmarshal(payload, &value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carrier := ContextCarrier(ctx)
|
||||
setJSONField(value, "traceparent", carrier.Traceparent)
|
||||
setJSONField(value, "tracestate", carrier.Tracestate)
|
||||
return json.Marshal(value)
|
||||
}
|
||||
|
||||
func setJSONField(value map[string]json.RawMessage, key, field string) {
|
||||
if field == "" {
|
||||
delete(value, key)
|
||||
return
|
||||
}
|
||||
encoded, _ := json.Marshal(field)
|
||||
value[key] = encoded
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) {
|
||||
otel.SetTextMapPropagator(propagation.TraceContext{})
|
||||
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
|
||||
require.NoError(t, err)
|
||||
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
|
||||
require.NoError(t, err)
|
||||
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
|
||||
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
|
||||
}))
|
||||
|
||||
payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`))
|
||||
require.NoError(t, err)
|
||||
var value map[string]any
|
||||
require.NoError(t, json.Unmarshal(payload, &value))
|
||||
require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"])
|
||||
require.NotContains(t, value, "baggage")
|
||||
|
||||
extracted := ExtractJSONMessage(context.Background(), payload)
|
||||
require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID())
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/contrib/bridges/otelslog"
|
||||
otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
otelprometheus "go.opentelemetry.io/otel/exporters/prometheus"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/sdk/log"
|
||||
"go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
"go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEnvironment = "lab"
|
||||
defaultNamespace = "mtusi-highload"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ServiceName string
|
||||
Version string
|
||||
InstanceID string
|
||||
Endpoint string
|
||||
Environment string
|
||||
Namespace string
|
||||
Insecure bool
|
||||
}
|
||||
|
||||
type Runtime struct {
|
||||
traces *trace.TracerProvider
|
||||
metrics *metric.MeterProvider
|
||||
logs *log.LoggerProvider
|
||||
logger *slog.Logger
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func ConfigFromEnvironment(serviceName, instanceID string) Config {
|
||||
resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES"))
|
||||
return Config{
|
||||
ServiceName: serviceName,
|
||||
Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"),
|
||||
InstanceID: instanceID,
|
||||
Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")),
|
||||
Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment),
|
||||
Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace),
|
||||
Insecure: true,
|
||||
}
|
||||
}
|
||||
|
||||
func parseResourceAttributes(raw string) map[string]string {
|
||||
result := make(map[string]string)
|
||||
for _, pair := range strings.Split(raw, ",") {
|
||||
key, value, found := strings.Cut(pair, "=")
|
||||
if found && strings.TrimSpace(key) != "" {
|
||||
result[strings.TrimSpace(key)] = strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func firstValue(values ...string) string {
|
||||
for _, value := range values {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func New(ctx context.Context, config Config) (*Runtime, error) {
|
||||
if err := config.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := newResource(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registry := prometheus.NewRegistry()
|
||||
prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Prometheus reader: %w", err)
|
||||
}
|
||||
|
||||
traceOptions := []trace.TracerProviderOption{trace.WithResource(res)}
|
||||
metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)}
|
||||
logOptions := []log.LoggerProviderOption{log.WithResource(res)}
|
||||
if config.Endpoint != "" {
|
||||
traceExporter, err := newTraceExporter(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metricExporter, err := newMetricExporter(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logExporter, err := newLogExporter(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
traceOptions = append(traceOptions, trace.WithBatcher(traceExporter,
|
||||
trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512),
|
||||
trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second),
|
||||
))
|
||||
metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader(
|
||||
metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second),
|
||||
)))
|
||||
logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter,
|
||||
log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512),
|
||||
log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second),
|
||||
)))
|
||||
}
|
||||
|
||||
traces := trace.NewTracerProvider(traceOptions...)
|
||||
metrics := metric.NewMeterProvider(metricOptions...)
|
||||
logs := log.NewLoggerProvider(logOptions...)
|
||||
propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{})
|
||||
otel.SetTracerProvider(traces)
|
||||
otel.SetMeterProvider(metrics)
|
||||
otel.SetTextMapPropagator(propagator)
|
||||
if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil {
|
||||
return nil, fmt.Errorf("start Go runtime metrics: %w", err)
|
||||
}
|
||||
|
||||
stdout := slog.NewJSONHandler(os.Stdout, nil)
|
||||
otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs))
|
||||
logger := slog.New(newCorrelationHandler(stdout, otelHandler))
|
||||
registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
|
||||
return &Runtime{
|
||||
traces: traces, metrics: metrics, logs: logs, logger: logger,
|
||||
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
switch {
|
||||
case strings.TrimSpace(c.ServiceName) == "":
|
||||
return errors.New("telemetry service name is required")
|
||||
case strings.TrimSpace(c.InstanceID) == "":
|
||||
return errors.New("telemetry instance ID is required")
|
||||
case strings.TrimSpace(c.Environment) == "":
|
||||
return errors.New("telemetry environment is required")
|
||||
case strings.TrimSpace(c.Namespace) == "":
|
||||
return errors.New("telemetry service namespace is required")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func newResource(ctx context.Context, config Config) (*resource.Resource, error) {
|
||||
res, err := resource.New(ctx,
|
||||
resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(),
|
||||
resource.WithAttributes(
|
||||
semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version),
|
||||
semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace),
|
||||
semconv.DeploymentEnvironmentName(config.Environment),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create telemetry resource: %w", err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) {
|
||||
var options []otlptracegrpc.Option
|
||||
if strings.Contains(config.Endpoint, "://") {
|
||||
options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint))
|
||||
} else {
|
||||
options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint))
|
||||
}
|
||||
if config.Insecure {
|
||||
options = append(options, otlptracegrpc.WithInsecure())
|
||||
}
|
||||
exporter, err := otlptracegrpc.New(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create OTLP trace exporter: %w", err)
|
||||
}
|
||||
return exporter, nil
|
||||
}
|
||||
|
||||
func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) {
|
||||
var options []otlpmetricgrpc.Option
|
||||
if strings.Contains(config.Endpoint, "://") {
|
||||
options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint))
|
||||
} else {
|
||||
options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint))
|
||||
}
|
||||
if config.Insecure {
|
||||
options = append(options, otlpmetricgrpc.WithInsecure())
|
||||
}
|
||||
exporter, err := otlpmetricgrpc.New(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create OTLP metric exporter: %w", err)
|
||||
}
|
||||
return exporter, nil
|
||||
}
|
||||
|
||||
func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) {
|
||||
var options []otlploggrpc.Option
|
||||
if strings.Contains(config.Endpoint, "://") {
|
||||
options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint))
|
||||
} else {
|
||||
options = append(options, otlploggrpc.WithEndpoint(config.Endpoint))
|
||||
}
|
||||
if config.Insecure {
|
||||
options = append(options, otlploggrpc.WithInsecure())
|
||||
}
|
||||
exporter, err := otlploggrpc.New(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create OTLP log exporter: %w", err)
|
||||
}
|
||||
return exporter, nil
|
||||
}
|
||||
|
||||
func (r *Runtime) Logger() *slog.Logger {
|
||||
return r.logger
|
||||
}
|
||||
|
||||
func (r *Runtime) MeterProvider() *metric.MeterProvider {
|
||||
return r.metrics
|
||||
}
|
||||
|
||||
func (r *Runtime) MetricsHandler() http.Handler {
|
||||
return r.handler
|
||||
}
|
||||
|
||||
func (r *Runtime) Shutdown(ctx context.Context) error {
|
||||
var result error
|
||||
for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} {
|
||||
if err := shutdown(ctx); err != nil {
|
||||
result = errors.Join(result, err)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TestConfigValidation(t *testing.T) {
|
||||
valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"}
|
||||
require.NoError(t, valid.validate())
|
||||
|
||||
invalid := valid
|
||||
invalid.ServiceName = ""
|
||||
require.EqualError(t, invalid.validate(), "telemetry service name is required")
|
||||
}
|
||||
|
||||
func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) {
|
||||
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
|
||||
require.NoError(t, err)
|
||||
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
|
||||
require.NoError(t, err)
|
||||
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
|
||||
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
|
||||
}))
|
||||
first := &recordHandler{}
|
||||
second := &recordHandler{}
|
||||
handler := newCorrelationHandler(first, second)
|
||||
record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0)
|
||||
record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret"))
|
||||
|
||||
require.NoError(t, handler.Handle(ctx, record))
|
||||
for _, captured := range []*recordHandler{first, second} {
|
||||
require.Equal(t, traceID.String(), captured.attributes["trace_id"])
|
||||
require.Equal(t, spanID.String(), captured.attributes["span_id"])
|
||||
require.Equal(t, "http_request_completed", captured.attributes["event"])
|
||||
require.NotContains(t, captured.attributes, "authorization")
|
||||
}
|
||||
}
|
||||
|
||||
type recordHandler struct {
|
||||
attributes map[string]string
|
||||
}
|
||||
|
||||
func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true }
|
||||
func (h *recordHandler) Handle(_ context.Context, record slog.Record) error {
|
||||
h.attributes = make(map[string]string)
|
||||
record.Attrs(func(attribute slog.Attr) bool {
|
||||
h.attributes[attribute.Key] = attribute.Value.String()
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
}
|
||||
func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
|
||||
func (h *recordHandler) WithGroup(string) slog.Handler { return h }
|
||||
Reference in New Issue
Block a user