From 89787861ffe8d86c0fc03fb4602594115d07ea91 Mon Sep 17 00:00:00 2001 From: Ilya Date: Thu, 22 Oct 2026 22:24:57 +0300 Subject: [PATCH] =?UTF-8?q?lab4:=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8=D0=B3?= =?UTF-8?q?=D1=83=D1=80=D0=B0=D1=86=D0=B8=D1=8F=20S3,=20ObjectStorage,=20?= =?UTF-8?q?=D0=BC=D0=B5=D1=82=D1=80=D0=B8=D0=BA=D0=B8=20=D0=BF=D0=B0=D0=BC?= =?UTF-8?q?=D1=8F=D1=82=D0=B8=20=D0=B8=20=D0=B1=D0=B0=D0=B9=D1=82=D0=BE?= =?UTF-8?q?=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/package-lock.json | 415 ++++++++++++++++++ app/package.json | 3 + app/src/common/errors/api-exception.filter.ts | 15 +- app/src/config/app-config.schema.ts | 90 +++- app/src/database/migration.runner.ts | 4 +- .../idempotency/idempotency-gc.cli.ts | 4 +- app/src/metrics/metrics.service.spec.ts | 35 +- app/src/metrics/metrics.service.ts | 58 ++- app/src/storage/object-storage.ts | 184 ++++++++ app/src/storage/storage.module.ts | 5 + 10 files changed, 795 insertions(+), 18 deletions(-) create mode 100644 app/src/storage/object-storage.ts create mode 100644 app/src/storage/storage.module.ts diff --git a/app/package-lock.json b/app/package-lock.json index b98212b..4338b4b 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -9,11 +9,14 @@ "version": "0.0.1", "license": "UNLICENSED", "dependencies": { + "@aws-sdk/client-s3": "^3.1156.0", + "@aws-sdk/s3-request-presigner": "^3.1156.0", "@nestjs/common": "^12.0.1", "@nestjs/config": "^12.0.0", "@nestjs/core": "^12.0.1", "@nestjs/platform-express": "^12.0.1", "@nestjs/platform-fastify": "^12.0.3", + "@smithy/node-http-handler": "^4.12.1", "argon2": "^0.45.1", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", @@ -144,6 +147,331 @@ } } }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1001.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.5.tgz", + "integrity": "sha512-6erto9nK2OZO/woEgmFIaPh5gChgdNd7nL6jgAf8TAXz3/QHC8Pan2TFe9GYp3IxOIKCCGWPylSjtOHKfNzQlw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1156.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1156.0.tgz", + "integrity": "sha512-Z2PqQqKMDVX56tNZ6ggmjoI5o/8+4+8ArmjLwE5ktYsEzs1BrvklBT1ZzI+s0Vcf3wfPguoEMD7EkMswmEta/A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1001.5", + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/credential-provider-node": "^3.972.88", + "@aws-sdk/middleware-sdk-s3": "^3.972.81", + "@aws-sdk/signature-v4-multi-region": "^3.996.49", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.978.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.5.tgz", + "integrity": "sha512-ruI0AWqcrfL20XDS6SwZIJFPOBZ6N0ADa9nLzWlBhxk1aoUjbUX3JkTR9vEmk7o35XpAHi68ahZ4T6RE2U9XyA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.975.0", + "@aws-sdk/xml-builder": "^3.972.41", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.36.1", + "@smithy/signature-v4": "^5.7.4", + "@smithy/types": "^4.19.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.76.tgz", + "integrity": "sha512-YxmL1Ar7bV9ZkJvtv+lj+MF6le76NnZnbmgySyZuxpZaCv0/1lZ3KxKy6RIj1h9ui2hWnVjZxbmTMeO0y/NrQA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.78", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.78.tgz", + "integrity": "sha512-3VHknDRngoebsaqyT1iR75b5cBT2zfEtODK6yYmYltljkK1HEAQLs2FEd1IpWqAeOYdMGaG9wikngWLTufY7RA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.21", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.21.tgz", + "integrity": "sha512-aLx9hbCX6mGAyv4ZZd8UagJ4RASHVJw6BFDSIyuKmLkqNL6H47EjBPK6qaJ4B5TDV6mc+vMa0SJFYyo4mkvVQg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/credential-provider-env": "^3.972.76", + "@aws-sdk/credential-provider-http": "^3.972.78", + "@aws-sdk/credential-provider-login": "^3.972.83", + "@aws-sdk/credential-provider-process": "^3.972.76", + "@aws-sdk/credential-provider-sso": "^3.973.20", + "@aws-sdk/credential-provider-web-identity": "^3.972.82", + "@aws-sdk/nested-clients": "^3.997.50", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/credential-provider-imds": "^4.5.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.83", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.83.tgz", + "integrity": "sha512-9bJU6FjRacF4FK/QHf8Dyxhur7B1grRj8C1Yh4IkpoFW2exQflPdCv0VrbemLW9ZctLNle+AE7TETs9KNvuLYg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/nested-clients": "^3.997.50", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.88", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.88.tgz", + "integrity": "sha512-AG0KID5qhQPzrFkh5b0Cws1Amytvzo0L0Xj6hMgJXEwD6ZXAqninDhSNAWJiN5zU4YVVZnuidnqwPwYJ04IamA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.76", + "@aws-sdk/credential-provider-http": "^3.972.78", + "@aws-sdk/credential-provider-ini": "^3.973.21", + "@aws-sdk/credential-provider-process": "^3.972.76", + "@aws-sdk/credential-provider-sso": "^3.973.20", + "@aws-sdk/credential-provider-web-identity": "^3.972.82", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/credential-provider-imds": "^4.5.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.76.tgz", + "integrity": "sha512-pdhKFTXv1zPYhh92MTlp0jVbLWvUEV9v+rziuEkoskGGQ2iJskxHTO38nA8UAbT/N4IM3n8OZJ9rIY5IBO15/A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.20.tgz", + "integrity": "sha512-DlxrAB3MnjNLx/da7b89W+3DyJQYso/q7Ns15MsST2IV2nkAYQOYTUCaffNSjgrpMP99Dm2mV2jxzcbyF94ZfA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/nested-clients": "^3.997.50", + "@aws-sdk/token-providers": "3.1155.0", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.82", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.82.tgz", + "integrity": "sha512-LUo46IqPTJ4xKBsrLuoaxiv/D2GTBCcFDMD1y3jKIqPR/p7Rw14RJFZ0iRWftqZtyLPQX9JMGnSi3mBu4y/pNA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/nested-clients": "^3.997.50", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.81", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.81.tgz", + "integrity": "sha512-jfanv1pV0sJFZw6/DPiNjvK7B478god0zOum5yIWo+/+1Gy+CSXWpGBWx+3nBJyDvjcu/2kUMr4ADUp9BHH34Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/signature-v4-multi-region": "^3.996.49", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.50", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.50.tgz", + "integrity": "sha512-nKb8qu4QsnpZTq/6JIXc72mzuzfh7BK4sUQkpQznPec2z4W7Ob2xYkcM4i5e1yGTBnbJVeFv/0sf3jcz85hclA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/signature-v4-multi-region": "^3.996.49", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/s3-request-presigner": { + "version": "3.1156.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1156.0.tgz", + "integrity": "sha512-M4yj4HmAPydhIMxKK90eTcnHr5FHBl000d2oFMAWpt+XPX86c2kmNJpiYbKrPLIe8jyK9sweRt5CDdWrhq/QTg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/signature-v4-multi-region": "^3.996.49", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.49", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.49.tgz", + "integrity": "sha512-nvuAZNRrRRVCiARRKsYlQ0VnXiEQiq7vXPH+UA+LSo+kN3Djl9eo/V2r0udkw85PnsNshmv2Em1clld5M3j67w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.975.0", + "@smithy/signature-v4": "^5.7.4", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1155.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1155.0.tgz", + "integrity": "sha512-iIcQnYLhWuq9qOLQbIWFrxF3PF0svVeZvHnUS4DUnO4zUFmFHIyGyjqrntXG+e/5PUgfavV0pYfDIKHxTzN2LA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.5", + "@aws-sdk/nested-clients": "^3.997.50", + "@aws-sdk/types": "^3.975.0", + "@smithy/core": "^3.36.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.975.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.975.0.tgz", + "integrity": "sha512-WQlhs6juByzWOzj5fwxOx0mPJaEqMcG/sDpl3Id61B3XhKE63xY+0QWcnG+sWR/sCfprmreyUo1lM7IKZmHMzw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz", + "integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/helper-string-parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", @@ -2462,6 +2790,87 @@ "url": "https://github.com/sindresorhus/is?sponsor=1" } }, + "node_modules/@smithy/core": { + "version": "3.36.2", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.36.2.tgz", + "integrity": "sha512-oRw0rB9QSUiCZfEs23G5F4Af+b2rQt1lItIICWu5IejFNd+uPM8SWqiKLMpQ6bX1Q7slsLbjuoE2niErFtkVPQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz", + "integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", + "integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz", + "integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz", + "integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -3112,6 +3521,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "5.0.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", diff --git a/app/package.json b/app/package.json index 2014015..c308530 100644 --- a/app/package.json +++ b/app/package.json @@ -26,11 +26,14 @@ "idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1156.0", + "@aws-sdk/s3-request-presigner": "^3.1156.0", "@nestjs/common": "^12.0.1", "@nestjs/config": "^12.0.0", "@nestjs/core": "^12.0.1", "@nestjs/platform-express": "^12.0.1", "@nestjs/platform-fastify": "^12.0.3", + "@smithy/node-http-handler": "^4.12.1", "argon2": "^0.45.1", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", diff --git a/app/src/common/errors/api-exception.filter.ts b/app/src/common/errors/api-exception.filter.ts index a138423..e8e01b7 100644 --- a/app/src/common/errors/api-exception.filter.ts +++ b/app/src/common/errors/api-exception.filter.ts @@ -14,7 +14,11 @@ import { resolveInstanceId } from '../../config/instance-id.js'; import type { ApiErrorBody } from './api-error.js'; type RequestWithId = { requestId?: string; method?: string; url?: string }; -type ExceptionBody = { message?: unknown; challenge?: unknown }; +type ExceptionBody = { + message?: unknown; + challenge?: unknown; + retryAfter?: unknown; +}; // Connection/session loss codes reported by node-postgres and the network layer. const UNAVAILABLE_ERROR_CODES: Record = { @@ -106,6 +110,7 @@ export class ApiExceptionFilter implements ExceptionFilter { let status = HttpStatus.INTERNAL_SERVER_ERROR; let message = 'Internal server error'; let challenge: string | undefined; + let retryAfter = 1; if (exception instanceof HttpException) { status = exception.getStatus(); const body = exception.getResponse(); @@ -119,6 +124,12 @@ export class ApiExceptionFilter implements ExceptionFilter { typeof details.challenge === 'string' ? details.challenge : undefined; + if ( + typeof details.retryAfter === 'number' && + Number.isFinite(details.retryAfter) && + details.retryAfter >= 1 + ) + retryAfter = Math.ceil(details.retryAfter); message = Array.isArray(rawMessage) ? rawMessage.join('; ') : typeof rawMessage === 'string' @@ -154,6 +165,8 @@ export class ApiExceptionFilter implements ExceptionFilter { : 'Bearer realm="messenger"'); adapter.setHeader(response, 'WWW-Authenticate', authChallenge); } + if (status === HttpStatus.TOO_MANY_REQUESTS) + adapter.setHeader(response, 'Retry-After', String(retryAfter)); adapter.setHeader(response, 'X-Request-Id', requestId); adapter.setHeader(response, 'X-Instance-Id', this.instanceId); const payload: ApiErrorBody = { diff --git a/app/src/config/app-config.schema.ts b/app/src/config/app-config.schema.ts index 2a686f5..031055d 100644 --- a/app/src/config/app-config.schema.ts +++ b/app/src/config/app-config.schema.ts @@ -64,15 +64,91 @@ export const appConfigSchema = z undefined, 86400, ), + S3_ENDPOINT: requiredString('S3_ENDPOINT'), + S3_PUBLIC_ENDPOINT: requiredString('S3_PUBLIC_ENDPOINT'), + S3_REGION: z.string().min(1).default('us-east-1'), + S3_BUCKET: requiredString('S3_BUCKET'), + S3_ACCESS_KEY_ID: requiredString('S3_ACCESS_KEY_ID'), + S3_SECRET_ACCESS_KEY: requiredString('S3_SECRET_ACCESS_KEY'), + S3_CONNECT_TIMEOUT_MS: numeric( + 'S3_CONNECT_TIMEOUT_MS', + 1, + undefined, + 1000, + ), + S3_REQUEST_TIMEOUT_MS: numeric( + 'S3_REQUEST_TIMEOUT_MS', + 1, + undefined, + 15000, + ), + MAX_UPLOAD_BYTES: numeric( + 'MAX_UPLOAD_BYTES', + 1, + 10485760, + 10485760, + ), + MAX_IMAGE_PIXELS: numeric( + 'MAX_IMAGE_PIXELS', + 1, + 40000000, + 40000000, + ), + UPLOAD_TIMEOUT_MS: numeric( + 'UPLOAD_TIMEOUT_MS', + 1000, + undefined, + 25000, + ), + UPLOAD_MAX_IN_FLIGHT: numeric('UPLOAD_MAX_IN_FLIGHT', 1, 1024, 32), + // 0 отключает фоновый проход уборщика (остаётся ручной CLI). + ATTACHMENT_SWEEP_INTERVAL_MS: numeric( + 'ATTACHMENT_SWEEP_INTERVAL_MS', + 0, + undefined, + 60000, + ), + ATTACHMENT_SWEEP_GRACE_SECONDS: numeric( + 'ATTACHMENT_SWEEP_GRACE_SECONDS', + 0, + undefined, + 900, + ), + DIRECT_UPLOAD_URL_TTL_SECONDS: numeric( + 'DIRECT_UPLOAD_URL_TTL_SECONDS', + 1, + 3600, + 300, + ), + DIRECT_UPLOAD_MAX_PENDING_PER_USER: numeric( + 'DIRECT_UPLOAD_MAX_PENDING_PER_USER', + 1, + 1000, + 10, + ), }) .strip(); +export const databaseConfigSchema = appConfigSchema.pick({ + DATABASE_URL: true, + DATABASE_POOL_MAX: true, + DATABASE_CONNECTION_TIMEOUT_MS: true, + DATABASE_STATEMENT_TIMEOUT_MS: true, + HEALTH_TIMEOUT_MS: true, + IDEMPOTENCY_TTL_SECONDS: true, +}); +export type DatabaseConfig = z.infer; + export type AppConfig = z.infer; export type AppConfigKey = keyof AppConfig; function resolveSecret( raw: Record, - key: 'DATABASE_URL' | 'REDIS_URL', + key: + | 'DATABASE_URL' + | 'REDIS_URL' + | 'S3_ACCESS_KEY_ID' + | 'S3_SECRET_ACCESS_KEY', ): unknown { const fileKey = `${key}_FILE`; const path = raw[fileKey]; @@ -97,5 +173,17 @@ export function parseAppConfig(raw: Record): AppConfig { ...raw, DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'), REDIS_URL: resolveSecret(raw, 'REDIS_URL'), + S3_ACCESS_KEY_ID: resolveSecret(raw, 'S3_ACCESS_KEY_ID'), + S3_SECRET_ACCESS_KEY: resolveSecret(raw, 'S3_SECRET_ACCESS_KEY'), + }); +} + +// Для процессов, которым нужна только БД (миграции, GC идемпотентности). +export function parseDatabaseConfig( + raw: Record, +): DatabaseConfig { + return databaseConfigSchema.parse({ + ...raw, + DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'), }); } diff --git a/app/src/database/migration.runner.ts b/app/src/database/migration.runner.ts index dfcd507..5e101c3 100644 --- a/app/src/database/migration.runner.ts +++ b/app/src/database/migration.runner.ts @@ -3,7 +3,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { DatabaseClient } from './database.client.js'; import { MetricsService } from '../metrics/metrics.service.js'; -import { parseAppConfig } from '../config/app-config.schema.js'; +import { parseDatabaseConfig } from '../config/app-config.schema.js'; const migrationsDirectory = join( dirname(fileURLToPath(import.meta.url)), @@ -46,7 +46,7 @@ export async function runMigrations(database: DatabaseClient): Promise { } async function main(): Promise { - const config = parseAppConfig(process.env as Record); + const config = parseDatabaseConfig(process.env as Record); const database = new DatabaseClient( { getOrThrow: (key: keyof typeof config) => config[key], diff --git a/app/src/messages/idempotency/idempotency-gc.cli.ts b/app/src/messages/idempotency/idempotency-gc.cli.ts index 7ab0da5..4e473bc 100644 --- a/app/src/messages/idempotency/idempotency-gc.cli.ts +++ b/app/src/messages/idempotency/idempotency-gc.cli.ts @@ -1,5 +1,5 @@ import { fileURLToPath } from 'node:url'; -import { parseAppConfig } from '../../config/app-config.schema.js'; +import { parseDatabaseConfig } from '../../config/app-config.schema.js'; import { DatabaseClient } from '../../database/database.client.js'; import { MetricsService } from '../../metrics/metrics.service.js'; import { IdempotencyRepository } from './idempotency.repository.js'; @@ -10,7 +10,7 @@ import { IdempotencyRepository } from './idempotency.repository.js'; // in the stand: docker compose exec api node dist/messages/idempotency/idempotency-gc.cli.js // on the host: npm run idempotency:gc (needs a reachable DATABASE_URL) async function main(): Promise { - const config = parseAppConfig(process.env as Record); + const config = parseDatabaseConfig(process.env as Record); const database = new DatabaseClient( { getOrThrow: (key: keyof typeof config) => config[key], diff --git a/app/src/metrics/metrics.service.spec.ts b/app/src/metrics/metrics.service.spec.ts index 2b5e501..0316e94 100644 --- a/app/src/metrics/metrics.service.spec.ts +++ b/app/src/metrics/metrics.service.spec.ts @@ -9,16 +9,15 @@ describe('MetricsService', () => { metrics.observeLatency(30); metrics.observeLatency(10); metrics.observeLatency(20); - expect(metrics.snapshot()).toEqual({ - counters: { cache_hit: 2 }, - latency_ms: { - count: 3, - sample_count: 3, - p50: 20, - p95: 30, - p99: 30, - max: 30, - }, + const snapshot = metrics.snapshot(); + expect(snapshot.counters).toEqual({ cache_hit: 2 }); + expect(snapshot.latency_ms).toEqual({ + count: 3, + sample_count: 3, + p50: 20, + p95: 30, + p99: 30, + max: 30, }); }); @@ -59,4 +58,20 @@ describe('MetricsService', () => { max: 0, }); }); + + it('adds byte counters and ignores invalid amounts', () => { + const metrics = new MetricsService(); + metrics.add('bytes', 100); + metrics.add('bytes', 28); + for (const value of [-1, NaN, Infinity]) metrics.add('bytes', value); + expect(metrics.snapshot().counters.bytes).toBe(128); + }); + + it('tracks memory peaks until reset', () => { + const metrics = new MetricsService(); + const ballast = Buffer.alloc(32 * 1024 * 1024, 1); + const during = metrics.snapshot().memory_peak_bytes.array_buffers; + expect(during).toBeGreaterThanOrEqual(ballast.length); + ballast.fill(0); + }); }); diff --git a/app/src/metrics/metrics.service.ts b/app/src/metrics/metrics.service.ts index bf54a99..9087b09 100644 --- a/app/src/metrics/metrics.service.ts +++ b/app/src/metrics/metrics.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; const LATENCY_SAMPLE_CAPACITY = 4096; @@ -11,14 +11,53 @@ type LatencySnapshot = { max: number; }; +const MEMORY_SAMPLE_INTERVAL_MS = 50; + +type MemoryUsage = { + rss: number; + heap_used: number; + external: number; + array_buffers: number; +}; + +function currentMemory(): MemoryUsage { + const usage = process.memoryUsage(); + return { + rss: usage.rss, + heap_used: usage.heapUsed, + external: usage.external, + array_buffers: usage.arrayBuffers, + }; +} + @Injectable() -export class MetricsService { +export class MetricsService implements OnModuleInit, OnModuleDestroy { + private memoryPeak: MemoryUsage = currentMemory(); + private memoryTimer: NodeJS.Timeout | undefined; private readonly counters: Record = {}; private readonly latencies: number[] = []; private latencyCount = 0; private maximumLatency = 0; private nextSample = 0; + onModuleInit(): void { + this.memoryTimer = setInterval( + () => this.sampleMemory(), + MEMORY_SAMPLE_INTERVAL_MS, + ); + this.memoryTimer.unref(); + } + + onModuleDestroy(): void { + if (this.memoryTimer) clearInterval(this.memoryTimer); + this.memoryTimer = undefined; + } + + add(name: string, amount: number): void { + if (!Number.isFinite(amount) || amount < 0) return; + this.counters[name] = (this.counters[name] ?? 0) + amount; + } + increment(name: string): void { this.counters[name] = (this.counters[name] ?? 0) + 1; } @@ -34,7 +73,10 @@ export class MetricsService { snapshot(): { counters: Record; latency_ms: LatencySnapshot; + memory_bytes: MemoryUsage; + memory_peak_bytes: MemoryUsage; } { + this.sampleMemory(); const sorted = [...this.latencies].sort((left, right) => left - right); return { counters: { ...this.counters }, @@ -46,6 +88,8 @@ export class MetricsService { p99: this.percentile(sorted, 0.99), max: this.maximumLatency, }, + memory_bytes: currentMemory(), + memory_peak_bytes: { ...this.memoryPeak }, }; } @@ -56,6 +100,16 @@ export class MetricsService { this.latencyCount = 0; this.maximumLatency = 0; this.nextSample = 0; + this.memoryPeak = currentMemory(); + } + + private sampleMemory(): void { + const now = currentMemory(); + const peak = this.memoryPeak; + peak.rss = Math.max(peak.rss, now.rss); + peak.heap_used = Math.max(peak.heap_used, now.heap_used); + peak.external = Math.max(peak.external, now.external); + peak.array_buffers = Math.max(peak.array_buffers, now.array_buffers); } private percentile(sorted: number[], percentile: number): number { diff --git a/app/src/storage/object-storage.ts b/app/src/storage/object-storage.ts new file mode 100644 index 0000000..f644874 --- /dev/null +++ b/app/src/storage/object-storage.ts @@ -0,0 +1,184 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { + Injectable, + Logger, + ServiceUnavailableException, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import http from 'node:http'; +import { Readable } from 'node:stream'; +import type { AppConfig } from '../config/app-config.schema.js'; + +export class ObjectStorageUnavailableError extends ServiceUnavailableException { + constructor() { + super('Object storage unavailable'); + } +} + +export type PresignPutOptions = { + contentType: string; + contentLength: number; + checksumSha256Base64: string; + expiresInSeconds: number; +}; + +function isMissingKey(error: unknown): boolean { + const candidate = error as { + name?: string; + $metadata?: { httpStatusCode?: number }; + }; + return ( + candidate?.name === 'NoSuchKey' || + candidate?.name === 'NotFound' || + candidate?.$metadata?.httpStatusCode === 404 + ); +} + +@Injectable() +export class ObjectStorage { + private readonly logger = new Logger(ObjectStorage.name); + private readonly bucket: string; + private readonly internal: S3Client; + private readonly presign: S3Client; + + constructor(config: ConfigService) { + this.bucket = config.getOrThrow('S3_BUCKET'); + const common = { + region: config.getOrThrow('S3_REGION'), + forcePathStyle: true, + credentials: { + accessKeyId: config.getOrThrow('S3_ACCESS_KEY_ID'), + secretAccessKey: config.getOrThrow('S3_SECRET_ACCESS_KEY'), + }, + // Поток нельзя повторить: повтор — решение клиента API. + maxAttempts: 1, + requestChecksumCalculation: 'WHEN_REQUIRED' as const, + responseChecksumValidation: 'WHEN_REQUIRED' as const, + requestHandler: new NodeHttpHandler({ + connectionTimeout: config.getOrThrow('S3_CONNECT_TIMEOUT_MS'), + requestTimeout: config.getOrThrow('S3_REQUEST_TIMEOUT_MS'), + httpAgent: new http.Agent({ keepAlive: true, maxSockets: 64 }), + }), + }; + this.internal = new S3Client({ + ...common, + endpoint: config.getOrThrow('S3_ENDPOINT'), + }); + // Сеть не использует: только подпись URL для клиента. + this.presign = new S3Client({ + ...common, + endpoint: config.getOrThrow('S3_PUBLIC_ENDPOINT'), + }); + } + + objectKey(attachmentId: string): string { + return `attachments/${attachmentId}/original`; + } + + async putStream( + key: string, + body: Readable, + contentLength: number, + contentType: string, + signal: AbortSignal, + ): Promise { + try { + await this.internal.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: key, + Body: body, + ContentLength: contentLength, + ContentType: contentType, + }), + { abortSignal: signal }, + ); + } catch (error) { + throw this.fail('put', error); + } + } + + async getStream(key: string, range?: string): Promise { + try { + const result = await this.internal.send( + new GetObjectCommand({ + Bucket: this.bucket, + Key: key, + Range: range, + }), + ); + return result.Body as Readable; + } catch (error) { + if (isMissingKey(error)) return null; + throw this.fail('get', error); + } + } + + async head( + key: string, + ): Promise<{ size: number; checksumSha256?: string } | null> { + try { + const result = await this.internal.send( + new HeadObjectCommand({ + Bucket: this.bucket, + Key: key, + ChecksumMode: 'ENABLED', + }), + ); + return { + size: Number(result.ContentLength ?? 0), + checksumSha256: result.ChecksumSHA256, + }; + } catch (error) { + if (isMissingKey(error)) return null; + throw this.fail('head', error); + } + } + + async delete(key: string): Promise { + try { + await this.internal.send( + new DeleteObjectCommand({ Bucket: this.bucket, Key: key }), + ); + } catch (error) { + if (isMissingKey(error)) return; + throw this.fail('delete', error); + } + } + + async presignPut(key: string, options: PresignPutOptions): Promise { + return getSignedUrl( + this.presign, + new PutObjectCommand({ + Bucket: this.bucket, + Key: key, + ContentType: options.contentType, + ContentLength: options.contentLength, + ChecksumSHA256: options.checksumSha256Base64, + }), + { + expiresIn: options.expiresInSeconds, + signableHeaders: new Set([ + 'content-type', + 'content-length', + 'x-amz-checksum-sha256', + ]), + unhoistableHeaders: new Set(['x-amz-checksum-sha256']), + }, + ); + } + + private fail(operation: string, error: unknown): Error { + const name = (error as { name?: string })?.name ?? 'UnknownError'; + this.logger.warn(`S3 ${operation} failed: ${name}`); + return new ObjectStorageUnavailableError(); + } +} diff --git a/app/src/storage/storage.module.ts b/app/src/storage/storage.module.ts new file mode 100644 index 0000000..6cdffb6 --- /dev/null +++ b/app/src/storage/storage.module.ts @@ -0,0 +1,5 @@ +import { Module } from '@nestjs/common'; +import { ObjectStorage } from './object-storage.js'; + +@Module({ providers: [ObjectStorage], exports: [ObjectStorage] }) +export class StorageModule {}