lab4: конфигурация S3, ObjectStorage, метрики памяти и байтов

This commit is contained in:
2026-10-22 22:24:57 +03:00
parent 4107ee27a3
commit 89787861ff
10 changed files with 795 additions and 18 deletions
+415
View File
@@ -9,11 +9,14 @@
"version": "0.0.1",
"license": "UNLICENSED",
"dependencies": {
"@aws-sdk/client-s3": "^3.1156.0",
"@aws-sdk/s3-request-presigner": "^3.1156.0",
"@nestjs/common": "^12.0.1",
"@nestjs/config": "^12.0.0",
"@nestjs/core": "^12.0.1",
"@nestjs/platform-express": "^12.0.1",
"@nestjs/platform-fastify": "^12.0.3",
"@smithy/node-http-handler": "^4.12.1",
"argon2": "^0.45.1",
"class-transformer": "^0.5.1",
"class-validator": "^0.15.1",
@@ -144,6 +147,331 @@
}
}
},
"node_modules/@aws-sdk/checksums": {
"version": "3.1001.5",
"resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.5.tgz",
"integrity": "sha512-6erto9nK2OZO/woEgmFIaPh5gChgdNd7nL6jgAf8TAXz3/QHC8Pan2TFe9GYp3IxOIKCCGWPylSjtOHKfNzQlw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/client-s3": {
"version": "3.1156.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1156.0.tgz",
"integrity": "sha512-Z2PqQqKMDVX56tNZ6ggmjoI5o/8+4+8ArmjLwE5ktYsEzs1BrvklBT1ZzI+s0Vcf3wfPguoEMD7EkMswmEta/A==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/checksums": "^3.1001.5",
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/credential-provider-node": "^3.972.88",
"@aws-sdk/middleware-sdk-s3": "^3.972.81",
"@aws-sdk/signature-v4-multi-region": "^3.996.49",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/core": {
"version": "3.978.5",
"resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.5.tgz",
"integrity": "sha512-ruI0AWqcrfL20XDS6SwZIJFPOBZ6N0ADa9nLzWlBhxk1aoUjbUX3JkTR9vEmk7o35XpAHi68ahZ4T6RE2U9XyA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.975.0",
"@aws-sdk/xml-builder": "^3.972.41",
"@aws/lambda-invoke-store": "^0.3.0",
"@smithy/core": "^3.36.1",
"@smithy/signature-v4": "^5.7.4",
"@smithy/types": "^4.19.0",
"bowser": "^2.11.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-env": {
"version": "3.972.76",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.76.tgz",
"integrity": "sha512-YxmL1Ar7bV9ZkJvtv+lj+MF6le76NnZnbmgySyZuxpZaCv0/1lZ3KxKy6RIj1h9ui2hWnVjZxbmTMeO0y/NrQA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-http": {
"version": "3.972.78",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.78.tgz",
"integrity": "sha512-3VHknDRngoebsaqyT1iR75b5cBT2zfEtODK6yYmYltljkK1HEAQLs2FEd1IpWqAeOYdMGaG9wikngWLTufY7RA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-ini": {
"version": "3.973.21",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.21.tgz",
"integrity": "sha512-aLx9hbCX6mGAyv4ZZd8UagJ4RASHVJw6BFDSIyuKmLkqNL6H47EjBPK6qaJ4B5TDV6mc+vMa0SJFYyo4mkvVQg==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/credential-provider-env": "^3.972.76",
"@aws-sdk/credential-provider-http": "^3.972.78",
"@aws-sdk/credential-provider-login": "^3.972.83",
"@aws-sdk/credential-provider-process": "^3.972.76",
"@aws-sdk/credential-provider-sso": "^3.973.20",
"@aws-sdk/credential-provider-web-identity": "^3.972.82",
"@aws-sdk/nested-clients": "^3.997.50",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/credential-provider-imds": "^4.5.2",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-login": {
"version": "3.972.83",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.83.tgz",
"integrity": "sha512-9bJU6FjRacF4FK/QHf8Dyxhur7B1grRj8C1Yh4IkpoFW2exQflPdCv0VrbemLW9ZctLNle+AE7TETs9KNvuLYg==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/nested-clients": "^3.997.50",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-node": {
"version": "3.972.88",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.88.tgz",
"integrity": "sha512-AG0KID5qhQPzrFkh5b0Cws1Amytvzo0L0Xj6hMgJXEwD6ZXAqninDhSNAWJiN5zU4YVVZnuidnqwPwYJ04IamA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/credential-provider-env": "^3.972.76",
"@aws-sdk/credential-provider-http": "^3.972.78",
"@aws-sdk/credential-provider-ini": "^3.973.21",
"@aws-sdk/credential-provider-process": "^3.972.76",
"@aws-sdk/credential-provider-sso": "^3.973.20",
"@aws-sdk/credential-provider-web-identity": "^3.972.82",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/credential-provider-imds": "^4.5.2",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-process": {
"version": "3.972.76",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.76.tgz",
"integrity": "sha512-pdhKFTXv1zPYhh92MTlp0jVbLWvUEV9v+rziuEkoskGGQ2iJskxHTO38nA8UAbT/N4IM3n8OZJ9rIY5IBO15/A==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-sso": {
"version": "3.973.20",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.20.tgz",
"integrity": "sha512-DlxrAB3MnjNLx/da7b89W+3DyJQYso/q7Ns15MsST2IV2nkAYQOYTUCaffNSjgrpMP99Dm2mV2jxzcbyF94ZfA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/nested-clients": "^3.997.50",
"@aws-sdk/token-providers": "3.1155.0",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-web-identity": {
"version": "3.972.82",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.82.tgz",
"integrity": "sha512-LUo46IqPTJ4xKBsrLuoaxiv/D2GTBCcFDMD1y3jKIqPR/p7Rw14RJFZ0iRWftqZtyLPQX9JMGnSi3mBu4y/pNA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/nested-clients": "^3.997.50",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/middleware-sdk-s3": {
"version": "3.972.81",
"resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.81.tgz",
"integrity": "sha512-jfanv1pV0sJFZw6/DPiNjvK7B478god0zOum5yIWo+/+1Gy+CSXWpGBWx+3nBJyDvjcu/2kUMr4ADUp9BHH34Q==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/signature-v4-multi-region": "^3.996.49",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/nested-clients": {
"version": "3.997.50",
"resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.50.tgz",
"integrity": "sha512-nKb8qu4QsnpZTq/6JIXc72mzuzfh7BK4sUQkpQznPec2z4W7Ob2xYkcM4i5e1yGTBnbJVeFv/0sf3jcz85hclA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/signature-v4-multi-region": "^3.996.49",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/s3-request-presigner": {
"version": "3.1156.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1156.0.tgz",
"integrity": "sha512-M4yj4HmAPydhIMxKK90eTcnHr5FHBl000d2oFMAWpt+XPX86c2kmNJpiYbKrPLIe8jyK9sweRt5CDdWrhq/QTg==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/signature-v4-multi-region": "^3.996.49",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/signature-v4-multi-region": {
"version": "3.996.49",
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.49.tgz",
"integrity": "sha512-nvuAZNRrRRVCiARRKsYlQ0VnXiEQiq7vXPH+UA+LSo+kN3Djl9eo/V2r0udkw85PnsNshmv2Em1clld5M3j67w==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.975.0",
"@smithy/signature-v4": "^5.7.4",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/token-providers": {
"version": "3.1155.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1155.0.tgz",
"integrity": "sha512-iIcQnYLhWuq9qOLQbIWFrxF3PF0svVeZvHnUS4DUnO4zUFmFHIyGyjqrntXG+e/5PUgfavV0pYfDIKHxTzN2LA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.5",
"@aws-sdk/nested-clients": "^3.997.50",
"@aws-sdk/types": "^3.975.0",
"@smithy/core": "^3.36.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/types": {
"version": "3.975.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.975.0.tgz",
"integrity": "sha512-WQlhs6juByzWOzj5fwxOx0mPJaEqMcG/sDpl3Id61B3XhKE63xY+0QWcnG+sWR/sCfprmreyUo1lM7IKZmHMzw==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/xml-builder": {
"version": "3.972.41",
"resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz",
"integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws/lambda-invoke-store": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
"integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@babel/helper-string-parser": {
"version": "7.29.7",
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz",
@@ -2462,6 +2790,87 @@
"url": "https://github.com/sindresorhus/is?sponsor=1"
}
},
"node_modules/@smithy/core": {
"version": "3.36.2",
"resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.36.2.tgz",
"integrity": "sha512-oRw0rB9QSUiCZfEs23G5F4Af+b2rQt1lItIICWu5IejFNd+uPM8SWqiKLMpQ6bX1Q7slsLbjuoE2niErFtkVPQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/credential-provider-imds": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz",
"integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.2",
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/fetch-http-handler": {
"version": "5.8.0",
"resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz",
"integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.3",
"@smithy/types": "^4.18.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/node-http-handler": {
"version": "4.12.1",
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz",
"integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.3",
"@smithy/types": "^4.18.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/signature-v4": {
"version": "5.7.4",
"resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz",
"integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/types": {
"version": "4.19.0",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz",
"integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@standard-schema/spec": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
@@ -3112,6 +3521,12 @@
"url": "https://opencollective.com/express"
}
},
"node_modules/bowser": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
"license": "MIT"
},
"node_modules/brace-expansion": {
"version": "5.0.12",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz",
+3
View File
@@ -26,11 +26,14 @@
"idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.1156.0",
"@aws-sdk/s3-request-presigner": "^3.1156.0",
"@nestjs/common": "^12.0.1",
"@nestjs/config": "^12.0.0",
"@nestjs/core": "^12.0.1",
"@nestjs/platform-express": "^12.0.1",
"@nestjs/platform-fastify": "^12.0.3",
"@smithy/node-http-handler": "^4.12.1",
"argon2": "^0.45.1",
"class-transformer": "^0.5.1",
"class-validator": "^0.15.1",
+14 -1
View File
@@ -14,7 +14,11 @@ import { resolveInstanceId } from '../../config/instance-id.js';
import type { ApiErrorBody } from './api-error.js';
type RequestWithId = { requestId?: string; method?: string; url?: string };
type ExceptionBody = { message?: unknown; challenge?: unknown };
type ExceptionBody = {
message?: unknown;
challenge?: unknown;
retryAfter?: unknown;
};
// Connection/session loss codes reported by node-postgres and the network layer.
const UNAVAILABLE_ERROR_CODES: Record<string, true> = {
@@ -106,6 +110,7 @@ export class ApiExceptionFilter implements ExceptionFilter {
let status = HttpStatus.INTERNAL_SERVER_ERROR;
let message = 'Internal server error';
let challenge: string | undefined;
let retryAfter = 1;
if (exception instanceof HttpException) {
status = exception.getStatus();
const body = exception.getResponse();
@@ -119,6 +124,12 @@ export class ApiExceptionFilter implements ExceptionFilter {
typeof details.challenge === 'string'
? details.challenge
: undefined;
if (
typeof details.retryAfter === 'number' &&
Number.isFinite(details.retryAfter) &&
details.retryAfter >= 1
)
retryAfter = Math.ceil(details.retryAfter);
message = Array.isArray(rawMessage)
? rawMessage.join('; ')
: typeof rawMessage === 'string'
@@ -154,6 +165,8 @@ export class ApiExceptionFilter implements ExceptionFilter {
: 'Bearer realm="messenger"');
adapter.setHeader(response, 'WWW-Authenticate', authChallenge);
}
if (status === HttpStatus.TOO_MANY_REQUESTS)
adapter.setHeader(response, 'Retry-After', String(retryAfter));
adapter.setHeader(response, 'X-Request-Id', requestId);
adapter.setHeader(response, 'X-Instance-Id', this.instanceId);
const payload: ApiErrorBody = {
+89 -1
View File
@@ -64,15 +64,91 @@ export const appConfigSchema = z
undefined,
86400,
),
S3_ENDPOINT: requiredString('S3_ENDPOINT'),
S3_PUBLIC_ENDPOINT: requiredString('S3_PUBLIC_ENDPOINT'),
S3_REGION: z.string().min(1).default('us-east-1'),
S3_BUCKET: requiredString('S3_BUCKET'),
S3_ACCESS_KEY_ID: requiredString('S3_ACCESS_KEY_ID'),
S3_SECRET_ACCESS_KEY: requiredString('S3_SECRET_ACCESS_KEY'),
S3_CONNECT_TIMEOUT_MS: numeric(
'S3_CONNECT_TIMEOUT_MS',
1,
undefined,
1000,
),
S3_REQUEST_TIMEOUT_MS: numeric(
'S3_REQUEST_TIMEOUT_MS',
1,
undefined,
15000,
),
MAX_UPLOAD_BYTES: numeric(
'MAX_UPLOAD_BYTES',
1,
10485760,
10485760,
),
MAX_IMAGE_PIXELS: numeric(
'MAX_IMAGE_PIXELS',
1,
40000000,
40000000,
),
UPLOAD_TIMEOUT_MS: numeric(
'UPLOAD_TIMEOUT_MS',
1000,
undefined,
25000,
),
UPLOAD_MAX_IN_FLIGHT: numeric('UPLOAD_MAX_IN_FLIGHT', 1, 1024, 32),
// 0 отключает фоновый проход уборщика (остаётся ручной CLI).
ATTACHMENT_SWEEP_INTERVAL_MS: numeric(
'ATTACHMENT_SWEEP_INTERVAL_MS',
0,
undefined,
60000,
),
ATTACHMENT_SWEEP_GRACE_SECONDS: numeric(
'ATTACHMENT_SWEEP_GRACE_SECONDS',
0,
undefined,
900,
),
DIRECT_UPLOAD_URL_TTL_SECONDS: numeric(
'DIRECT_UPLOAD_URL_TTL_SECONDS',
1,
3600,
300,
),
DIRECT_UPLOAD_MAX_PENDING_PER_USER: numeric(
'DIRECT_UPLOAD_MAX_PENDING_PER_USER',
1,
1000,
10,
),
})
.strip();
export const databaseConfigSchema = appConfigSchema.pick({
DATABASE_URL: true,
DATABASE_POOL_MAX: true,
DATABASE_CONNECTION_TIMEOUT_MS: true,
DATABASE_STATEMENT_TIMEOUT_MS: true,
HEALTH_TIMEOUT_MS: true,
IDEMPOTENCY_TTL_SECONDS: true,
});
export type DatabaseConfig = z.infer<typeof databaseConfigSchema>;
export type AppConfig = z.infer<typeof appConfigSchema>;
export type AppConfigKey = keyof AppConfig;
function resolveSecret(
raw: Record<string, unknown>,
key: 'DATABASE_URL' | 'REDIS_URL',
key:
| 'DATABASE_URL'
| 'REDIS_URL'
| 'S3_ACCESS_KEY_ID'
| 'S3_SECRET_ACCESS_KEY',
): unknown {
const fileKey = `${key}_FILE`;
const path = raw[fileKey];
@@ -97,5 +173,17 @@ export function parseAppConfig(raw: Record<string, unknown>): AppConfig {
...raw,
DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'),
REDIS_URL: resolveSecret(raw, 'REDIS_URL'),
S3_ACCESS_KEY_ID: resolveSecret(raw, 'S3_ACCESS_KEY_ID'),
S3_SECRET_ACCESS_KEY: resolveSecret(raw, 'S3_SECRET_ACCESS_KEY'),
});
}
// Для процессов, которым нужна только БД (миграции, GC идемпотентности).
export function parseDatabaseConfig(
raw: Record<string, unknown>,
): DatabaseConfig {
return databaseConfigSchema.parse({
...raw,
DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'),
});
}
+2 -2
View File
@@ -3,7 +3,7 @@ import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { DatabaseClient } from './database.client.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { parseAppConfig } from '../config/app-config.schema.js';
import { parseDatabaseConfig } from '../config/app-config.schema.js';
const migrationsDirectory = join(
dirname(fileURLToPath(import.meta.url)),
@@ -46,7 +46,7 @@ export async function runMigrations(database: DatabaseClient): Promise<void> {
}
async function main(): Promise<void> {
const config = parseAppConfig(process.env as Record<string, unknown>);
const config = parseDatabaseConfig(process.env as Record<string, unknown>);
const database = new DatabaseClient(
{
getOrThrow: (key: keyof typeof config) => config[key],
@@ -1,5 +1,5 @@
import { fileURLToPath } from 'node:url';
import { parseAppConfig } from '../../config/app-config.schema.js';
import { parseDatabaseConfig } from '../../config/app-config.schema.js';
import { DatabaseClient } from '../../database/database.client.js';
import { MetricsService } from '../../metrics/metrics.service.js';
import { IdempotencyRepository } from './idempotency.repository.js';
@@ -10,7 +10,7 @@ import { IdempotencyRepository } from './idempotency.repository.js';
// in the stand: docker compose exec api node dist/messages/idempotency/idempotency-gc.cli.js
// on the host: npm run idempotency:gc (needs a reachable DATABASE_URL)
async function main(): Promise<void> {
const config = parseAppConfig(process.env as Record<string, unknown>);
const config = parseDatabaseConfig(process.env as Record<string, unknown>);
const database = new DatabaseClient(
{
getOrThrow: (key: keyof typeof config) => config[key],
+25 -10
View File
@@ -9,16 +9,15 @@ describe('MetricsService', () => {
metrics.observeLatency(30);
metrics.observeLatency(10);
metrics.observeLatency(20);
expect(metrics.snapshot()).toEqual({
counters: { cache_hit: 2 },
latency_ms: {
count: 3,
sample_count: 3,
p50: 20,
p95: 30,
p99: 30,
max: 30,
},
const snapshot = metrics.snapshot();
expect(snapshot.counters).toEqual({ cache_hit: 2 });
expect(snapshot.latency_ms).toEqual({
count: 3,
sample_count: 3,
p50: 20,
p95: 30,
p99: 30,
max: 30,
});
});
@@ -59,4 +58,20 @@ describe('MetricsService', () => {
max: 0,
});
});
it('adds byte counters and ignores invalid amounts', () => {
const metrics = new MetricsService();
metrics.add('bytes', 100);
metrics.add('bytes', 28);
for (const value of [-1, NaN, Infinity]) metrics.add('bytes', value);
expect(metrics.snapshot().counters.bytes).toBe(128);
});
it('tracks memory peaks until reset', () => {
const metrics = new MetricsService();
const ballast = Buffer.alloc(32 * 1024 * 1024, 1);
const during = metrics.snapshot().memory_peak_bytes.array_buffers;
expect(during).toBeGreaterThanOrEqual(ballast.length);
ballast.fill(0);
});
});
+56 -2
View File
@@ -1,4 +1,4 @@
import { Injectable } from '@nestjs/common';
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
const LATENCY_SAMPLE_CAPACITY = 4096;
@@ -11,14 +11,53 @@ type LatencySnapshot = {
max: number;
};
const MEMORY_SAMPLE_INTERVAL_MS = 50;
type MemoryUsage = {
rss: number;
heap_used: number;
external: number;
array_buffers: number;
};
function currentMemory(): MemoryUsage {
const usage = process.memoryUsage();
return {
rss: usage.rss,
heap_used: usage.heapUsed,
external: usage.external,
array_buffers: usage.arrayBuffers,
};
}
@Injectable()
export class MetricsService {
export class MetricsService implements OnModuleInit, OnModuleDestroy {
private memoryPeak: MemoryUsage = currentMemory();
private memoryTimer: NodeJS.Timeout | undefined;
private readonly counters: Record<string, number> = {};
private readonly latencies: number[] = [];
private latencyCount = 0;
private maximumLatency = 0;
private nextSample = 0;
onModuleInit(): void {
this.memoryTimer = setInterval(
() => this.sampleMemory(),
MEMORY_SAMPLE_INTERVAL_MS,
);
this.memoryTimer.unref();
}
onModuleDestroy(): void {
if (this.memoryTimer) clearInterval(this.memoryTimer);
this.memoryTimer = undefined;
}
add(name: string, amount: number): void {
if (!Number.isFinite(amount) || amount < 0) return;
this.counters[name] = (this.counters[name] ?? 0) + amount;
}
increment(name: string): void {
this.counters[name] = (this.counters[name] ?? 0) + 1;
}
@@ -34,7 +73,10 @@ export class MetricsService {
snapshot(): {
counters: Record<string, number>;
latency_ms: LatencySnapshot;
memory_bytes: MemoryUsage;
memory_peak_bytes: MemoryUsage;
} {
this.sampleMemory();
const sorted = [...this.latencies].sort((left, right) => left - right);
return {
counters: { ...this.counters },
@@ -46,6 +88,8 @@ export class MetricsService {
p99: this.percentile(sorted, 0.99),
max: this.maximumLatency,
},
memory_bytes: currentMemory(),
memory_peak_bytes: { ...this.memoryPeak },
};
}
@@ -56,6 +100,16 @@ export class MetricsService {
this.latencyCount = 0;
this.maximumLatency = 0;
this.nextSample = 0;
this.memoryPeak = currentMemory();
}
private sampleMemory(): void {
const now = currentMemory();
const peak = this.memoryPeak;
peak.rss = Math.max(peak.rss, now.rss);
peak.heap_used = Math.max(peak.heap_used, now.heap_used);
peak.external = Math.max(peak.external, now.external);
peak.array_buffers = Math.max(peak.array_buffers, now.array_buffers);
}
private percentile(sorted: number[], percentile: number): number {
+184
View File
@@ -0,0 +1,184 @@
import {
DeleteObjectCommand,
GetObjectCommand,
HeadObjectCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { NodeHttpHandler } from '@smithy/node-http-handler';
import {
Injectable,
Logger,
ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import http from 'node:http';
import { Readable } from 'node:stream';
import type { AppConfig } from '../config/app-config.schema.js';
export class ObjectStorageUnavailableError extends ServiceUnavailableException {
constructor() {
super('Object storage unavailable');
}
}
export type PresignPutOptions = {
contentType: string;
contentLength: number;
checksumSha256Base64: string;
expiresInSeconds: number;
};
function isMissingKey(error: unknown): boolean {
const candidate = error as {
name?: string;
$metadata?: { httpStatusCode?: number };
};
return (
candidate?.name === 'NoSuchKey' ||
candidate?.name === 'NotFound' ||
candidate?.$metadata?.httpStatusCode === 404
);
}
@Injectable()
export class ObjectStorage {
private readonly logger = new Logger(ObjectStorage.name);
private readonly bucket: string;
private readonly internal: S3Client;
private readonly presign: S3Client;
constructor(config: ConfigService<AppConfig>) {
this.bucket = config.getOrThrow('S3_BUCKET');
const common = {
region: config.getOrThrow('S3_REGION'),
forcePathStyle: true,
credentials: {
accessKeyId: config.getOrThrow('S3_ACCESS_KEY_ID'),
secretAccessKey: config.getOrThrow('S3_SECRET_ACCESS_KEY'),
},
// Поток нельзя повторить: повтор — решение клиента API.
maxAttempts: 1,
requestChecksumCalculation: 'WHEN_REQUIRED' as const,
responseChecksumValidation: 'WHEN_REQUIRED' as const,
requestHandler: new NodeHttpHandler({
connectionTimeout: config.getOrThrow('S3_CONNECT_TIMEOUT_MS'),
requestTimeout: config.getOrThrow('S3_REQUEST_TIMEOUT_MS'),
httpAgent: new http.Agent({ keepAlive: true, maxSockets: 64 }),
}),
};
this.internal = new S3Client({
...common,
endpoint: config.getOrThrow('S3_ENDPOINT'),
});
// Сеть не использует: только подпись URL для клиента.
this.presign = new S3Client({
...common,
endpoint: config.getOrThrow('S3_PUBLIC_ENDPOINT'),
});
}
objectKey(attachmentId: string): string {
return `attachments/${attachmentId}/original`;
}
async putStream(
key: string,
body: Readable,
contentLength: number,
contentType: string,
signal: AbortSignal,
): Promise<void> {
try {
await this.internal.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: key,
Body: body,
ContentLength: contentLength,
ContentType: contentType,
}),
{ abortSignal: signal },
);
} catch (error) {
throw this.fail('put', error);
}
}
async getStream(key: string, range?: string): Promise<Readable | null> {
try {
const result = await this.internal.send(
new GetObjectCommand({
Bucket: this.bucket,
Key: key,
Range: range,
}),
);
return result.Body as Readable;
} catch (error) {
if (isMissingKey(error)) return null;
throw this.fail('get', error);
}
}
async head(
key: string,
): Promise<{ size: number; checksumSha256?: string } | null> {
try {
const result = await this.internal.send(
new HeadObjectCommand({
Bucket: this.bucket,
Key: key,
ChecksumMode: 'ENABLED',
}),
);
return {
size: Number(result.ContentLength ?? 0),
checksumSha256: result.ChecksumSHA256,
};
} catch (error) {
if (isMissingKey(error)) return null;
throw this.fail('head', error);
}
}
async delete(key: string): Promise<void> {
try {
await this.internal.send(
new DeleteObjectCommand({ Bucket: this.bucket, Key: key }),
);
} catch (error) {
if (isMissingKey(error)) return;
throw this.fail('delete', error);
}
}
async presignPut(key: string, options: PresignPutOptions): Promise<string> {
return getSignedUrl(
this.presign,
new PutObjectCommand({
Bucket: this.bucket,
Key: key,
ContentType: options.contentType,
ContentLength: options.contentLength,
ChecksumSHA256: options.checksumSha256Base64,
}),
{
expiresIn: options.expiresInSeconds,
signableHeaders: new Set([
'content-type',
'content-length',
'x-amz-checksum-sha256',
]),
unhoistableHeaders: new Set(['x-amz-checksum-sha256']),
},
);
}
private fail(operation: string, error: unknown): Error {
const name = (error as { name?: string })?.name ?? 'UnknownError';
this.logger.warn(`S3 ${operation} failed: ${name}`);
return new ObjectStorageUnavailableError();
}
}
+5
View File
@@ -0,0 +1,5 @@
import { Module } from '@nestjs/common';
import { ObjectStorage } from './object-storage.js';
@Module({ providers: [ObjectStorage], exports: [ObjectStorage] })
export class StorageModule {}