Исправить readiness LB и ресурсные лимиты

This commit is contained in:
EgorAnti
2026-11-05 12:48:30 +03:00
parent 7080fdd3e2
commit 9c3e4fd126
24 changed files with 466 additions and 45 deletions
@@ -4,6 +4,7 @@ import com.example.mtusi_messenger.security.ApiAuthenticationFilter;
import com.example.mtusi_messenger.security.DatabaseAuthenticationProvider;
import com.example.mtusi_messenger.security.RestAuthenticationEntryPoint;
import com.example.mtusi_messenger.service.AuthenticationService;
import com.example.mtusi_messenger.web.ReadinessGuardFilter;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
@@ -61,10 +62,20 @@ public class SecurityConfig {
return registration;
}
@Bean
FilterRegistrationBean<ReadinessGuardFilter> disableReadinessGuardServletRegistration(
ReadinessGuardFilter filter
) {
var registration = new FilterRegistrationBean<>(filter);
registration.setEnabled(false);
return registration;
}
@Bean
SecurityFilterChain securityFilterChain(
HttpSecurity http,
ApiAuthenticationFilter apiAuthenticationFilter,
ReadinessGuardFilter readinessGuardFilter,
@Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint
) throws Exception {
http
@@ -86,7 +97,11 @@ public class SecurityConfig {
).permitAll()
.requestMatchers(HttpMethod.POST, "/api/v1/users").permitAll()
.anyRequest().authenticated())
.addFilterBefore(apiAuthenticationFilter, AnonymousAuthenticationFilter.class);
// Refuse business traffic before authentication touches storage.
// Nginx retries this 503 on another peer and passively excludes
// the refusing replica according to max_fails/fail_timeout.
.addFilterBefore(readinessGuardFilter, AnonymousAuthenticationFilter.class)
.addFilterAfter(apiAuthenticationFilter, ReadinessGuardFilter.class);
return http.build();
}
}
@@ -1,31 +1,18 @@
package com.example.mtusi_messenger.controller;
import org.springframework.http.ResponseEntity;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.boot.availability.ApplicationAvailability;
import org.springframework.boot.availability.ReadinessState;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import com.example.mtusi_messenger.dto.HealthResponse;
import com.example.mtusi_messenger.exception.ApiExceptions;
import com.example.mtusi_messenger.service.ReadinessService;
@RestController
public class HealthController {
private final JdbcTemplate jdbc;
private final StringRedisTemplate redis;
private final ApplicationAvailability availability;
private final ReadinessService readiness;
public HealthController(
JdbcTemplate jdbc,
StringRedisTemplate redis,
ApplicationAvailability availability
) {
this.jdbc = jdbc;
this.redis = redis;
this.availability = availability;
public HealthController(ReadinessService readiness) {
this.readiness = readiness;
}
@GetMapping("/health/live")
@@ -35,24 +22,11 @@ public class HealthController {
@GetMapping("/health/ready")
public ResponseEntity<HealthResponse> ready() {
if (availability.getReadinessState() != ReadinessState.ACCEPTING_TRAFFIC) {
// Always probe dependencies, even after a failure, so a restored schema
// can move this process back to ACCEPTING_TRAFFIC without a restart.
if (!readiness.probeDependencies() || !readiness.acceptsBusinessTraffic()) {
throw ApiExceptions.unavailable("Application is not accepting traffic");
}
try {
// Warm the lazy Lettuce connection on every replica. Redis remains
// an optional cache: failure is ignored and business reads fall
// back to PostgreSQL.
redis.execute((RedisCallback<String>) connection -> connection.ping());
} catch (DataAccessException ignored) {
// Optional cache unavailable; PostgreSQL remains authoritative.
}
try {
// A schema-level probe prevents a replica from becoming ready
// before the separately executed migration job has completed.
jdbc.queryForObject("SELECT count(*) FROM message_idempotency", Long.class);
return ResponseEntity.ok(new HealthResponse("ok"));
} catch (DataAccessException e) {
throw ApiExceptions.unavailable("Database is unavailable");
}
return ResponseEntity.ok(new HealthResponse("ok"));
}
}
@@ -0,0 +1,81 @@
package com.example.mtusi_messenger.service;
import org.springframework.boot.availability.ApplicationAvailability;
import org.springframework.boot.availability.AvailabilityChangeEvent;
import org.springframework.boot.availability.ReadinessState;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import java.util.concurrent.atomic.AtomicBoolean;
@Service
public class ReadinessService {
private final JdbcTemplate jdbc;
private final StringRedisTemplate redis;
private final ApplicationAvailability availability;
private final ApplicationEventPublisher events;
private final AtomicBoolean dependencyUnavailable = new AtomicBoolean(false);
private final AtomicBoolean shuttingDown = new AtomicBoolean(false);
public ReadinessService(
JdbcTemplate jdbc,
StringRedisTemplate redis,
ApplicationAvailability availability,
ApplicationEventPublisher events
) {
this.jdbc = jdbc;
this.redis = redis;
this.availability = availability;
this.events = events;
}
public boolean probeDependencies() {
try {
// Redis is an optional cache. Ping only warms the lazy connection;
// its failure must not make the authoritative API unavailable.
redis.execute((RedisCallback<String>) connection -> connection.ping());
} catch (DataAccessException ignored) {
// PostgreSQL remains the source of truth.
}
try {
// The schema-level probe also proves that the separate V3 migration
// completed before this replica is allowed to serve business traffic.
jdbc.queryForObject("SELECT count(*) FROM message_idempotency", Long.class);
markDependenciesAvailable();
return true;
} catch (DataAccessException e) {
markDependenciesUnavailable();
return false;
}
}
public boolean acceptsBusinessTraffic() {
return !shuttingDown.get()
&& !dependencyUnavailable.get()
&& availability.getReadinessState() == ReadinessState.ACCEPTING_TRAFFIC;
}
@EventListener(ContextClosedEvent.class)
void onContextClosed() {
shuttingDown.set(true);
}
private void markDependenciesUnavailable() {
dependencyUnavailable.set(true);
AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC);
}
private void markDependenciesAvailable() {
boolean recovered = dependencyUnavailable.getAndSet(false);
if (recovered && !shuttingDown.get()) {
AvailabilityChangeEvent.publish(events, this, ReadinessState.ACCEPTING_TRAFFIC);
}
}
}
@@ -0,0 +1,56 @@
package com.example.mtusi_messenger.web;
import com.example.mtusi_messenger.dto.ErrorResponse;
import com.example.mtusi_messenger.service.ReadinessService;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
import java.io.IOException;
@Component
public class ReadinessGuardFilter extends OncePerRequestFilter {
private final ReadinessService readiness;
private final ObjectMapper mapper;
public ReadinessGuardFilter(ReadinessService readiness, ObjectMapper mapper) {
this.readiness = readiness;
this.mapper = mapper;
}
@Override
protected boolean shouldNotFilter(HttpServletRequest request) {
String path = request.getRequestURI();
return path.startsWith("/health/")
|| path.equals("/swagger-ui.html")
|| path.startsWith("/swagger-ui/")
|| path.startsWith("/v3/api-docs/")
|| path.equals("/v3/api-docs");
}
@Override
protected void doFilterInternal(
HttpServletRequest request,
HttpServletResponse response,
FilterChain filterChain
) throws ServletException, IOException {
if (readiness.acceptsBusinessTraffic()) {
filterChain.doFilter(request, response);
return;
}
response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
response.setContentType(MediaType.APPLICATION_JSON_VALUE);
String requestId = (String) request.getAttribute(RequestIdFilter.ATTRIBUTE);
mapper.writeValue(response.getOutputStream(), new ErrorResponse(
new ErrorResponse.ErrorBody(
"unavailable",
"Application is not ready to serve business traffic",
requestId)));
}
}