Serialize build retirement and clone classification, bound ops accepts
Review fixes for M7.6-M7.9. Schema v9 applies one lock order: the barrier takes a database-wide upgrade lock exclusively, while acquisitions, lease checks, provisioning and the retired-build trigger take it shared first, so a queued acquisition can no longer regain authority for a just-retired build and concurrent barriers cannot retire the last current build. Provisioning takes a per-world lock before classifying a world as original or a clone of one source. The operations listener makes at most four accept attempts per serve(). Race regressions prove each contender is blocked before commit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,9 +1,11 @@
|
||||
// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is
|
||||
// never authority, readiness follows the lease, metrics are bounded, and the
|
||||
// responder is bounded against idle, excess and malformed clients.
|
||||
#include "evo/server/ops_http.hpp"
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <arpa/inet.h>
|
||||
#include <atomic>
|
||||
#include <netinet/in.h>
|
||||
#include <source_location>
|
||||
#include <sys/socket.h>
|
||||
@@ -184,9 +186,84 @@ void stale_process_is_live_not_ready() {
|
||||
current.signal(SIGTERM);
|
||||
CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0);
|
||||
}
|
||||
|
||||
// Review fix: accept work per serve() is bounded, refused connections included, so
|
||||
// replenished connection pressure cannot hold the host loop; accepted connections
|
||||
// are still answered while the pressure continues.
|
||||
void accept_work_is_bounded_per_serve() {
|
||||
using evo::server::OpsServer;
|
||||
auto made = OpsServer::listen("127.0.0.1:0");
|
||||
CHECK(made);
|
||||
auto& ops = **made;
|
||||
const auto handler = [](std::string_view) { return evo::server::OpsResponse{200, "ok\n"}; };
|
||||
constexpr auto per = OpsServer::max_accepts_per_serve;
|
||||
std::vector<int> clients;
|
||||
clients.reserve(64);
|
||||
for (int i = 0; i < 16; ++i)
|
||||
clients.push_back(connect_local(ops.port())); // Queued in the listen backlog.
|
||||
std::this_thread::sleep_for(50ms);
|
||||
ops.serve(handler);
|
||||
CHECK(ops.open_connections() == per && ops.refused() == 0);
|
||||
ops.serve(handler);
|
||||
CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == 0);
|
||||
// At the connection cap, each call still refuses at most the budget.
|
||||
ops.serve(handler);
|
||||
CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == per);
|
||||
// The first accepted client is answered while new connections keep arriving.
|
||||
const std::string request = "GET /livez HTTP/1.1\r\n\r\n";
|
||||
CHECK(::send(clients[0], request.data(), request.size(), MSG_NOSIGNAL) ==
|
||||
static_cast<ssize_t>(request.size()));
|
||||
for (int round = 0; round < 6; ++round) {
|
||||
clients.push_back(connect_local(ops.port()));
|
||||
const auto before = ops.refused();
|
||||
ops.serve(handler);
|
||||
CHECK(ops.refused() - before <= per);
|
||||
}
|
||||
char reply[64] = {};
|
||||
CHECK(::recv(clients[0], reply, sizeof reply - 1, 0) > 0);
|
||||
CHECK(std::string_view(reply).starts_with("HTTP/1.1 200"));
|
||||
for (const int fd : clients)
|
||||
::close(fd);
|
||||
}
|
||||
|
||||
// A real server under sustained connection churn keeps its lease and readiness.
|
||||
void connection_pressure_keeps_authority() {
|
||||
const auto r = provision(54, 0x0D);
|
||||
Child server{with_ops(r)};
|
||||
const auto port = ops_port(server);
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
std::atomic<bool> stop{false};
|
||||
std::thread flood([&] {
|
||||
while (!stop.load()) {
|
||||
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
sockaddr_in address{};
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_port = htons(port);
|
||||
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
if (fd >= 0) {
|
||||
// Refusal or backlog overflow is part of the pressure; ignore it.
|
||||
[[maybe_unused]] const int connected =
|
||||
::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address);
|
||||
::close(fd);
|
||||
}
|
||||
}
|
||||
});
|
||||
// Longer than the fast lease (4 s): renewals must keep happening under pressure.
|
||||
server.send(input_lines(1, 60, false));
|
||||
const bool acked = server.wait_line("ack " + request_hex(60) + " 60 ", 20s);
|
||||
stop = true;
|
||||
flood.join();
|
||||
CHECK(acked);
|
||||
CHECK(!server.printed("ready 0"));
|
||||
CHECK(get(port, "/readyz").status == 200);
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_ops_tests() {
|
||||
accept_work_is_bounded_per_serve();
|
||||
connection_pressure_keeps_authority();
|
||||
probes_follow_authority_and_stay_bounded();
|
||||
unavailable_database_is_live_not_ready();
|
||||
stale_process_is_live_not_ready();
|
||||
|
||||
Reference in New Issue
Block a user