diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d63b199..1cfb6ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,14 @@ jobs: - uses: actions/setup-go@v5 with: go-version: '1.24' + - name: Install Go protobuf generator + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 + - name: Generate Go protobuf + run: cd tests && make proto + - name: Download Go deps + run: cd tests && go mod download + - name: Validate CI gate contracts + run: cd tests && go test ./pkg/contracts -count=1 - name: Go vet run: cd tests && go vet ./... - name: Go fmt check @@ -33,8 +41,40 @@ jobs: exit 1 fi + service-artifacts: + runs-on: ubuntu-22.04 + if: github.event_name != 'push' || github.ref != 'refs/heads/main' + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Build native-service builder image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/ci/Dockerfile + load: true + tags: chatnow-ci-builder:ci + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build all Compose services once + run: | + docker run --rm -v "$PWD:/workspace" -w /workspace chatnow-ci-builder:ci bash -lc ' + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release + cmake --build build --parallel "$(nproc)" --target conversation_server gateway_server identity_server media_server message_server presence_server push_server relationship_server transmite_server + ' + - name: Package Compose service artifacts + run: docker run --rm -v "$PWD:/workspace" -w /workspace chatnow-ci-builder:ci ./scripts/package_compose_artifacts.sh build compose-artifacts + - name: Validate Compose service artifacts + run: docker run --rm -v "$PWD:/workspace" -w /workspace chatnow-ci-builder:ci ./scripts/validate_compose_artifacts.sh compose-artifacts + - name: Upload Compose service artifacts + uses: actions/upload-artifact@v4 + with: + name: compose-service-artifacts + path: compose-artifacts + if-no-files-found: error + bvt: - needs: build + needs: service-artifacts runs-on: ubuntu-22.04 if: github.event_name != 'push' || github.ref != 'refs/heads/main' steps: @@ -46,6 +86,32 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler netcat-openbsd + - name: Install Go protobuf generator + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 + - &download-compose-artifacts + name: Download Compose service artifacts + uses: actions/download-artifact@v4 + with: + name: compose-service-artifacts + path: compose-artifacts + - &restore-compose-artifacts + name: Restore Compose build contexts + run: | + for service in conversation gateway identity media message presence push relationship transmite; do + rm -rf "$service/build" "$service/depends" + cp -a "compose-artifacts/$service/build" "$service/build" + cp -a "compose-artifacts/$service/depends" "$service/depends" + done + - &chmod-compose-artifacts + name: Restore service executable modes + run: | + for service in conversation gateway identity media message presence push relationship transmite; do + chmod +x "compose-artifacts/$service/build/${service}_server" + chmod +x "$service/build/${service}_server" + done + - &validate-compose-artifacts + name: Validate downloaded Compose service artifacts + run: docker run --rm -v "$PWD:/workspace" -w /workspace ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 ./scripts/validate_compose_artifacts.sh compose-artifacts - name: Start full stack run: docker compose up -d --build - name: Wait for services @@ -61,7 +127,7 @@ jobs: run: docker compose down -v func: - needs: bvt + needs: [service-artifacts, bvt] runs-on: ubuntu-22.04 if: github.event_name == 'pull_request' || github.event_name == 'schedule' steps: @@ -73,6 +139,12 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler netcat-openbsd + - name: Install Go protobuf generator + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 + - *download-compose-artifacts + - *restore-compose-artifacts + - *chmod-compose-artifacts + - *validate-compose-artifacts - name: Start full stack run: docker compose up -d --build - name: Wait for services @@ -87,6 +159,76 @@ jobs: if: always() run: docker compose down -v + reliability: + needs: service-artifacts + runs-on: ubuntu-22.04 + if: github.event_name == 'pull_request' || github.event_name == 'schedule' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler netcat-openbsd + - name: Install Go protobuf generator + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 + - *download-compose-artifacts + - *restore-compose-artifacts + - *chmod-compose-artifacts + - *validate-compose-artifacts + - name: Start full stack + run: docker compose up -d --build + - name: Wait for services + run: ./scripts/wait_for_services.sh + - name: Generate Go protobuf + run: cd tests && make proto + - name: Download Go deps + run: cd tests && go mod download + - name: Run cache reliability gate + run: cd tests && make test-reliability + - name: Tear down + if: always() + run: docker compose down -v + + perf-cache: + needs: service-artifacts + runs-on: ubuntu-22.04 + if: github.event_name == 'schedule' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler netcat-openbsd + - name: Install Go protobuf generator + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 + - *download-compose-artifacts + - *restore-compose-artifacts + - *chmod-compose-artifacts + - *validate-compose-artifacts + - name: Start full stack with PF-09 rate limits + env: + # Transmite flags are int32; use the maximum valid value for gate-only headroom. + TRANSMITE_RATE_LIMIT_USER_MAX: '2147483647' + TRANSMITE_RATE_LIMIT_SESSION_MAX: '2147483647' + run: docker compose up -d --build + - name: Wait for services + run: ./scripts/wait_for_services.sh + - name: Generate Go protobuf + run: cd tests && make proto + - name: Download Go deps + run: cd tests && go mod download + - name: Run cache performance gate + run: cd tests && make test-perf-cache-gate + - name: Tear down + if: always() + run: docker compose down -v + perf: needs: func runs-on: ubuntu-22.04 @@ -100,6 +242,8 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler netcat-openbsd + - name: Install Go protobuf generator + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 - name: Start full stack run: docker compose up -d --build - name: Wait for services diff --git a/common/dao/data_redis.hpp b/common/dao/data_redis.hpp index 7e096ea..59a6027 100644 --- a/common/dao/data_redis.hpp +++ b/common/dao/data_redis.hpp @@ -20,20 +20,183 @@ #include #include #include +#include +#include #include +#include #include #include +#include +#include +#include +#include #include #include #include #include "infra/logger.hpp" +#include "infra/metrics.hpp" #include "utils/cache_version.hpp" +#include "utils/local_rate_limiter.hpp" #include "utils/random_ttl.hpp" +#include "utils/redis_circuit_breaker.hpp" #include "utils/redis_keys.hpp" namespace chatnow { +inline bool is_redis_pool_wait_error(const sw::redis::Error &error) noexcept { + if (typeid(error) != typeid(sw::redis::Error)) return false; + constexpr std::string_view prefix = "Failed to fetch a connection in "; + constexpr std::string_view suffix = " milliseconds"; + const std::string_view message(error.what()); + if (message.size() <= prefix.size() + suffix.size() || + message.compare(0, prefix.size(), prefix) != 0 || + message.compare(message.size() - suffix.size(), suffix.size(), suffix) != 0) { + return false; + } + const auto milliseconds = message.substr( + prefix.size(), message.size() - prefix.size() - suffix.size()); + for (const char ch : milliseconds) { + if (ch < '0' || ch > '9') return false; + } + return true; +} + +class RedisPipeline { +public: + RedisPipeline(sw::redis::Pipeline pipeline, + std::shared_ptr breaker, + RedisCircuitBreaker::Permit permit) + : _pipeline(std::move(pipeline)), _breaker(std::move(breaker)), _permit(permit) {} + + RedisPipeline(RedisPipeline &&other) noexcept + : _pipeline(std::move(other._pipeline)), + _breaker(std::move(other._breaker)), + _permit(other._permit), + _settled(other._settled) { + other._settled = true; + } + RedisPipeline &operator=(RedisPipeline &&other) noexcept { + if (this == &other) return *this; + abandon_(); + _pipeline = std::move(other._pipeline); + _breaker = std::move(other._breaker); + _permit = other._permit; + _settled = other._settled; + other._settled = true; + return *this; + } + RedisPipeline(const RedisPipeline &) = delete; + RedisPipeline &operator=(const RedisPipeline &) = delete; + ~RedisPipeline() { abandon_(); } + + template + RedisPipeline &hset(Args &&...args) { + return queue_([&] { _pipeline.hset(std::forward(args)...); }); + } + + template + RedisPipeline &set(Args &&...args) { + return queue_([&] { _pipeline.set(std::forward(args)...); }); + } + + template + RedisPipeline &expire(Args &&...args) { + return queue_([&] { _pipeline.expire(std::forward(args)...); }); + } + + template + RedisPipeline &get(Args &&...args) { + return queue_([&] { _pipeline.get(std::forward(args)...); }); + } + + sw::redis::QueuedReplies exec() { + if (_settled) { + throw std::logic_error("RedisPipeline::exec called more than once"); + } + try { + auto replies = _pipeline.exec(); + settle_success_(); + return replies; + } catch (const sw::redis::IoError &) { + record_connection_failure_(); + throw; + } catch (const sw::redis::ClosedError &) { + record_connection_failure_(); + throw; + } catch (const sw::redis::ReplyError &) { + settle_success_(); + throw; + } catch (const sw::redis::Error &error) { + if (is_redis_pool_wait_error(error)) record_connection_failure_(); + else abandon_(); + throw; + } catch (...) { + abandon_(); + throw; + } + } + +private: + template + RedisPipeline &queue_(F &&queue_command) { + if (_settled) { + throw std::logic_error("RedisPipeline command queued after settlement"); + } + try { + std::forward(queue_command)(); + return *this; + } catch (const sw::redis::IoError &) { + record_connection_failure_(); + throw; + } catch (const sw::redis::ClosedError &) { + record_connection_failure_(); + throw; + } catch (const sw::redis::Error &error) { + if (is_redis_pool_wait_error(error)) record_connection_failure_(); + else abandon_(); + throw; + } catch (...) { + abandon_(); + throw; + } + } + + void record_transition_(RedisCircuitBreaker::Transition transition) noexcept { + if (transition == RedisCircuitBreaker::Transition::Opened) { + metrics::g_redis_circuit_open_total << 1; + LOG_WARN("Redis circuit transition ->Open"); + } else if (transition == RedisCircuitBreaker::Transition::Recovered) { + metrics::g_redis_circuit_recovered_total << 1; + LOG_INFO("Redis circuit transition HalfOpen->Closed"); + } + } + + void settle_success_() noexcept { + if (_settled) return; + _settled = true; + record_transition_(_breaker->on_success(_permit)); + } + + void record_connection_failure_() noexcept { + if (_settled) return; + _settled = true; + metrics::g_redis_call_failure_total << 1; + record_transition_(_breaker->on_connection_failure(_permit)); + } + + void abandon_() noexcept { + if (_settled || !_breaker) return; + _settled = true; + record_transition_(_breaker->on_abandoned(_permit)); + } + + sw::redis::Pipeline _pipeline; + std::shared_ptr _breaker; + RedisCircuitBreaker::Permit _permit; + bool _settled = false; +}; + // 类型擦除 Redis 客户端适配器：根据持有的后端类型透明转发到 // sw::redis::Redis（单机）或 sw::redis::RedisCluster。所有 cache 类 // 统一使用 RedisClient::ptr，对调用方完全透明。每次调用一次分支 @@ -43,122 +206,157 @@ class RedisClient public: using ptr = std::shared_ptr; - RedisClient(std::shared_ptr r) : _r(std::move(r)) {} - RedisClient(std::shared_ptr rc) : _rc(std::move(rc)) {} + RedisClient(std::shared_ptr r) + : _r(std::move(r)), _breaker(std::make_shared()) {} + RedisClient(std::shared_ptr rc) + : _rc(std::move(rc)), _breaker(std::make_shared()) {} // --- String commands --- sw::redis::OptionalString get(const std::string &key) { - return _rc ? _rc->get(key) : _r->get(key); + return guarded_([&] { return _rc ? _rc->get(key) : _r->get(key); }); } bool set(const std::string &key, const std::string &val, std::chrono::seconds ttl = std::chrono::seconds(0)) { - return _rc ? _rc->set(key, val, ttl) : _r->set(key, val, ttl); + return guarded_([&] { return _rc ? _rc->set(key, val, ttl) : _r->set(key, val, ttl); }); } bool set(const std::string &key, const std::string &val, std::chrono::milliseconds ttl) { - return _rc ? _rc->set(key, val, ttl) : _r->set(key, val, ttl); + return guarded_([&] { return _rc ? _rc->set(key, val, ttl) : _r->set(key, val, ttl); }); } bool set(const std::string &key, const std::string &val, std::chrono::seconds ttl, sw::redis::UpdateType type) { - return _rc ? _rc->set(key, val, ttl, type) : _r->set(key, val, ttl, type); + return guarded_([&] { return _rc ? _rc->set(key, val, ttl, type) : _r->set(key, val, ttl, type); }); } bool set(const std::string &key, const std::string &val, std::chrono::milliseconds ttl, sw::redis::UpdateType type) { - return _rc ? _rc->set(key, val, ttl, type) : _r->set(key, val, ttl, type); + return guarded_([&] { return _rc ? _rc->set(key, val, ttl, type) : _r->set(key, val, ttl, type); }); } long long del(const std::string &key) { - return _rc ? _rc->del(key) : _r->del(key); + return guarded_([&] { return _rc ? _rc->del(key) : _r->del(key); }); } void expire(const std::string &key, std::chrono::seconds ttl) { - _rc ? _rc->expire(key, ttl) : _r->expire(key, ttl); + guarded_void_([&] { _rc ? _rc->expire(key, ttl) : _r->expire(key, ttl); }); } long long incr(const std::string &key) { - return _rc ? _rc->incr(key) : _r->incr(key); + return guarded_([&] { return _rc ? _rc->incr(key) : _r->incr(key); }); } // --- Set commands --- template long long sadd(const std::string &key, const T &member) { - return _rc ? _rc->sadd(key, member) : _r->sadd(key, member); + return guarded_([&] { return _rc ? _rc->sadd(key, member) : _r->sadd(key, member); }); } template long long sadd(const std::string &key, It first, It last) { - return _rc ? _rc->sadd(key, first, last) : _r->sadd(key, first, last); + return guarded_([&] { return _rc ? _rc->sadd(key, first, last) : _r->sadd(key, first, last); }); } template void smembers(const std::string &key, Out out) { - _rc ? _rc->smembers(key, out) : _r->smembers(key, out); + guarded_void_([&] { _rc ? _rc->smembers(key, out) : _r->smembers(key, out); }); } template long long srem(const std::string &key, const T &member) { - return _rc ? _rc->srem(key, member) : _r->srem(key, member); + return guarded_([&] { return _rc ? _rc->srem(key, member) : _r->srem(key, member); }); } long long scard(const std::string &key) { - return _rc ? _rc->scard(key) : _r->scard(key); + return guarded_([&] { return _rc ? _rc->scard(key) : _r->scard(key); }); } // --- Hash commands --- long long hset(const std::string &key, const std::string &field, const std::string &val) { - return _rc ? _rc->hset(key, field, val) : _r->hset(key, field, val); + return guarded_([&] { return _rc ? _rc->hset(key, field, val) : _r->hset(key, field, val); }); } sw::redis::OptionalString hget(const std::string &key, const std::string &field) { - return _rc ? _rc->hget(key, field) : _r->hget(key, field); + return guarded_([&] { return _rc ? _rc->hget(key, field) : _r->hget(key, field); }); } long long hdel(const std::string &key, const std::string &field) { - return _rc ? _rc->hdel(key, field) : _r->hdel(key, field); + return guarded_([&] { return _rc ? _rc->hdel(key, field) : _r->hdel(key, field); }); } template void hkeys(const std::string &key, Out out) { - _rc ? _rc->hkeys(key, out) : _r->hkeys(key, out); + guarded_void_([&] { _rc ? _rc->hkeys(key, out) : _r->hkeys(key, out); }); } template void hgetall(const std::string &key, Out out) { - _rc ? _rc->hgetall(key, out) : _r->hgetall(key, out); + guarded_void_([&] { _rc ? _rc->hgetall(key, out) : _r->hgetall(key, out); }); } long long hlen(const std::string &key) { - return _rc ? _rc->hlen(key) : _r->hlen(key); + return guarded_([&] { return _rc ? _rc->hlen(key) : _r->hlen(key); }); } // --- Sorted Set commands --- long long zadd(const std::string &key, const std::string &member, double score) { - return _rc ? _rc->zadd(key, member, score) : _r->zadd(key, member, score); + return guarded_([&] { return _rc ? _rc->zadd(key, member, score) : _r->zadd(key, member, score); }); } long long zadd(const std::string &key, const std::string &member, double score, sw::redis::UpdateType type) { - return _rc ? _rc->zadd(key, member, score, type) : _r->zadd(key, member, score, type); + return guarded_([&] { return _rc ? _rc->zadd(key, member, score, type) : _r->zadd(key, member, score, type); }); } long long zrem(const std::string &key, const std::string &member) { - return _rc ? _rc->zrem(key, member) : _r->zrem(key, member); + return guarded_([&] { return _rc ? _rc->zrem(key, member) : _r->zrem(key, member); }); } template void zrange(const std::string &key, long long start, long long stop, Out out) { - _rc ? _rc->zrange(key, start, stop, out) : _r->zrange(key, start, stop, out); + guarded_void_([&] { _rc ? _rc->zrange(key, start, stop, out) : _r->zrange(key, start, stop, out); }); } template void zrangebyscore(const std::string &key, const sw::redis::BoundedInterval &interval, const sw::redis::LimitOptions &opts, Out out) { - _rc ? _rc->zrangebyscore(key, interval, opts, out) - : _r->zrangebyscore(key, interval, opts, out); + guarded_void_([&] { + _rc ? _rc->zrangebyscore(key, interval, opts, out) + : _r->zrangebyscore(key, interval, opts, out); + }); } // --- Lua scripting --- template Ret eval(const std::string &script, KeyIt key_first, KeyIt key_last, ArgIt arg_first, ArgIt arg_last) { - return _rc ? _rc->eval(script, key_first, key_last, arg_first, arg_last) - : _r->eval(script, key_first, key_last, arg_first, arg_last); + return guarded_([&]() -> Ret { + return _rc ? _rc->eval(script, key_first, key_last, arg_first, arg_last) + : _r->eval(script, key_first, key_last, arg_first, arg_last); + }); } template void eval(const std::string &script, KeyIt key_first, KeyIt key_last, ArgIt arg_first, ArgIt arg_last, Out out) { - _rc ? _rc->eval(script, key_first, key_last, arg_first, arg_last, out) - : _r->eval(script, key_first, key_last, arg_first, arg_last, out); + guarded_void_([&] { + _rc ? _rc->eval(script, key_first, key_last, arg_first, arg_last, out) + : _r->eval(script, key_first, key_last, arg_first, arg_last, out); + }); } // --- Pipeline --- - auto pipeline(const sw::redis::StringView &hash_tag = {}) { - return _rc ? _rc->pipeline(hash_tag) : _r->pipeline(); + RedisPipeline pipeline(const sw::redis::StringView &hash_tag = {}) { + auto permit = before_call_(); + try { + return RedisPipeline(_rc ? _rc->pipeline(hash_tag) : _r->pipeline(), + _breaker, permit); + } catch (const sw::redis::IoError &) { + record_connection_failure_(permit); + throw; + } catch (const sw::redis::ClosedError &) { + record_connection_failure_(permit); + throw; + } catch (const sw::redis::ReplyError &) { + record_success_(permit); + throw; + } catch (const sw::redis::Error &error) { + if (is_redis_pool_wait_error(error)) record_connection_failure_(permit); + else abandon_(permit); + throw; + } catch (...) { + abandon_(permit); + throw; + } + } + + template + void mget(Input first, Input last, Output out) { + guarded_void_([&] { + _rc ? _rc->mget(first, last, out) : _r->mget(first, last, out); + }); } // --- SCAN --- @@ -166,27 +364,115 @@ class RedisClient // 续扫，因此任意 cursor 都重启一次完整扫描并返回 0。 template long long scan(long long cursor, const std::string &pattern, long long count, Out out) { - if (_rc) { - if (cursor != 0) { - LOG_WARN("RedisCluster scan cannot resume cursor {}; restarting full cluster scan", cursor); - } - _rc->for_each([&](sw::redis::Redis &r) { - long long cur = 0; - while (true) { - cur = r.scan(cur, pattern, count, out); - if (cur == 0) break; + return guarded_([&]() -> long long { + if (_rc) { + if (cursor != 0) { + LOG_WARN("RedisCluster scan cannot resume cursor {}; restarting full cluster scan", cursor); } - }); - return 0; - } - return _r->scan(cursor, pattern, count, out); + _rc->for_each([&](sw::redis::Redis &r) { + long long cur = 0; + while (true) { + cur = r.scan(cur, pattern, count, out); + if (cur == 0) break; + } + }); + return 0; + } + return static_cast(_r->scan(cursor, pattern, count, out)); + }); } bool is_cluster() const { return _rc != nullptr; } private: + RedisCircuitBreaker::Permit before_call_() { + try { + auto permit = _breaker->before_call(); + if (permit.probe) LOG_INFO("Redis circuit transition Open->HalfOpen"); + return permit; + } catch (const RedisCircuitOpen &) { + metrics::g_redis_circuit_rejected_total << 1; + throw; + } + } + + static void record_transition_(RedisCircuitBreaker::Transition transition) noexcept { + if (transition == RedisCircuitBreaker::Transition::Opened) { + metrics::g_redis_circuit_open_total << 1; + LOG_WARN("Redis circuit transition ->Open"); + } else if (transition == RedisCircuitBreaker::Transition::Recovered) { + metrics::g_redis_circuit_recovered_total << 1; + LOG_INFO("Redis circuit transition HalfOpen->Closed"); + } + } + + void record_success_(RedisCircuitBreaker::Permit permit) noexcept { + record_transition_(_breaker->on_success(permit)); + } + + void abandon_(RedisCircuitBreaker::Permit permit) noexcept { + record_transition_(_breaker->on_abandoned(permit)); + } + + void record_connection_failure_(RedisCircuitBreaker::Permit permit) noexcept { + metrics::g_redis_call_failure_total << 1; + record_transition_(_breaker->on_connection_failure(permit)); + } + + template + auto guarded_(F &&fn) -> std::invoke_result_t { + auto permit = before_call_(); + try { + decltype(auto) result = std::forward(fn)(); + record_success_(permit); + return std::forward(result); + } catch (const sw::redis::IoError &) { + record_connection_failure_(permit); + throw; + } catch (const sw::redis::ClosedError &) { + record_connection_failure_(permit); + throw; + } catch (const sw::redis::ReplyError &) { + record_success_(permit); + throw; + } catch (const sw::redis::Error &error) { + if (is_redis_pool_wait_error(error)) record_connection_failure_(permit); + else abandon_(permit); + throw; + } catch (...) { + abandon_(permit); + throw; + } + } + + template + void guarded_void_(F &&fn) { + auto permit = before_call_(); + try { + std::forward(fn)(); + record_success_(permit); + } catch (const sw::redis::IoError &) { + record_connection_failure_(permit); + throw; + } catch (const sw::redis::ClosedError &) { + record_connection_failure_(permit); + throw; + } catch (const sw::redis::ReplyError &) { + record_success_(permit); + throw; + } catch (const sw::redis::Error &error) { + if (is_redis_pool_wait_error(error)) record_connection_failure_(permit); + else abandon_(permit); + throw; + } catch (...) { + abandon_(permit); + throw; + } + } + std::shared_ptr _r; std::shared_ptr _rc; + std::shared_ptr _breaker; }; /* brief: 默认 TTL 常量 */ @@ -196,6 +482,7 @@ inline constexpr std::chrono::seconds kCodeTtl(60 * 5); // 验证码 inline constexpr std::chrono::seconds kLastMsgTtl(24 * 3600); // 最近消息预览 24 小时 inline constexpr std::chrono::seconds kReadAckTtl(24 * 3600); // 已读暂存 24 小时 inline constexpr std::chrono::seconds kMembersTtl(30 * 60); // 成员缓存 30 分钟 +inline constexpr std::chrono::seconds kUserInfoTtl(3600); // 用户资料缓存 1 小时 inline constexpr std::chrono::seconds kOnlineTtl(30); // 在线路由 30s（依赖心跳续期，每 heartbeat 刷新） inline constexpr std::chrono::seconds kUnackedTtl(7 * 24 * 3600); // 未 ack 重传缓冲 7 天 @@ -215,12 +502,12 @@ class RedisClientFactory copts.port = port; copts.db = db; copts.keep_alive = keep_alive; - copts.connect_timeout = std::chrono::milliseconds(2000); - copts.socket_timeout = std::chrono::milliseconds(2000); + copts.connect_timeout = std::chrono::milliseconds(50); + copts.socket_timeout = std::chrono::milliseconds(50); sw::redis::ConnectionPoolOptions popts; popts.size = pool_size; - popts.wait_timeout = std::chrono::milliseconds(500); + popts.wait_timeout = std::chrono::milliseconds(20); popts.connection_lifetime = std::chrono::minutes(30); return std::make_shared(copts, popts); @@ -255,7 +542,7 @@ class RedisClusterFactory sw::redis::ConnectionPoolOptions popts; popts.size = pool_size; - popts.wait_timeout = std::chrono::milliseconds(500); + popts.wait_timeout = std::chrono::milliseconds(20); popts.connection_lifetime = std::chrono::minutes(30); // 逐个尝试种子节点，直到成功连接（sw::redis++ RedisCluster 仅需一个种子 @@ -267,8 +554,8 @@ class RedisClusterFactory copts.host = host; copts.port = port; copts.keep_alive = keep_alive; - copts.connect_timeout = std::chrono::milliseconds(2000); - copts.socket_timeout = std::chrono::milliseconds(2000); + copts.connect_timeout = std::chrono::milliseconds(50); + copts.socket_timeout = std::chrono::milliseconds(50); auto cluster = std::make_shared(copts, popts); // 验证连接可用（立即尝试一个轻量命令） @@ -298,7 +585,7 @@ class Session /* brief: 写入登录态，TTL 7 天 */ void append(const std::string &ssid, const std::string &uid, std::chrono::seconds ttl = kSessionTtl) { - try { _c->set(key::kSession + ssid, uid, ttl); } + try { _c->set(key::kSession + ssid, uid, randomized_ttl(ttl)); } catch(std::exception &e) { LOG_ERROR("Session.append 失败 {}: {}", ssid, e.what()); } } void remove(const std::string &ssid) { @@ -311,7 +598,7 @@ class Session } /* brief: 续期（每次心跳调用） */ void touch(const std::string &ssid, std::chrono::seconds ttl = kSessionTtl) { - try { _c->expire(key::kSession + ssid, ttl); } + try { _c->expire(key::kSession + ssid, randomized_ttl(ttl)); } catch(std::exception &e) { LOG_ERROR("Session.touch 失败 {}: {}", ssid, e.what()); } } private: @@ -324,7 +611,7 @@ class Status using ptr = std::shared_ptr; Status(const RedisClient::ptr &c) : _c(c) {} void append(const std::string &uid, std::chrono::seconds ttl = kStatusTtl) { - try { _c->set(key::kStatus + uid, "1", ttl); } + try { _c->set(key::kStatus + uid, "1", randomized_ttl(ttl)); } catch(std::exception &e) { LOG_ERROR("Status.append 失败 {}: {}", uid, e.what()); } } void remove(const std::string &uid) { @@ -337,7 +624,7 @@ class Status } /* brief: 心跳续期 */ void touch(const std::string &uid, std::chrono::seconds ttl = kStatusTtl) { - try { _c->expire(key::kStatus + uid, ttl); } + try { _c->expire(key::kStatus + uid, randomized_ttl(ttl)); } catch(std::exception &e) { LOG_ERROR("Status.touch 失败 {}: {}", uid, e.what()); } } private: @@ -351,7 +638,7 @@ class Codes Codes(const RedisClient::ptr &c) : _c(c) {} void append(const std::string &cid, const std::string &code, std::chrono::seconds ttl = kCodeTtl) { - try { _c->set(key::kVerifyCode + cid, code, ttl); } + try { _c->set(key::kVerifyCode + cid, code, randomized_ttl(ttl)); } catch(std::exception &e) { LOG_ERROR("Codes.append 失败 {}: {}", cid, e.what()); } } void remove(const std::string &cid) { @@ -496,27 +783,74 @@ class DeviceSet /* brief: 用户某设备上线 */ void add(const std::string &uid, const std::string &device_id) { - try { _c->sadd(key::kDeviceSet + uid, device_id); } + try { + const auto effective_ttl = randomized_ttl(kSessionTtl); + std::vector keys = {key::device_set_key(uid)}; + std::vector args = { + device_id, std::to_string(effective_ttl.count())}; + _c->eval(kAddLua, keys.begin(), keys.end(), + args.begin(), args.end()); + } catch(std::exception &e) { LOG_ERROR("DeviceSet.add 失败 {}-{}: {}", uid, device_id, e.what()); } } + /* brief: 用户设备活动时续期 */ + void touch(const std::string &uid, std::chrono::seconds ttl = kSessionTtl) { + try { _c->expire(key::device_set_key(uid), randomized_ttl(ttl)); } + catch(std::exception &e) { LOG_ERROR("DeviceSet.touch 失败 {}: {}", uid, e.what()); } + // Rolling-upgrade compatibility only: legacy keys get a bounded grace TTL. + try { _c->expire(key::legacy_device_set_key(uid), kLegacyGraceTtl); } + catch(std::exception &) {} + } /* brief: 用户某设备下线 */ void remove(const std::string &uid, const std::string &device_id) { - try { _c->srem(key::kDeviceSet + uid, device_id); } + try { _c->srem(key::device_set_key(uid), device_id); } catch(std::exception &e) { LOG_ERROR("DeviceSet.rem 失败 {}-{}: {}", uid, device_id, e.what()); } + try { _c->srem(key::legacy_device_set_key(uid), device_id); } + catch(std::exception &) {} } /* brief: 取用户当前所有在线设备 */ std::vector list(const std::string &uid) { std::vector res; - try { _c->smembers(key::kDeviceSet + uid, std::inserter(res, res.end())); } + try { _c->smembers(key::device_set_key(uid), std::back_inserter(res)); } catch(std::exception &e) { LOG_ERROR("DeviceSet.list 失败 {}: {}", uid, e.what()); } + std::vector legacy; + try { _c->smembers(key::legacy_device_set_key(uid), std::back_inserter(legacy)); } + catch(std::exception &) { return res; } + if (!legacy.empty()) { + std::unordered_set merged(res.begin(), res.end()); + merged.insert(legacy.begin(), legacy.end()); + res.assign(merged.begin(), merged.end()); + // Cross-slot migration is intentionally best-effort. New writes use only + // the tagged key; the old key expires after the rolling-upgrade window. + try { + _c->sadd(key::device_set_key(uid), legacy.begin(), legacy.end()); + _c->expire(key::device_set_key(uid), randomized_ttl(kSessionTtl)); + _c->expire(key::legacy_device_set_key(uid), kLegacyGraceTtl); + } catch (std::exception &) {} + } return res; } /* brief: 用户是否有任意在线设备 */ bool any(const std::string &uid) { - try { return _c->scard(key::kDeviceSet + uid) > 0; } - catch(std::exception &e) { LOG_ERROR("DeviceSet.any 失败 {}: {}", uid, e.what()); return false; } + return !list(uid).empty(); } private: + static constexpr std::chrono::seconds kLegacyGraceTtl{24 * 3600}; + static constexpr const char *kAddLua = R"lua( +local function key_type(k) + local t = redis.call('TYPE', k) + if type(t) == 'table' then return t.ok end + return t +end +local ttl = tonumber(ARGV[2]) +if not ttl or ttl <= 0 then return redis.error_reply('invalid ttl') end +local t = key_type(KEYS[1]) +if t ~= 'none' and t ~= 'set' then return redis.error_reply('device key wrong type') end +redis.call('SADD', KEYS[1], ARGV[1]) +redis.call('EXPIRE', KEYS[1], ttl) +return 1 +)lua"; + RedisClient::ptr _c; }; @@ -792,6 +1126,290 @@ class Members RedisClient::ptr _c; }; +// ============================================================================= +// 用户资料缓存（value 为序列化 UserInfo protobuf；DAO 不依赖 protobuf 类型） +// ============================================================================= + +enum class GenerationWriteResult { + Committed, + Conflict, + Unavailable, +}; + +enum class UserInfoL1Publication { + GenerationFenced, + Denied, + ShortLivedFallback, +}; + +constexpr UserInfoL1Publication +user_info_l1_publication(GenerationWriteResult result) noexcept { + switch (result) { + case GenerationWriteResult::Committed: + return UserInfoL1Publication::GenerationFenced; + case GenerationWriteResult::Conflict: + return UserInfoL1Publication::Denied; + case GenerationWriteResult::Unavailable: + return UserInfoL1Publication::ShortLivedFallback; + } + return UserInfoL1Publication::Denied; +} + +template +GenerationWriteResult execute_generation_write(bool redis_available, + Eval &&eval) noexcept { + if (!redis_available) return GenerationWriteResult::Unavailable; + try { + return std::forward(eval)() == 0 + ? GenerationWriteResult::Conflict + : GenerationWriteResult::Committed; + } catch (const RedisCircuitOpen &) { + return GenerationWriteResult::Unavailable; + } catch (const std::exception &) { + return GenerationWriteResult::Unavailable; + } +} + +template +void publish_user_info_l1(GenerationWriteResult result, Publish &&publish) { + const auto publication = user_info_l1_publication(result); + if (publication == UserInfoL1Publication::Denied) return; + std::forward(publish)(publication); +} + +class UserInfoCache +{ +public: + using ptr = std::shared_ptr; + using GenerationEval = std::function; + + struct BatchResult { + std::unordered_map hits; + struct Miss { + std::string uid; + std::optional observed_generation; + }; + std::vector misses; + }; + struct FencedValue { + std::string serialized; + uint64_t observed_generation; + }; + + explicit UserInfoCache(const RedisClient::ptr &c, + GenerationEval generation_eval = {}) + : _c(c), _generation_eval(std::move(generation_eval)) {} + + std::optional get(const std::string &uid) { + if (!_c) return std::nullopt; + try { + auto value = _c->get(key::user_info_key(uid)); + if (!value) return std::nullopt; + return *value; + } catch (const RedisCircuitOpen &) { + return std::nullopt; + } catch (const std::exception &) { + return std::nullopt; + } + } + + BatchResult batch_get(const std::vector &uids) { + BatchResult result; + const size_t count = std::min(uids.size(), 2000); + if (!_c) { + for (size_t i = 0; i < count; ++i) result.misses.push_back({uids[i], std::nullopt}); + return result; + } + std::unordered_map>> groups; + for (size_t i = 0; i < count; ++i) { + groups[key::user_info_bucket(uids[i])].push_back( + {uids[i], key::user_info_key(uids[i])}); + } + for (const auto &group : groups) { + const auto &entries = group.second; + std::vector keys; + keys.reserve(entries.size()); + for (const auto &entry : entries) { + keys.push_back(entry.second); + keys.push_back(key::user_info_generation_key(entry.first)); + } + try { + std::vector values; + values.reserve(keys.size()); + _c->mget(keys.begin(), keys.end(), std::back_inserter(values)); + for (size_t i = 0; i < entries.size(); ++i) { + const size_t value_index = i * 2; + if (value_index < values.size() && values[value_index]) { + result.hits[entries[i].first] = *values[value_index]; + } else { + std::optional observed; + if (value_index + 1 < values.size()) { + try { + observed = values[value_index + 1] + ? std::stoull(*values[value_index + 1]) : uint64_t{0}; + } catch (const std::exception &) {} + } + result.misses.push_back({entries[i].first, observed}); + } + } + } catch (const RedisCircuitOpen &) { + for (const auto &entry : entries) result.misses.push_back({entry.first, std::nullopt}); + } catch (const std::exception &) { + for (const auto &entry : entries) result.misses.push_back({entry.first, std::nullopt}); + } + } + return result; + } + + bool set(const std::string &uid, const std::string &serialized, + uint64_t observed_generation) { + return set_if_generation(uid, serialized, observed_generation) == + GenerationWriteResult::Committed; + } + + std::optional generation(const std::string &uid) { + if (!_c) return uint64_t{0}; + try { + auto value = _c->get(key::user_info_generation_key(uid)); + return value ? std::stoull(*value) : uint64_t{0}; + } catch (const RedisCircuitOpen &) { + return std::nullopt; + } catch (const std::exception &) { + return std::nullopt; + } + } + + GenerationWriteResult set_if_generation(const std::string &uid, + const std::string &serialized, + uint64_t expected_generation) { + return execute_generation_write( + static_cast(_c) || static_cast(_generation_eval), [&] { + const auto ttl = serialized.empty() + ? randomized_ttl(std::chrono::seconds(5)) + : randomized_ttl(kUserInfoTtl); + std::vector keys = { + key::user_info_key(uid), key::user_info_generation_key(uid)}; + std::vector args = { + std::to_string(expected_generation), serialized, + std::to_string(ttl.count())}; + if (_generation_eval) return _generation_eval(); + return _c->eval(kSetIfGenerationLua, keys.begin(), keys.end(), + args.begin(), args.end()); + }); + } + + size_t batch_set(const std::unordered_map &values) { + if (!_c) return 0; + std::unordered_map>> groups; + size_t count = 0; + for (const auto &entry : values) { + if (count++ >= 2000) break; + groups[key::user_info_bucket(entry.first)].push_back(entry); + } + size_t written = 0; + for (const auto &group : groups) { + std::vector keys; + std::vector args; + keys.reserve(group.second.size() * 2); + args.reserve(group.second.size() * 3); + for (const auto &[uid, value] : group.second) { + keys.push_back(key::user_info_key(uid)); + keys.push_back(key::user_info_generation_key(uid)); + args.push_back(std::to_string(value.observed_generation)); + args.push_back(value.serialized); + const auto ttl = value.serialized.empty() + ? randomized_ttl(std::chrono::seconds(5)) + : randomized_ttl(kUserInfoTtl); + args.push_back(std::to_string(ttl.count())); + } + try { + written += static_cast(_c->eval( + kBatchSetIfGenerationLua, keys.begin(), keys.end(), + args.begin(), args.end())); + } catch (const RedisCircuitOpen &) { + } catch (const std::exception &) {} + } + return written; + } + + bool invalidate(const std::string &uid) noexcept { + if (!_c) return true; + try { + std::vector keys = { + key::user_info_key(uid), key::user_info_generation_key(uid)}; + std::vector args; + _c->eval(kInvalidateLua, keys.begin(), keys.end(), + args.begin(), args.end()); + return true; + } catch (const RedisCircuitOpen &) { + metrics::g_user_info_invalidation_failure_total << 1; + return false; + } catch (const std::exception &) { + metrics::g_user_info_invalidation_failure_total << 1; + return false; + } + } + +private: + static constexpr const char *kSetIfGenerationLua = R"lua( +local generation = tonumber(redis.call('GET', KEYS[2]) or '0') +if generation ~= tonumber(ARGV[1]) then return 0 end +redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3]) +redis.call('EXPIRE', KEYS[2], 604800) +return 1 +)lua"; + static constexpr const char *kInvalidateLua = R"lua( +redis.call('INCR', KEYS[2]) +redis.call('EXPIRE', KEYS[2], 604800) +redis.call('DEL', KEYS[1]) +return 1 +)lua"; + // Redis scripts are isolated but do not roll back writes after a runtime + // error. Validate every entry before mutating so deterministic late errors + // cannot partially fill a bucket. Server OOM during SET is a Redis-level + // limitation and is surfaced as a failed best-effort cache fill. + static constexpr const char *kBatchSetIfGenerationLua = R"lua( +if (#KEYS % 2) ~= 0 or #ARGV ~= (#KEYS / 2) * 3 then + return redis.error_reply('invalid batch cache arguments') +end +local function key_type(key) + local reply = redis.call('TYPE', key) + if type(reply) == 'table' then return reply.ok end + return reply +end +for i = 1, #KEYS, 2 do + local arg = ((i - 1) / 2) * 3 + 1 + local value_type = key_type(KEYS[i]) + local generation_type = key_type(KEYS[i + 1]) + local expected = tonumber(ARGV[arg]) + local ttl = tonumber(ARGV[arg + 2]) + local generation = nil + if generation_type == 'none' then generation = 0 end + if generation_type == 'string' then generation = tonumber(redis.call('GET', KEYS[i + 1])) end + if (value_type ~= 'none' and value_type ~= 'string') or + (generation_type ~= 'none' and generation_type ~= 'string') or + generation == nil or expected == nil or expected < 0 or + expected ~= math.floor(expected) or ARGV[arg + 1] == nil or ttl == nil or + ttl <= 0 or ttl ~= math.floor(ttl) then + return redis.error_reply('invalid batch cache entry') + end +end +local written = 0 +for i = 1, #KEYS, 2 do + local arg = ((i - 1) / 2) * 3 + 1 + local generation = tonumber(redis.call('GET', KEYS[i + 1]) or '0') + if generation == tonumber(ARGV[arg]) then + redis.call('SET', KEYS[i], ARGV[arg + 1], 'EX', ARGV[arg + 2]) + redis.call('EXPIRE', KEYS[i + 1], 604800) + written = written + 1 + end +end +return written +)lua"; + RedisClient::ptr _c; + GenerationEval _generation_eval; +}; + // ============================================================================= // 在线路由表（多 Push 实例下：uid -> 持有 ws 连接的 push_instance_id 集合） // ============================================================================= @@ -807,23 +1425,63 @@ class OnlineRoute const std::string &push_instance, std::chrono::seconds ttl = kOnlineTtl) { try { - std::string k = key::online_key(uid); - _c->hset(k, device_id, push_instance); - _c->expire(k, randomized_ttl(ttl)); + const auto route_ttl = randomized_ttl(ttl); + const auto device_ttl = randomized_ttl(kSessionTtl); + std::vector keys = { + key::online_key(uid), key::device_set_key(uid)}; + std::vector args = { + device_id, push_instance, std::to_string(route_ttl.count()), + std::to_string(device_ttl.count())}; + _c->eval(kBindLua, keys.begin(), keys.end(), + args.begin(), args.end()); } catch(std::exception &e) { LOG_ERROR("OnlineRoute.bind 失败 {}-{}-{}: {}", uid, device_id, push_instance, e.what()); } } /* brief: 心跳续期（续整个 uid 的 HASH） */ void touch(const std::string &uid, std::chrono::seconds ttl = kOnlineTtl) { - try { _c->expire(key::online_key(uid), randomized_ttl(ttl)); } + try { + const auto route_ttl = randomized_ttl(ttl); + const auto device_ttl = randomized_ttl(kSessionTtl); + std::vector keys = { + key::online_key(uid), key::device_set_key(uid)}; + std::vector args = { + std::to_string(route_ttl.count()), + std::to_string(device_ttl.count())}; + _c->eval(kTouchLua, keys.begin(), keys.end(), + args.begin(), args.end()); + } catch(std::exception &e) { LOG_ERROR("OnlineRoute.touch 失败 {}: {}", uid, e.what()); } + try { _c->expire(key::legacy_device_set_key(uid), kLegacyDeviceGraceTtl); } + catch (const std::exception &) {} } /* brief: 设备下线 — HDEL uid did */ void unbind(const std::string &uid, const std::string &device_id, const std::string &push_instance) { - try { _c->hdel(key::online_key(uid), device_id); } + std::vector legacy_devices; + if (device_id.empty()) { + try { + std::unordered_map routes; + _c->hgetall(key::online_key(uid), std::inserter(routes, routes.end())); + for (const auto &[did, instance] : routes) { + if (instance == push_instance) legacy_devices.push_back(did); + } + } catch (const std::exception &) {} + } else { + legacy_devices.push_back(device_id); + } + try { + std::vector keys = { + key::online_key(uid), key::device_set_key(uid)}; + std::vector args = {device_id, push_instance}; + _c->eval(kUnbindLua, keys.begin(), keys.end(), + args.begin(), args.end()); + } catch(std::exception &e) { LOG_ERROR("OnlineRoute.unbind 失败 {}-{}-{}: {}", uid, device_id, push_instance, e.what()); } + for (const auto &did : legacy_devices) { + try { _c->srem(key::legacy_device_set_key(uid), did); } + catch (const std::exception &) {} + } } /* brief: 取用户所有在线设备 → device_id 列表 */ std::vector devices(const std::string &uid) { @@ -843,6 +1501,14 @@ class OnlineRoute } return res; } + // Push delivery is a truth-source path: callers must distinguish an empty + // route from an unavailable Redis route table. + std::unordered_map device_instances_map_strict( + const std::string &uid) { + std::unordered_map res; + _c->hgetall(key::online_key(uid), std::inserter(res, res.end())); + return res; + } /* brief: 取设备所在 Push 实例 */ std::string device_instance(const std::string &uid, const std::string &device_id) { try { @@ -859,6 +1525,68 @@ class OnlineRoute catch(std::exception &e) { LOG_ERROR("OnlineRoute.online 失败 {}: {}", uid, e.what()); return false; } } private: + static constexpr std::chrono::seconds kLegacyDeviceGraceTtl{24 * 3600}; + static constexpr const char *kBindLua = R"lua( +local function key_type(k) + local t = redis.call('TYPE', k) + if type(t) == 'table' then return t.ok end + return t +end +local route_ttl = tonumber(ARGV[3]) +local device_ttl = tonumber(ARGV[4]) +if not route_ttl or route_ttl <= 0 or not device_ttl or device_ttl <= 0 then + return redis.error_reply('invalid ttl') +end +local rt = key_type(KEYS[1]) +local dt = key_type(KEYS[2]) +if rt ~= 'none' and rt ~= 'hash' then return redis.error_reply('route key wrong type') end +if dt ~= 'none' and dt ~= 'set' then return redis.error_reply('device key wrong type') end +redis.call('HSET', KEYS[1], ARGV[1], ARGV[2]) +redis.call('SADD', KEYS[2], ARGV[1]) +redis.call('EXPIRE', KEYS[1], route_ttl) +redis.call('EXPIRE', KEYS[2], device_ttl) +return 1 +)lua"; + + static constexpr const char *kTouchLua = R"lua( +local route_ttl = tonumber(ARGV[1]) +local device_ttl = tonumber(ARGV[2]) +if not route_ttl or route_ttl <= 0 or not device_ttl or device_ttl <= 0 then + return redis.error_reply('invalid ttl') +end +redis.call('EXPIRE', KEYS[1], route_ttl) +redis.call('EXPIRE', KEYS[2], device_ttl) +return 1 +)lua"; + + static constexpr const char *kUnbindLua = R"lua( +local function key_type(k) + local t = redis.call('TYPE', k) + if type(t) == 'table' then return t.ok end + return t +end +local rt = key_type(KEYS[1]) +local dt = key_type(KEYS[2]) +if rt ~= 'none' and rt ~= 'hash' then return redis.error_reply('route key wrong type') end +if dt ~= 'none' and dt ~= 'set' then return redis.error_reply('device key wrong type') end +local removed = 0 +if ARGV[1] == '' then + local entries = redis.call('HGETALL', KEYS[1]) + for i = 1, #entries, 2 do + if entries[i + 1] == ARGV[2] then + redis.call('HDEL', KEYS[1], entries[i]) + redis.call('SREM', KEYS[2], entries[i]) + removed = removed + 1 + end + end +elseif redis.call('HGET', KEYS[1], ARGV[1]) == ARGV[2] then + redis.call('HDEL', KEYS[1], ARGV[1]) + redis.call('SREM', KEYS[2], ARGV[1]) + removed = 1 +end +return removed +)lua"; + RedisClient::ptr _c; }; @@ -878,21 +1606,25 @@ class RateLimiter * - 命中限制返回 false（业务可返回 429 / RATE_LIMITED） */ bool allow(const std::string &key_full, int max_count, int window_sec) { + std::vector keys = {key_full}; + auto now_ms = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + std::vector args = { + std::to_string(max_count), + std::to_string(window_sec), + std::to_string(now_ms) + }; try { - std::vector keys = {key_full}; - auto now_ms = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - std::vector args = { - std::to_string(max_count), - std::to_string(window_sec), - std::to_string(now_ms) - }; long long cur = _c->eval(kRateLimitScript, keys.begin(), keys.end(), args.begin(), args.end()); return cur == 1; - } catch(std::exception &e) { - LOG_ERROR("RateLimiter.allow {}: {}", key_full, e.what()); - return true; + } catch (const RedisCircuitOpen &) { + return allow_local_(key_full, max_count, window_sec); + } catch (const sw::redis::Error &e) { + if (should_log_redis_error_()) { + LOG_WARN("RateLimiter Redis unavailable; using local fallback: {}", e.what()); + } + return allow_local_(key_full, max_count, window_sec); } } bool allow_user(const std::string &uid, int max_count, int window_sec) { @@ -902,7 +1634,23 @@ class RateLimiter return allow(key::kRateSsid + ssid, max_count, window_sec); } private: + bool should_log_redis_error_() noexcept { + const auto now = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); + auto previous = _last_redis_error_log_sec.load(std::memory_order_relaxed); + return now > previous && _last_redis_error_log_sec.compare_exchange_strong( + previous, now, std::memory_order_relaxed, std::memory_order_relaxed); + } + bool allow_local_(const std::string &key_full, int max_count, int window_sec) { + metrics::g_rate_limit_local_fallback_total << 1; + const bool allowed = _local.allow(key_full, max_count, window_sec); + if (!allowed) metrics::g_rate_limit_local_rejected_total << 1; + return allowed; + } + RedisClient::ptr _c; + LocalRateLimiter _local; + std::atomic _last_redis_error_log_sec{0}; static const std::string kRateLimitScript; }; @@ -1070,22 +1818,26 @@ class UnackedPush static std::string idx_key_for(const std::string &uid, const std::string &device_id) { return std::string(key::kUnacked) + "idx:{" + uid + ":" + device_id + "}"; } + // HSCAN continuation for bounded HASH-only orphan repair. HASH mutations + // (push/ack) delete it; score-only bump preserves and renews it. + static std::string repair_key_for(const std::string &uid, const std::string &device_id) { + return std::string(key::kUnacked) + "repair:{" + uid + ":" + device_id + "}"; + } /* brief: 入待重传队列（per-device，存 payload_b64 直接用） */ void push(const std::string &uid, const std::string &device_id, unsigned long user_seq, const std::string &payload_b64, long long score_ts, std::chrono::seconds ttl = kUnackedTtl) { - try { - std::string k = key_for(uid, device_id); - std::string ik = idx_key_for(uid, device_id); - std::string member = std::to_string(user_seq) + ":" + payload_b64; - _c->zadd(k, member, static_cast(score_ts)); - _c->hset(ik, std::to_string(user_seq), payload_b64); - _c->expire(k, ttl); - _c->expire(ik, ttl); - } catch(std::exception &e) { - LOG_ERROR("UnackedPush.push 失败 {}-{}-{}: {}", uid, device_id, user_seq, e.what()); - } + std::string k = key_for(uid, device_id); + std::string ik = idx_key_for(uid, device_id); + std::string rk = repair_key_for(uid, device_id); + const auto effective_ttl = randomized_ttl(ttl); + std::vector keys = {k, ik, rk}; + std::vector args = { + std::to_string(score_ts), std::to_string(user_seq), payload_b64, + std::to_string(effective_ttl.count())}; + _c->eval(kPushLua, keys.begin(), keys.end(), + args.begin(), args.end()); } /* brief: 客户端 ACK 后移除（per-device，O(1) via HASH index） */ void ack(const std::string &uid, const std::string &device_id, @@ -1093,12 +1845,11 @@ class UnackedPush try { std::string k = key_for(uid, device_id); std::string ik = idx_key_for(uid, device_id); - auto payload = _c->hget(ik, std::to_string(user_seq)); - if (payload) { - std::string member = std::to_string(user_seq) + ":" + *payload; - _c->zrem(k, member); - _c->hdel(ik, std::to_string(user_seq)); - } + std::string rk = repair_key_for(uid, device_id); + std::vector keys = {k, ik, rk}; + std::vector args = {std::to_string(user_seq)}; + _c->eval(kAckLua, keys.begin(), keys.end(), + args.begin(), args.end()); } catch(std::exception &e) { LOG_ERROR("UnackedPush.ack 失败 {}-{}-{}: {}", uid, device_id, user_seq, e.what()); } @@ -1111,19 +1862,17 @@ class UnackedPush if(limit <= 0) return res; try { std::string k = key_for(uid, device_id); + std::string ik = idx_key_for(uid, device_id); + std::string rk = repair_key_for(uid, device_id); long long now = static_cast(time(nullptr)); - using namespace sw::redis; std::vector raw; - _c->zrangebyscore(k, - BoundedInterval(0, static_cast(now - max_age_sec), - BoundType::CLOSED), - LimitOptions{0, limit}, - std::back_inserter(raw)); - for (const auto &s : raw) { - auto pos = s.find(':'); - if (pos == std::string::npos) continue; - unsigned long seq = std::stoull(s.substr(0, pos)); - res.emplace_back(seq, s.substr(pos + 1)); + std::vector keys = {k, ik, rk}; + std::vector args = { + std::to_string(now - max_age_sec), std::to_string(limit)}; + _c->eval(kPeekDueLua, keys.begin(), keys.end(), + args.begin(), args.end(), std::back_inserter(raw)); + for (size_t i = 0; i + 1 < raw.size(); i += 2) { + res.emplace_back(std::stoull(raw[i]), std::move(raw[i + 1])); } } catch(std::exception &e) { LOG_ERROR("UnackedPush.peek_due 失败 {}-{}-{}: {}", uid, device_id, e.what()); @@ -1138,22 +1887,160 @@ class UnackedPush try { std::string k = key_for(uid, device_id); std::string ik = idx_key_for(uid, device_id); + std::string rk = repair_key_for(uid, device_id); long long now = static_cast(time(nullptr)); + const auto effective_ttl = randomized_ttl(ttl); + std::vector keys = {k, ik, rk}; + std::vector args = { + std::to_string(effective_ttl.count()), std::to_string(now)}; + args.reserve(2 + user_seqs.size()); for (unsigned long seq : user_seqs) { - auto payload = _c->hget(ik, std::to_string(seq)); - if (payload) { - std::string member = std::to_string(seq) + ":" + *payload; - _c->zadd(k, member, static_cast(now), sw::redis::UpdateType::EXIST); - } + args.push_back(std::to_string(seq)); } - _c->expire(k, ttl); - _c->expire(ik, ttl); + _c->eval(kBumpScoreLua, keys.begin(), keys.end(), + args.begin(), args.end()); } catch(std::exception &e) { LOG_ERROR("UnackedPush.bump_score 失败 {}-{}-{}: {}", uid, device_id, e.what()); } } private: + static constexpr const char *kPushLua = R"lua( +local function key_type(k) + local t = redis.call('TYPE', k) + if type(t) == 'table' then return t.ok end + return t +end +local score = tonumber(ARGV[1]) +local ttl = tonumber(ARGV[4]) +if not score or not ttl or ttl <= 0 then return redis.error_reply('invalid arguments') end +local zt = key_type(KEYS[1]) +local ht = key_type(KEYS[2]) +if zt ~= 'none' and zt ~= 'zset' then return redis.error_reply('unacked key wrong type') end +if ht ~= 'none' and ht ~= 'hash' then return redis.error_reply('unacked index wrong type') end +redis.call('ZADD', KEYS[1], score, ARGV[2]) +redis.call('HSET', KEYS[2], ARGV[2], ARGV[3]) +redis.call('EXPIRE', KEYS[1], ttl) +redis.call('EXPIRE', KEYS[2], ttl) +redis.call('DEL', KEYS[3]) +return 1 +)lua"; + + static constexpr const char *kPeekDueLua = R"lua( +local function key_type(k) + local t = redis.call('TYPE', k) + if type(t) == 'table' then return t.ok end + return t +end +local cutoff = tonumber(ARGV[1]) +local limit = tonumber(ARGV[2]) +if not cutoff or not limit or limit <= 0 then return redis.error_reply('invalid arguments') end +local zt = key_type(KEYS[1]) +local ht = key_type(KEYS[2]) +local rt = key_type(KEYS[3]) +if zt ~= 'none' and zt ~= 'zset' then return redis.error_reply('unacked key wrong type') end +if ht ~= 'none' and ht ~= 'hash' then return redis.error_reply('unacked index wrong type') end +local cursor = '0' +if rt == 'string' then + cursor = redis.call('GET', KEYS[3]) or '0' + if not string.match(cursor, '^%d+$') then + redis.call('DEL', KEYS[3]) + cursor = '0' + end +elseif rt ~= 'none' then + redis.call('DEL', KEYS[3]) +end +local scan = redis.pcall('HSCAN', KEYS[2], cursor, 'COUNT', limit) +if type(scan) == 'table' and scan.err then + redis.call('DEL', KEYS[3]) + scan = redis.call('HSCAN', KEYS[2], 0, 'COUNT', limit) +end +local result = {} +local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', cutoff, 'LIMIT', 0, limit) +for _, seq in ipairs(due) do + local payload = redis.call('HGET', KEYS[2], seq) + if payload then + table.insert(result, seq) + table.insert(result, payload) + else + redis.call('ZREM', KEYS[1], seq) + end +end +local fields = scan[2] +for i = 1, #fields, 2 do + local seq = fields[i] + if not redis.call('ZSCORE', KEYS[1], seq) then + redis.call('HDEL', KEYS[2], seq) + end +end +local next_cursor = scan[1] +if next_cursor == '0' then + redis.call('DEL', KEYS[3]) +else + -- The cursor describes the HASH iteration, so it may never outlive that HASH. + local httl = redis.call('PTTL', KEYS[2]) + if httl > 0 then + redis.call('SET', KEYS[3], next_cursor, 'PX', httl) + elseif httl == -1 then + -- Corrupt persistent ledgers still make progress without leaking metadata forever. + redis.call('SET', KEYS[3], next_cursor, 'PX', 300000) + else + redis.call('DEL', KEYS[3]) + end +end +return result +)lua"; + + static constexpr const char *kBumpScoreLua = R"lua( +local function key_type(k) + local t = redis.call('TYPE', k) + if type(t) == 'table' then return t.ok end + return t +end +local ttl = tonumber(ARGV[1]) +local score = tonumber(ARGV[2]) +if not ttl or ttl <= 0 or not score then return redis.error_reply('invalid arguments') end +local zt = key_type(KEYS[1]) +local ht = key_type(KEYS[2]) +if zt ~= 'none' and zt ~= 'zset' then return redis.error_reply('unacked key wrong type') end +if ht ~= 'none' and ht ~= 'hash' then return redis.error_reply('unacked index wrong type') end +local updated = 0 +for i = 3, #ARGV do + local payload = redis.call('HGET', KEYS[2], ARGV[i]) + if payload then + if redis.call('ZSCORE', KEYS[1], ARGV[i]) then + redis.call('ZADD', KEYS[1], 'XX', score, ARGV[i]) + updated = updated + 1 + end + else + redis.call('ZREM', KEYS[1], ARGV[i]) + end +end +redis.call('EXPIRE', KEYS[1], ttl) +redis.call('EXPIRE', KEYS[2], ttl) +if redis.call('EXISTS', KEYS[3]) == 1 then + redis.call('EXPIRE', KEYS[3], ttl) +end +return updated +)lua"; + + static constexpr const char *kAckLua = R"lua( +local function key_type(k) + local t = redis.call('TYPE', k) + if type(t) == 'table' then return t.ok end + return t +end +local zt = key_type(KEYS[1]) +local ht = key_type(KEYS[2]) +if zt ~= 'none' and zt ~= 'zset' then return redis.error_reply('unacked key wrong type') end +if ht ~= 'none' and ht ~= 'hash' then return redis.error_reply('unacked index wrong type') end +local removed = redis.call('ZREM', KEYS[1], ARGV[1]) +removed = removed + redis.call('HDEL', KEYS[2], ARGV[1]) +-- HASH mutation invalidates an in-progress HSCAN continuation. +redis.call('DEL', KEYS[3]) +return removed +)lua"; + RedisClient::ptr _c; }; @@ -1201,7 +2088,7 @@ class PresenceRedis try { auto k = key::presence_device_key(uid, device_id); _r->hset(k, "state", "ONLINE"); - _r->expire(k, std::chrono::seconds(120)); + _r->expire(k, randomized_ttl(std::chrono::seconds(120))); } catch(std::exception &e) { LOG_ERROR("PresenceRedis.add_device 失败 {}-{}: {}", uid, device_id, e.what()); } @@ -1233,7 +2120,7 @@ class PresenceRedis try { auto k = key::kPresenceTyping + uid; _r->sadd(k, conv_id); - _r->expire(k, std::chrono::seconds(10)); + _r->expire(k, randomized_ttl(std::chrono::seconds(10))); } catch(std::exception &e) { LOG_ERROR("PresenceRedis.set_typing 失败 {}-{}: {}", uid, conv_id, e.what()); } diff --git a/common/infra/metrics.hpp b/common/infra/metrics.hpp index a347f9f..7a56d8b 100644 --- a/common/infra/metrics.hpp +++ b/common/infra/metrics.hpp @@ -19,6 +19,19 @@ inline bvar::Adder g_local_cache_expired_total("local_cache_expired_total" inline bvar::Adder g_members_cache_stale_l1_total("members_cache_stale_l1_total"); inline bvar::Adder g_members_cache_snapshot_race_total("members_cache_snapshot_race_total"); inline bvar::Adder g_members_cache_version_conflict_total("members_cache_version_conflict_total"); +inline bvar::Adder g_user_info_l1_hit_total("user_info_l1_hit_total"); +inline bvar::Adder g_user_info_l2_hit_total("user_info_l2_hit_total"); +inline bvar::Adder g_user_info_rpc_total("user_info_rpc_total"); +inline bvar::Adder g_user_info_invalidation_failure_total("user_info_invalidation_failure_total"); +inline bvar::Adder g_redis_circuit_open_total("redis_circuit_open_total"); +inline bvar::Adder g_redis_circuit_rejected_total("redis_circuit_rejected_total"); +inline bvar::Adder g_redis_circuit_recovered_total("redis_circuit_recovered_total"); +inline bvar::Adder g_redis_call_failure_total("redis_call_failure_total"); +inline bvar::Adder g_redis_mutex_retry_total("redis_mutex_retry_total"); +inline bvar::Adder g_rate_limit_local_fallback_total("rate_limit_local_fallback_total"); +inline bvar::Adder g_rate_limit_local_rejected_total("rate_limit_local_rejected_total"); +inline bvar::Adder g_push_unacked_persist_failure_total("push_unacked_persist_failure_total"); +inline bvar::Adder g_push_message_requeue_total("push_message_requeue_total"); template inline typename ::chatnow::LocalCache::MetricsSink local_cache_metrics_sink() { diff --git a/common/test/CMakeLists.txt b/common/test/CMakeLists.txt index 79e6ba4..e7e2233 100644 --- a/common/test/CMakeLists.txt +++ b/common/test/CMakeLists.txt @@ -88,6 +88,7 @@ if(CHATNOW_HAS_REDIS_PLUS_PLUS) set_property(TARGET test_redis_mutex_contract PROPERTY CXX_STANDARD 17) set_property(TARGET test_redis_mutex_contract PROPERTY CXX_STANDARD_REQUIRED ON) INSTALL(TARGETS test_redis_mutex_contract RUNTIME DESTINATION bin) + endif() # FIXME(3.0): common_tests has gflags link order issue with brpc static lib diff --git a/common/utils/local_rate_limiter.hpp b/common/utils/local_rate_limiter.hpp new file mode 100644 index 0000000..0a0f61b --- /dev/null +++ b/common/utils/local_rate_limiter.hpp @@ -0,0 +1,136 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace chatnow { + +class LocalRateLimiter { +public: + bool allow(const std::string &key, int capacity, int window_sec) { + const auto now_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); + return allow(key, capacity, window_sec, now_ms); + } + + bool allow(const std::string &key, int capacity, int window_sec, int64_t now_ms) { + if (capacity <= 0 || window_sec <= 0) return true; + + const size_t shard_index = std::hash{}(key) % kShardCount; + Shard &shard = _shards[shard_index]; + std::unique_lock lock(shard.mu); + + const uint64_t operation = _operations.fetch_add(1, std::memory_order_relaxed) + 1; + if (operation % 1024 == 0) { + cleanup_(shard, now_ms, window_sec); + } + + auto found = shard.buckets.find(key); + if (found == shard.buckets.end()) { + if (!reserve_bucket_()) { + cleanup_(shard, now_ms, window_sec); + if (!reserve_bucket_()) { + lock.unlock(); + reclaim_idle_bounded_(shard_index, now_ms, window_sec); + lock.lock(); + found = shard.buckets.find(key); + if (found != shard.buckets.end()) return consume_(found->second, capacity, window_sec, now_ms); + if (!reserve_bucket_()) return false; + } + } + try { + found = shard.buckets.emplace( + key, Bucket{static_cast(capacity - 1), now_ms, now_ms}).first; + } catch (...) { + _bucket_count.fetch_sub(1, std::memory_order_relaxed); + throw; + } + return true; + } + + return consume_(found->second, capacity, window_sec, now_ms); + } + +private: + struct Bucket { + double tokens; + int64_t refill_ms; + int64_t seen_ms; + }; + struct Shard { + std::mutex mu; + std::unordered_map buckets; + }; + + bool consume_(Bucket &bucket, int capacity, int window_sec, int64_t now_ms) { + const int64_t elapsed_ms = now_ms > bucket.refill_ms ? now_ms - bucket.refill_ms : 0; + const double refill = static_cast(elapsed_ms) * capacity / + (static_cast(window_sec) * 1000.0); + bucket.tokens = std::min(static_cast(capacity), bucket.tokens + refill); + bucket.refill_ms = now_ms; + bucket.seen_ms = now_ms; + if (bucket.tokens < 1.0) return false; + bucket.tokens -= 1.0; + return true; + } + + static constexpr size_t kShardCount = 64; + static constexpr size_t kMaxBuckets = 65536; + static constexpr size_t kReclaimScan = 8; + + bool reserve_bucket_() { + size_t count = _bucket_count.load(std::memory_order_relaxed); + while (count < kMaxBuckets) { + if (_bucket_count.compare_exchange_weak( + count, count + 1, std::memory_order_relaxed, std::memory_order_relaxed)) { + return true; + } + } + return false; + } + + void cleanup_(Shard &shard, int64_t now_ms, int window_sec) { + const int64_t idle_limit_ms = static_cast(window_sec) * 2000; + size_t removed = 0; + for (auto it = shard.buckets.begin(); it != shard.buckets.end();) { + if (now_ms > it->second.seen_ms && + now_ms - it->second.seen_ms > idle_limit_ms) { + it = shard.buckets.erase(it); + ++removed; + } else { + ++it; + } + } + if (removed != 0) { + _bucket_count.fetch_sub(removed, std::memory_order_relaxed); + } + } + + void reclaim_idle_bounded_(size_t target, int64_t now_ms, int window_sec) { + const size_t start = _reclaim_cursor.fetch_add(kReclaimScan, std::memory_order_relaxed); + for (size_t offset = 0; offset < kReclaimScan; ++offset) { + const size_t index = (start + offset) % kShardCount; + if (index == target) continue; + Shard &candidate = _shards[index]; + std::lock_guard lock(candidate.mu); + cleanup_(candidate, now_ms, window_sec); + if (_bucket_count.load(std::memory_order_relaxed) < kMaxBuckets) return; + } + } + + std::array _shards; + std::atomic _bucket_count{0}; + std::atomic _operations{0}; + std::atomic _reclaim_cursor{0}; +}; + +} // namespace chatnow diff --git a/common/utils/redis_circuit_breaker.hpp b/common/utils/redis_circuit_breaker.hpp new file mode 100644 index 0000000..eeb71fa --- /dev/null +++ b/common/utils/redis_circuit_breaker.hpp @@ -0,0 +1,124 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace chatnow { + +class RedisCircuitOpen final : public std::runtime_error { +public: + RedisCircuitOpen() : std::runtime_error("redis circuit open") {} +}; + +class RedisCircuitBreaker { +public: + enum class State : uint8_t { Closed, Open, HalfOpen }; + enum class Transition : uint8_t { None, Opened, Recovered }; + struct Permit { + uint64_t generation = 0; + bool probe = false; + }; + + Permit before_call(); + Transition on_success(Permit permit) noexcept; + Transition on_connection_failure(Permit permit) noexcept; + Transition on_abandoned(Permit permit) noexcept; + State state() const noexcept { return _state.load(std::memory_order_acquire); } + +private: + static constexpr uint32_t kFailureThreshold = 3; + static constexpr int64_t kOpenForMs = 1000; + static int64_t now_ms() noexcept; + Transition open_locked() noexcept; + + mutable std::mutex _mutex; + std::atomic _state{State::Closed}; + std::atomic _generation{0}; + std::atomic _consecutive_failures{0}; + int64_t _open_until_ms = 0; +}; + +inline int64_t RedisCircuitBreaker::now_ms() noexcept { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +inline RedisCircuitBreaker::Transition RedisCircuitBreaker::open_locked() noexcept { + if (_state.load(std::memory_order_relaxed) == State::Open) { + return Transition::None; + } + _consecutive_failures.store(0, std::memory_order_relaxed); + _open_until_ms = now_ms() + kOpenForMs; + _state.store(State::Open, std::memory_order_release); + _generation.fetch_add(1, std::memory_order_acq_rel); + return Transition::Opened; +} + +inline RedisCircuitBreaker::Permit RedisCircuitBreaker::before_call() { + const auto generation = _generation.load(std::memory_order_acquire); + const auto current = _state.load(std::memory_order_acquire); + if (current == State::Closed) { + return {generation, false}; + } + std::lock_guard lock(_mutex); + const auto locked_current = _state.load(std::memory_order_relaxed); + if (locked_current == State::Closed) { + return {_generation.load(std::memory_order_acquire), false}; + } + if (locked_current == State::HalfOpen || now_ms() < _open_until_ms) { + throw RedisCircuitOpen(); + } + _state.store(State::HalfOpen, std::memory_order_release); + return {_generation.load(std::memory_order_acquire), true}; +} + +inline RedisCircuitBreaker::Transition +RedisCircuitBreaker::on_success(Permit permit) noexcept { + const auto current = _state.load(std::memory_order_acquire); + if (current == State::Closed && !permit.probe) { + if (permit.generation == _generation.load(std::memory_order_acquire)) { + _consecutive_failures.store(0, std::memory_order_relaxed); + } + return Transition::None; + } + std::lock_guard lock(_mutex); + if (permit.generation != _generation.load(std::memory_order_relaxed)) return Transition::None; + if (current == State::HalfOpen && permit.probe) { + _consecutive_failures.store(0, std::memory_order_relaxed); + _state.store(State::Closed, std::memory_order_release); + return Transition::Recovered; + } + return Transition::None; +} + +inline RedisCircuitBreaker::Transition +RedisCircuitBreaker::on_connection_failure(Permit permit) noexcept { + if (permit.generation != _generation.load(std::memory_order_acquire)) return Transition::None; + const auto current = _state.load(std::memory_order_acquire); + if (current == State::Closed && !permit.probe) { + const auto failures = _consecutive_failures.fetch_add(1, std::memory_order_relaxed) + 1; + if (failures < kFailureThreshold) return Transition::None; + } + std::lock_guard lock(_mutex); + if (permit.generation != _generation.load(std::memory_order_relaxed)) return Transition::None; + const auto locked_current = _state.load(std::memory_order_relaxed); + if (locked_current == State::HalfOpen && permit.probe) return open_locked(); + if (locked_current != State::Closed || permit.probe) return Transition::None; + if (_consecutive_failures.load(std::memory_order_relaxed) < kFailureThreshold) return Transition::None; + return open_locked(); +} + +inline RedisCircuitBreaker::Transition +RedisCircuitBreaker::on_abandoned(Permit permit) noexcept { + std::lock_guard lock(_mutex); + if (permit.generation != _generation.load(std::memory_order_relaxed) || !permit.probe || + _state.load(std::memory_order_relaxed) != State::HalfOpen) { + return Transition::None; + } + return open_locked(); +} + +} // namespace chatnow diff --git a/common/utils/redis_keys.hpp b/common/utils/redis_keys.hpp index 7cd4dab..270eec0 100644 --- a/common/utils/redis_keys.hpp +++ b/common/utils/redis_keys.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace chatnow::key { @@ -65,6 +66,29 @@ inline std::string local_user_info_cache_key(const std::string &uid) { return std::string("local:user:") + hash_tag(uid); } +inline uint32_t fnv1a_32(const std::string &value) { + uint32_t hash = 2166136261u; + for (unsigned char c : value) { + hash ^= c; + hash *= 16777619u; + } + return hash; +} + +inline uint32_t user_info_bucket(const std::string &uid) { + return fnv1a_32(uid) % 64u; +} + +inline std::string user_info_key(const std::string &uid) { + auto bucket = std::to_string(user_info_bucket(uid)); + return "im:user:{" + bucket + "}:" + uid; +} + +inline std::string user_info_generation_key(const std::string &uid) { + auto bucket = std::to_string(user_info_bucket(uid)); + return "im:user-gen:{" + bucket + "}:" + uid; +} + inline std::string local_route_cache_key(const std::string &uid) { return std::string("local:route:") + hash_tag(uid); } @@ -73,6 +97,15 @@ inline std::string online_key(const std::string &uid) { return std::string(kOnline) + hash_tag(uid); } +inline std::string device_set_key(const std::string &uid) { + return std::string(kDeviceSet) + hash_tag(uid); +} + +// Pre-3.0 key retained only for bounded lazy migration during rolling upgrades. +inline std::string legacy_device_set_key(const std::string &uid) { + return std::string(kDeviceSet) + uid; +} + inline std::string online_scan_pattern() { return std::string(kOnline) + "*"; } diff --git a/common/utils/redis_mutex.hpp b/common/utils/redis_mutex.hpp index d1931ca..17468a0 100644 --- a/common/utils/redis_mutex.hpp +++ b/common/utils/redis_mutex.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -8,6 +9,7 @@ #include #include "dao/data_redis.hpp" #include "infra/logger.hpp" +#include "infra/metrics.hpp" namespace chatnow { @@ -30,11 +32,19 @@ class RedisMutex { bool try_lock(std::chrono::milliseconds timeout = std::chrono::milliseconds(100)) { auto deadline = std::chrono::steady_clock::now() + timeout; + int backoff_ms = 5; while (std::chrono::steady_clock::now() < deadline) { bool ok = _redis->set(_key, _token, std::chrono::milliseconds(_ttl_ms), sw::redis::UpdateType::NOT_EXIST); if (ok) { _locked = true; return true; } - std::this_thread::sleep_for(std::chrono::milliseconds(5)); + metrics::g_redis_mutex_retry_total << 1; + std::uniform_int_distribution jitter(0, backoff_ms / 2); + auto delay = std::chrono::milliseconds(backoff_ms + jitter(rng_())); + auto remaining = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()); + if (remaining <= std::chrono::milliseconds::zero()) break; + std::this_thread::sleep_for(std::min(delay, remaining)); + backoff_ms = std::min(backoff_ms * 2, 20); } return false; } @@ -58,6 +68,11 @@ class RedisMutex { } private: + static std::mt19937 &rng_() { + static thread_local std::mt19937 rng(std::random_device{}()); + return rng; + } + static std::string generate_token_() { static thread_local std::mt19937_64 rng(std::random_device{}()); std::uniform_int_distribution dist; diff --git a/conf/docker/push_server.conf b/conf/docker/push_server.conf index 920b0ac..f36ba8a 100644 --- a/conf/docker/push_server.conf +++ b/conf/docker/push_server.conf @@ -25,5 +25,7 @@ # M5 心跳触发未 ack 重传 -resend_batch=50 -resend_max_age_sec=5 +# Compose reliability tests pause this sole Push instance during Redis failover. +-route_l1_ttl_sec=30 # JWT — 统一从 auth.json 加载（与 identity/gateway 共享密钥源） -auth_config=/im/conf/auth.json diff --git a/conversation/Dockerfile b/conversation/Dockerfile index fb12a1e..fe7b7ec 100644 --- a/conversation/Dockerfile +++ b/conversation/Dockerfile @@ -1,7 +1,9 @@ -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends WORKDIR /im RUN mkdir -p /im/logs && mkdir -p /im/data && mkdir -p /im/conf && mkdir -p /im/bin COPY ./build/conversation_server /im/bin -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* CMD /im/bin/conversation_server -flagfile=/im/conf/conversation_server.conf diff --git a/docker-compose.yml b/docker-compose.yml index 2462db9..e3b5231 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -257,7 +257,7 @@ services: - rabbitmq - redis-cluster-init entrypoint: - /im/bin/entrypoint.sh -d etcd:2379,mysql:3306,redis-node1:6379,redis-node2:6380,redis-node3:6381,redis-node4:6382,redis-node5:6383,redis-node6:6384,rabbitmq:5672 -c "/im/bin/transmite_server -flagfile=/im/conf/transmite_server.conf -redis_seeds=redis-node1:6379,redis-node2:6380,redis-node3:6381,redis-node4:6382,redis-node5:6383,redis-node6:6384" + /im/bin/entrypoint.sh -d etcd:2379,mysql:3306,redis-node1:6379,redis-node2:6380,redis-node3:6381,redis-node4:6382,redis-node5:6383,redis-node6:6384,rabbitmq:5672 -c "/im/bin/transmite_server -flagfile=/im/conf/transmite_server.conf -redis_seeds=redis-node1:6379,redis-node2:6380,redis-node3:6381,redis-node4:6382,redis-node5:6383,redis-node6:6384 -rate_limit_user_max=${TRANSMITE_RATE_LIMIT_USER_MAX:-600} -rate_limit_session_max=${TRANSMITE_RATE_LIMIT_SESSION_MAX:-3000}" identity_server: build: ./identity container_name: identity_server-service diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile new file mode 100644 index 0000000..a4c9b06 --- /dev/null +++ b/docker/ci/Dockerfile @@ -0,0 +1,176 @@ +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} + +ENV DEBIAN_FRONTEND=noninteractive +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN apt-get -o Acquire::Retries=5 update \ + && apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + cmake \ + git \ + libboost-all-dev \ + libcpprest-dev \ + libcrypt-dev \ + libcurl4-openssl-dev \ + libev-dev \ + libfmt-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libgrpc-dev \ + libhiredis-dev \ + libjsoncpp-dev \ + libleveldb-dev \ + libmysqlclient-dev \ + libodb-boost-dev \ + libodb-dev \ + libodb-mysql-dev \ + libprotobuf-dev \ + libsnappy-dev \ + libspdlog-dev \ + libssl-dev \ + libtool \ + libunwind-dev \ + libxml2-dev \ + ninja-build \ + odb \ + pkg-config \ + protobuf-compiler \ + protobuf-compiler-grpc \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY docker/ci/dependencies.lock /tmp/dependencies.lock + +RUN set -euo pipefail; \ + . /tmp/dependencies.lock; \ + git init /tmp/brpc; \ + git -C /tmp/brpc remote add origin https://github.com/apache/brpc.git; \ + git -C /tmp/brpc fetch --depth 1 origin "$BRPC_REV"; \ + git -C /tmp/brpc checkout --detach FETCH_HEAD; \ + test "$(git -C /tmp/brpc rev-parse HEAD)" = "$BRPC_REV"; \ + cmake -S /tmp/brpc -B /tmp/brpc-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_BRPC_TOOLS=OFF \ + -DBUILD_UNIT_TESTS=OFF \ + -DWITH_GLOG=ON \ + -DWITH_SNAPPY=ON; \ + cmake --build /tmp/brpc-build --parallel "$(nproc)"; \ + cmake --install /tmp/brpc-build; \ + rm -rf /tmp/brpc /tmp/brpc-build + +RUN set -euo pipefail; \ + . /tmp/dependencies.lock; \ + git init /tmp/etcd-cpp-apiv3; \ + git -C /tmp/etcd-cpp-apiv3 remote add origin https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git; \ + git -C /tmp/etcd-cpp-apiv3 fetch --depth 1 origin "$ETCD_CPP_APIV3_REV"; \ + git -C /tmp/etcd-cpp-apiv3 checkout --detach FETCH_HEAD; \ + test "$(git -C /tmp/etcd-cpp-apiv3 rev-parse HEAD)" = "$ETCD_CPP_APIV3_REV"; \ + cmake -S /tmp/etcd-cpp-apiv3 -B /tmp/etcd-cpp-apiv3-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_ETCD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DETCD_W_STRICT=OFF; \ + cmake --build /tmp/etcd-cpp-apiv3-build --parallel "$(nproc)"; \ + cmake --install /tmp/etcd-cpp-apiv3-build; \ + rm -rf /tmp/etcd-cpp-apiv3 /tmp/etcd-cpp-apiv3-build + +RUN set -euo pipefail; \ + . /tmp/dependencies.lock; \ + git init /tmp/redis-plus-plus; \ + git -C /tmp/redis-plus-plus remote add origin https://github.com/sewenew/redis-plus-plus.git; \ + git -C /tmp/redis-plus-plus fetch --depth 1 origin "$REDIS_PLUS_PLUS_REV"; \ + git -C /tmp/redis-plus-plus checkout --detach FETCH_HEAD; \ + test "$(git -C /tmp/redis-plus-plus rev-parse HEAD)" = "$REDIS_PLUS_PLUS_REV"; \ + cmake -S /tmp/redis-plus-plus -B /tmp/redis-plus-plus-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DREDIS_PLUS_PLUS_BUILD_STATIC=OFF \ + -DREDIS_PLUS_PLUS_BUILD_TEST=OFF; \ + cmake --build /tmp/redis-plus-plus-build --parallel "$(nproc)"; \ + cmake --install /tmp/redis-plus-plus-build; \ + rm -rf /tmp/redis-plus-plus /tmp/redis-plus-plus-build + +RUN set -euo pipefail; \ + . /tmp/dependencies.lock; \ + git init /tmp/cpr; \ + git -C /tmp/cpr remote add origin https://github.com/whoshuu/cpr.git; \ + git -C /tmp/cpr fetch --depth 1 origin "$CPR_REV"; \ + git -C /tmp/cpr checkout --detach FETCH_HEAD; \ + test "$(git -C /tmp/cpr rev-parse HEAD)" = "$CPR_REV"; \ + cmake -S /tmp/cpr -B /tmp/cpr-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_CPR_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DUSE_SYSTEM_CURL=ON; \ + cmake --build /tmp/cpr-build --parallel "$(nproc)"; \ + cmake --install /tmp/cpr-build; \ + rm -rf /tmp/cpr /tmp/cpr-build + +RUN set -euo pipefail; \ + . /tmp/dependencies.lock; \ + git init /tmp/elasticlient; \ + git -C /tmp/elasticlient remote add origin https://github.com/seznam/elasticlient.git; \ + git -C /tmp/elasticlient fetch --depth 1 origin "$ELASTICLIENT_REV"; \ + git -C /tmp/elasticlient checkout --detach FETCH_HEAD; \ + test "$(git -C /tmp/elasticlient rev-parse HEAD)" = "$ELASTICLIENT_REV"; \ + cmake -S /tmp/elasticlient -B /tmp/elasticlient-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_ELASTICLIENT_EXAMPLE=OFF \ + -DBUILD_ELASTICLIENT_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DUSE_SYSTEM_CPR=ON \ + -DUSE_SYSTEM_JSONCPP=ON; \ + cmake --build /tmp/elasticlient-build --parallel "$(nproc)"; \ + cmake --install /tmp/elasticlient-build; \ + rm -rf /tmp/elasticlient /tmp/elasticlient-build + +RUN set -euo pipefail; \ + . /tmp/dependencies.lock; \ + git init /tmp/amqp-cpp; \ + git -C /tmp/amqp-cpp remote add origin https://github.com/CopernicaMarketingSoftware/AMQP-CPP.git; \ + git -C /tmp/amqp-cpp fetch --depth 1 origin "$AMQP_CPP_REV"; \ + git -C /tmp/amqp-cpp checkout --detach FETCH_HEAD; \ + test "$(git -C /tmp/amqp-cpp rev-parse HEAD)" = "$AMQP_CPP_REV"; \ + cmake -S /tmp/amqp-cpp -B /tmp/amqp-cpp-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DAMQP-CPP_BUILD_EXAMPLES=OFF \ + -DAMQP-CPP_BUILD_SHARED=ON; \ + cmake --build /tmp/amqp-cpp-build --parallel "$(nproc)"; \ + cmake --install /tmp/amqp-cpp-build; \ + rm -rf /tmp/amqp-cpp /tmp/amqp-cpp-build + +RUN set -euo pipefail; \ + . /tmp/dependencies.lock; \ + git init /tmp/aws-sdk-cpp; \ + git -C /tmp/aws-sdk-cpp remote add origin https://github.com/aws/aws-sdk-cpp.git; \ + git -C /tmp/aws-sdk-cpp fetch --depth 1 origin "$AWS_SDK_CPP_REV"; \ + git -C /tmp/aws-sdk-cpp checkout --detach FETCH_HEAD; \ + test "$(git -C /tmp/aws-sdk-cpp rev-parse HEAD)" = "$AWS_SDK_CPP_REV"; \ + git -C /tmp/aws-sdk-cpp submodule sync --recursive; \ + git -C /tmp/aws-sdk-cpp submodule update --init --recursive --depth 1; \ + test -z "$(git -C /tmp/aws-sdk-cpp submodule status --recursive | grep -E '^[+-]' || true)"; \ + cmake -S /tmp/aws-sdk-cpp -B /tmp/aws-sdk-cpp-build -G Ninja \ + -DBUILD_ONLY=s3 \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DENABLE_TESTING=OFF; \ + cmake --build /tmp/aws-sdk-cpp-build --parallel "$(nproc)"; \ + cmake --install /tmp/aws-sdk-cpp-build; \ + rm -rf /tmp/aws-sdk-cpp /tmp/aws-sdk-cpp-build + +RUN printf '%s\n' '/usr/local/lib' '/usr/local/lib64' > /etc/ld.so.conf.d/chatnow-local.conf \ + && ldconfig + +WORKDIR /workspace diff --git a/docker/ci/dependencies.lock b/docker/ci/dependencies.lock new file mode 100644 index 0000000..6fb6ee0 --- /dev/null +++ b/docker/ci/dependencies.lock @@ -0,0 +1,10 @@ +# Base image and source dependency revisions used by docker/ci/Dockerfile. +# Git dependencies are full commit IDs; do not replace them with branches or tags. +UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +BRPC_REV=041cec5fb84a5b4458bac6275ea7d34e048bc3f1 +ETCD_CPP_APIV3_REV=ba6216385fc332b23d95683966824c2b86c2474e +REDIS_PLUS_PLUS_REV=a63ac43bf192772910b52e27cd2b42a6098a0071 +CPR_REV=fe29aee4ca4bfc9d17802e7623c2ee574b1d1e9b +ELASTICLIENT_REV=d68e30e382b5f2817be8cd901494736b26d4896e +AMQP_CPP_REV=ca49382bfc5bc165dfb7988b891bb010a939a786 +AWS_SDK_CPP_REV=2cde9b1786bdbb3182faa93ce28d6e44ac2fe7e0 diff --git a/docs/superpowers/plans/2026-07-15-cache-resilience-completion-plan.md b/docs/superpowers/plans/2026-07-15-cache-resilience-completion-plan.md new file mode 100644 index 0000000..eece2b8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-15-cache-resilience-completion-plan.md @@ -0,0 +1,763 @@ +# Cache Resilience Completion Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve open cache issues #35, #37, #38, #45 and the cache/rate-limit portion of #48 with fast Redis failure isolation, bounded local degradation, UserInfo L2 caching, complete TTL jitter, and contention-safe lock retry. + +**Architecture:** A per-`RedisClient` lock-free circuit breaker guards all Redis commands; domain DAOs retain responsibility for semantic fallback. `RateLimiter` falls back to a 64-shard in-process token bucket, while UserInfo follows L1 → L2 → singleflight Identity RPC cache-aside. Tests use the PR #49/#54 pure-Go `func`, `reliability`, and `perf` layers. + +**Tech Stack:** C++17, redis-plus-plus, bvar, brpc/protobuf, Go 1.23 `testing` + `testify`, Docker Compose, Redis Cluster 7.2. + +## Global Constraints + +- Production baseline is exactly `origin/3.0-dev@1cad99a`; do not merge the unmerged PR #49 branch into this branch. +- Peak target is 5000 msg/s with 2–8 service instances and groups up to 2000 members. +- Once open, Redis circuit rejection must complete within 10–50ms; no request may retain the old 2s Redis wait. +- Redis outage must retain bounded rate limiting; Redis-backed truth sources must return unavailable instead of fabricated state. +- UserInfo hot-path Identity RPC calls must fall by at least 95%. +- All new tests are Go black-box tests with `func`, `reliability`, or `perf` tags; add no C++ gtest or Python contract tests. +- Do not add an external circuit-breaker service, background probe thread, service-discovery quota splitter, or new runtime dependency. +- UserInfo L2 keys use 64 virtual cluster hash buckets: `im:user:{bucket}:uid`, with `bucket = fnv1a(uid) % 64`. + +--- + +## File Structure + +**Create:** + +- `common/utils/redis_circuit_breaker.hpp` — atomic Closed/Open/HalfOpen state machine. +- `common/utils/local_rate_limiter.hpp` — bounded 64-shard fallback token bucket. +- `tests/pkg/chaos/redis.go` — stop/start/wait helpers for the six Redis nodes. +- `tests/pkg/verify/redis.go` — Redis CLI and bvar read helpers. +- `tests/func/cache_test.go` — FN-CA-01 through FN-CA-05. +- `tests/reliability/redis_failover_test.go` — RL-05. +- `tests/reliability/setup_test.go` — reliability-tag HTTP setup compatible with PR #49. +- `tests/perf/cache_test.go` — PF-09. + +**Modify:** + +- `common/dao/data_redis.hpp` — guarded RedisClient commands, UserInfoCache, jittered TTLs, RateLimiter fallback. +- `common/utils/redis_keys.hpp` — UserInfo bucket/key helpers. +- `common/utils/redis_mutex.hpp` — bounded exponential backoff with jitter. +- `common/infra/metrics.hpp` — circuit, fallback, UserInfo, and mutex metrics. +- `transmite/source/transmite_server.h` — UserInfo cache-aside path and dependency injection. +- `identity/source/identity_server.h` — invalidate UserInfo L2 after profile update. +- `tests/Makefile` — reliability target when absent; preserve PR #49-compatible target names. + +--- + +### Task 1: Go Redis chaos and verification helpers + +**Files:** + +- Create: `tests/pkg/chaos/redis.go` +- Create: `tests/pkg/verify/redis.go` +- Modify: `tests/pkg/client/config.go` +- Modify: `tests/config.yaml` + +**Interfaces:** + +- Produces: `chaos.StopRedisCluster(testing.TB)`, `chaos.StartRedisCluster(testing.TB)`, `chaos.WaitRedisCluster(testing.TB, time.Duration)`. +- Produces: `verify.RedisCLI(testing.TB, ...string) string`, `verify.RedisTTL(testing.TB, string) time.Duration`, `verify.BVar(testing.TB, string, string) int64`. + +- [ ] **Step 1: Add Redis and service metrics endpoints to test config** + +Add stable, environment-overridable fields to `tests/pkg/client/config.go` and values to `tests/config.yaml`: + +```go +type InfraConfig struct { + RedisContainer string `yaml:"redis_container"` + ComposeDir string `yaml:"compose_dir"` + TransmiteVars string `yaml:"transmite_vars"` +} + +// Config gains: +Infra InfraConfig `yaml:"infra"` +``` + +```yaml +infra: + redis_container: "redis-node1" + compose_dir: ".." + transmite_vars: "http://localhost:10004" +``` + +- [ ] **Step 2: Implement Docker Compose Redis control** + +Create `tests/pkg/chaos/redis.go` with these exact service names and recovery check: + +```go +package chaos + +import ( + "fmt" + "os/exec" + "testing" + "time" +) + +var redisServices = []string{ + "redis-node1", "redis-node2", "redis-node3", + "redis-node4", "redis-node5", "redis-node6", +} + +func compose(t testing.TB, args ...string) { + t.Helper() + cmd := exec.Command("docker", append([]string{"compose"}, args...)...) + cmd.Dir = ".." + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("docker compose %v: %v: %s", args, err, out) + } +} + +func StopRedisCluster(t testing.TB) { compose(t, append([]string{"stop"}, redisServices...)...) } +func StartRedisCluster(t testing.TB) { compose(t, append([]string{"start"}, redisServices...)...) } + +func WaitRedisCluster(t testing.TB, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + cmd := exec.Command("docker", "exec", "redis-node1", "redis-cli", "cluster", "info") + if out, err := cmd.Output(); err == nil && string(out) != "" { + return + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("redis cluster did not recover within %s", timeout) +} +``` + +- [ ] **Step 3: Implement Redis and bvar verification** + +Create `tests/pkg/verify/redis.go`: + +```go +package verify + +import ( + "fmt" + "io" + "net/http" + "os/exec" + "strconv" + "strings" + "testing" + "time" +) + +func RedisCLI(t testing.TB, args ...string) string { + t.Helper() + base := []string{"exec", "redis-node1", "redis-cli", "-c"} + out, err := exec.Command("docker", append(base, args...)...).CombinedOutput() + if err != nil { t.Fatalf("redis-cli %v: %v: %s", args, err, out) } + return strings.TrimSpace(string(out)) +} + +func RedisTTL(t testing.TB, key string) time.Duration { + seconds, err := strconv.ParseInt(RedisCLI(t, "TTL", key), 10, 64) + if err != nil || seconds < 0 { t.Fatalf("invalid TTL for %s: %d (%v)", key, seconds, err) } + return time.Duration(seconds) * time.Second +} + +func BVar(t testing.TB, baseURL, name string) int64 { + t.Helper() + rsp, err := http.Get(fmt.Sprintf("%s/vars/%s", baseURL, name)) + if err != nil { t.Fatalf("read bvar %s: %v", name, err) } + defer rsp.Body.Close() + body, err := io.ReadAll(rsp.Body) + if err != nil { t.Fatalf("read bvar body %s: %v", name, err) } + value, err := strconv.ParseInt(strings.TrimSpace(string(body)), 10, 64) + if err != nil { t.Fatalf("parse bvar %s=%q: %v", name, body, err) } + return value +} +``` + +- [ ] **Step 4: Compile helper packages** + +Run: `cd tests && gofmt -w pkg/chaos/redis.go pkg/verify/redis.go pkg/client/config.go && go test ./pkg/chaos ./pkg/verify ./pkg/client` + +Expected: PASS or `[no test files]` for each package, with no compile error. + +- [ ] **Step 5: Commit** + +```bash +git add tests/pkg/chaos/redis.go tests/pkg/verify/redis.go tests/pkg/client/config.go tests/config.yaml +git commit -m "test: add Redis chaos verification helpers" +``` + +--- + +### Task 2: Redis circuit breaker and guarded RedisClient + +**Files:** + +- Create: `common/utils/redis_circuit_breaker.hpp` +- Modify: `common/dao/data_redis.hpp` +- Modify: `common/infra/metrics.hpp` +- Create: `tests/reliability/setup_test.go` +- Create: `tests/reliability/redis_failover_test.go` +- Modify: `tests/Makefile` + +**Interfaces:** + +- Produces: `RedisCircuitBreaker::Permit before_call()`, `on_success(Permit)`, `on_connection_failure(Permit)`. +- Produces: `RedisCircuitOpen`, used by semantic fallbacks. +- `RedisClient` public command signatures remain source-compatible. + +- [ ] **Step 1: Write failing RL-05 fast-fail/recovery test** + +Create `tests/reliability/setup_test.go` with build tag `reliability`, load `tests/config.yaml`, and initialize the package-level HTTP client exactly like `tests/func/setup_test.go`. + +Create `tests/reliability/redis_failover_test.go`: + +```go +//go:build reliability + +package reliability_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "chatnow-tests/pkg/chaos" + "chatnow-tests/pkg/client" + "chatnow-tests/pkg/fixture" + identity "chatnow-tests/proto/chatnow/identity" +) + +// RL-05 | P0 | Redis 熔断后快速失败并自动恢复 +func TestRL_RedisCircuitFastFailAndRecovery(t *testing.T) { + user, _, _ := fixture.RegisterAndLogin(t, HTTP) + t.Cleanup(func() { chaos.StartRedisCluster(t); chaos.WaitRedisCluster(t, 60*time.Second) }) + chaos.StopRedisCluster(t) + + uid := user.UserID + for i := 0; i < 3; i++ { + rsp := &identity.GetProfileRsp{} + _ = user.DoAuth("/service/identity/get_profile", &identity.GetProfileReq{ + RequestId: client.NewRequestID(), UserId: &uid, + }, rsp) + } + + started := time.Now() + rsp := &identity.GetProfileRsp{} + err := user.DoAuth("/service/identity/get_profile", &identity.GetProfileReq{ + RequestId: client.NewRequestID(), UserId: &uid, + }, rsp) + require.NoError(t, err) + require.True(t, rsp.GetHeader().GetSuccess()) + require.Less(t, time.Since(started), 50*time.Millisecond) + + chaos.StartRedisCluster(t) + chaos.WaitRedisCluster(t, 60*time.Second) + time.Sleep(1100 * time.Millisecond) + rsp = &identity.GetProfileRsp{} + require.NoError(t, user.DoAuth("/service/identity/get_profile", &identity.GetProfileReq{ + RequestId: client.NewRequestID(), UserId: &uid, + }, rsp)) + require.True(t, rsp.GetHeader().GetSuccess()) +} +``` + +- [ ] **Step 2: Run RL-05 and verify the old implementation fails** + +Run on Linux stack: `cd tests && go test -tags=reliability ./reliability/... -run TestRL_RedisCircuitFastFailAndRecovery -v -count=1 -timeout=180s` + +Expected before implementation: FAIL because Redis-backed auth calls retain the old 2s timeout and the final request exceeds 50ms. + +- [ ] **Step 3: Implement the atomic circuit state machine** + +Create `common/utils/redis_circuit_breaker.hpp` with this public shape and constants: + +```cpp +#pragma once +#include +#include +#include +#include + +namespace chatnow { +class RedisCircuitOpen final : public std::runtime_error { +public: RedisCircuitOpen() : std::runtime_error("redis circuit open") {} +}; + +class RedisCircuitBreaker { +public: + enum class State : uint8_t { Closed, Open, HalfOpen }; + struct Permit { bool probe = false; }; + + Permit before_call(); + void on_success(Permit permit) noexcept; + void on_connection_failure(Permit permit) noexcept; + State state() const noexcept { return _state.load(std::memory_order_acquire); } + +private: + static constexpr uint32_t kFailureThreshold = 3; + static constexpr int64_t kOpenForMs = 1000; + static int64_t now_ms() noexcept; + void open() noexcept; + + std::atomic _state{State::Closed}; + std::atomic _consecutive_failures{0}; + std::atomic _open_until_ms{0}; +}; +} // namespace chatnow +``` + +Use these inline state transitions: + +```cpp +inline int64_t RedisCircuitBreaker::now_ms() noexcept { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +inline void RedisCircuitBreaker::open() noexcept { + _open_until_ms.store(now_ms() + kOpenForMs, std::memory_order_release); + _state.store(State::Open, std::memory_order_release); +} + +inline RedisCircuitBreaker::Permit RedisCircuitBreaker::before_call() { + auto state = _state.load(std::memory_order_acquire); + if (state == State::Closed) return {}; + if (state == State::HalfOpen) throw RedisCircuitOpen(); + if (now_ms() < _open_until_ms.load(std::memory_order_acquire)) { + throw RedisCircuitOpen(); + } + auto expected = State::Open; + if (_state.compare_exchange_strong(expected, State::HalfOpen, + std::memory_order_acq_rel)) { + return Permit{true}; + } + throw RedisCircuitOpen(); +} + +inline void RedisCircuitBreaker::on_success(Permit) noexcept { + _consecutive_failures.store(0, std::memory_order_relaxed); + _state.store(State::Closed, std::memory_order_release); +} + +inline void RedisCircuitBreaker::on_connection_failure(Permit permit) noexcept { + if (permit.probe || + _consecutive_failures.fetch_add(1, std::memory_order_relaxed) + 1 >= + kFailureThreshold) { + open(); + } +} +``` + +- [ ] **Step 4: Guard every RedisClient command** + +In `common/dao/data_redis.hpp`, add `_breaker` and private helpers: + +```cpp +template +decltype(auto) guarded_(F &&fn) { + auto permit = _breaker.before_call(); + try { + decltype(auto) result = std::forward(fn)(); + _breaker.on_success(permit); + return result; + } catch (const sw::redis::IoError &) { + _breaker.on_connection_failure(permit); throw; + } catch (const sw::redis::TimeoutError &) { + _breaker.on_connection_failure(permit); throw; + } catch (const sw::redis::ClosedError &) { + _breaker.on_connection_failure(permit); throw; + } +} + +template +void guarded_void_(F &&fn) { + auto permit = _breaker.before_call(); + try { std::forward(fn)(); _breaker.on_success(permit); } + catch (const sw::redis::IoError &) { _breaker.on_connection_failure(permit); throw; } + catch (const sw::redis::TimeoutError &) { _breaker.on_connection_failure(permit); throw; } + catch (const sw::redis::ClosedError &) { _breaker.on_connection_failure(permit); throw; } +} +``` + +Route `get`, all four `set` overloads, `del`, `expire`, `incr`, both `sadd` overloads, `smembers`, `srem`, `scard`, `hset`, `hget`, `hdel`, `hkeys`, `hgetall`, `hlen`, both `zadd` overloads, `zrem`, `zrange`, `zrangebyscore`, both `eval` overloads, and `scan` through these helpers. Preserve their existing public signatures. + +Add a same-slot multi-get used by UserInfo bucket groups: + +```cpp +template +void mget(Input first, Input last, Output out) { + guarded_void_([&] { + _rc ? _rc->mget(first, last, out) : _r->mget(first, last, out); + }); +} +``` + +Add a move-only `RedisPipeline` wrapper exposing `hset`, `set`, `expire`, `get`, and `exec`; it stores the permit and only reports success/failure when `exec()` runs. Change `RedisClient::pipeline` to return the wrapper. + +- [ ] **Step 5: Tighten Redis runtime timeouts and add metrics** + +Set standalone and cluster factory values to 50ms connect/socket and 20ms pool wait. Add bvar counters named exactly as the design specifies and increment them only on connection failure, open transition, fast rejection, and recovery. + +- [ ] **Step 6: Compile and rerun RL-05** + +Run: `cmake -S . -B build && cmake --build build -j2` + +Expected: all C++ targets compile. + +Run on Linux stack: `cd tests && go test -tags=reliability ./reliability/... -run TestRL_RedisCircuitFastFailAndRecovery -v -count=1 -timeout=180s` + +Expected: PASS; stable outage request under 50ms and recovery without service restart. + +- [ ] **Step 7: Commit** + +```bash +git add common/utils/redis_circuit_breaker.hpp common/dao/data_redis.hpp common/infra/metrics.hpp tests/reliability tests/Makefile +git commit -m "feat: add fast Redis circuit breaking" +``` + +--- + +### Task 3: Bounded local rate-limit fallback + +**Files:** + +- Create: `common/utils/local_rate_limiter.hpp` +- Modify: `common/dao/data_redis.hpp` +- Modify: `common/infra/metrics.hpp` +- Modify: `tests/reliability/redis_failover_test.go` +- Create: `tests/func/cache_test.go` + +**Interfaces:** + +- Produces: `LocalRateLimiter::allow(key, capacity, window, now)`. +- `RateLimiter::allow_user` and `allow_session` signatures remain unchanged. + +- [ ] **Step 1: Add failing Redis-outage rate-limit assertion** + +Extend RL-05 after prewarming a conversation. Send 650 unique messages while Redis is stopped and count responses whose header error message is `rate_limited`. Require the count to be greater than zero. Before implementation it is zero because `RateLimiter::allow` returns true on every Redis exception. + +Add `FN-CA-05` to `tests/func/cache_test.go`: send 650 messages with Redis healthy and require at least one rate-limited response. + +- [ ] **Step 2: Implement sharded token buckets** + +Create `common/utils/local_rate_limiter.hpp`: + +```cpp +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace chatnow { +class LocalRateLimiter { +public: + bool allow(const std::string &key, int capacity, int window_sec); +private: + struct Bucket { double tokens; int64_t refill_ms; int64_t seen_ms; }; + struct Shard { std::mutex mu; std::unordered_map buckets; }; + static constexpr size_t kShardCount = 64; + static constexpr size_t kMaxBuckets = 65536; + std::array _shards; + std::atomic _bucket_count{0}; + std::atomic _operations{0}; +}; +} // namespace chatnow +``` + +Use steady-clock milliseconds. Refill `elapsed * capacity / (window_sec * 1000)`, cap at capacity, consume one token when available, and update `seen_ms`. Every 1024 operations, remove entries idle for more than two windows from the current shard. At the hard cap, deny unknown keys after cleanup. + +- [ ] **Step 3: Wire semantic fallback** + +Give `RateLimiter` a `LocalRateLimiter _local`. In the existing catch block replace `return true` with: + +```cpp +metrics::g_rate_limit_local_fallback_total << 1; +const bool allowed = _local.allow(key_full, max_count, window_sec); +if (!allowed) metrics::g_rate_limit_local_rejected_total << 1; +return allowed; +``` + +Keep Redis Lua as the only normal path; do not double-write local state. + +- [ ] **Step 4: Verify** + +Run: `cmake --build build -j2` + +Run on Linux stack: `cd tests && go test -tags=func ./func/... -run TestFN_CA_RateLimit -v -count=1 && go test -tags=reliability ./reliability/... -run TestRL_RedisCircuitFastFailAndRecovery -v -count=1 -timeout=180s` + +Expected: both healthy Redis and stopped Redis produce bounded rate-limit rejection. + +- [ ] **Step 5: Commit** + +```bash +git add common/utils/local_rate_limiter.hpp common/dao/data_redis.hpp common/infra/metrics.hpp tests/func/cache_test.go tests/reliability/redis_failover_test.go +git commit -m "feat: retain rate limiting during Redis outages" +``` + +--- + +### Task 4: UserInfo L1/L2/RPC cache-aside + +**Files:** + +- Modify: `common/utils/redis_keys.hpp` +- Modify: `common/dao/data_redis.hpp` +- Modify: `common/infra/metrics.hpp` +- Modify: `transmite/source/transmite_server.h` +- Modify: `identity/source/identity_server.h` +- Modify: `tests/func/cache_test.go` + +**Interfaces:** + +- Produces: `key::user_info_bucket(uid)`, `key::user_info_key(uid)`. +- Produces: `UserInfoCache::get`, `batch_get`, `set`, `batch_set`, `invalidate` over serialized protobuf strings. +- `TransmiteServiceImpl` gains `UserInfoCache::ptr` but preserves RPC API. + +- [ ] **Step 1: Write failing cache behavior tests** + +Add these functions to `tests/func/cache_test.go`: + +- `TestFN_CA_UserInfoL2AvoidsRepeatedRPC`: record `user_info_rpc_total`, send 20 messages from one user, require delta ≤ 1 and Redis key existence. +- `TestFN_CA_UserInfoSingleflight`: start 200 goroutines sending from one cold user, require `user_info_rpc_total` delta ≤ 1. +- `TestFN_CA_UserInfoInvalidatedAfterProfileUpdate`: warm, update nickname, require old Redis key deleted, send again, then require the stored protobuf reflects the new nickname. + +Run on Linux: `cd tests && go test -tags=func ./func/... -run 'TestFN_CA_UserInfo' -v -count=1` + +Expected before implementation: FAIL because the L2 key and UserInfo metrics do not exist and every send calls Identity. + +- [ ] **Step 2: Add deterministic 64-bucket keys** + +In `common/utils/redis_keys.hpp`, add FNV-1a helpers: + +```cpp +inline uint32_t fnv1a_32(const std::string &value) { + uint32_t hash = 2166136261u; + for (unsigned char c : value) { hash ^= c; hash *= 16777619u; } + return hash; +} +inline uint32_t user_info_bucket(const std::string &uid) { return fnv1a_32(uid) % 64u; } +inline std::string user_info_key(const std::string &uid) { + auto bucket = std::to_string(user_info_bucket(uid)); + return "im:user:{" + bucket + "}:" + uid; +} +``` + +- [ ] **Step 3: Implement UserInfoCache DAO** + +In `common/dao/data_redis.hpp`, add `kUserInfoTtl{3600}` and a `UserInfoCache` class that stores strings. `get` catches Redis errors and returns nullopt; `set` uses `randomized_ttl(kUserInfoTtl)`; `invalidate` deletes the key. + +`batch_get` groups uids by `user_info_bucket`, caps input at 2000, performs one MGET per bucket, and returns `{hits, misses}` preserving uid association. `batch_set` groups the same way and uses one pipeline per bucket. An empty serialized value is a 5-second sentinel; dependency errors are never cached. + +- [ ] **Step 4: Replace Transmite's unconditional profile RPC** + +Inject `_user_info_cache` from `TransmiteServerBuilder::make_redis_object`. Replace `resolve_user_info` with: + +```cpp +std::optional resolve_user_info(const std::string &uid, + const std::string &rid, + brpc::Controller *caller) { + const auto lkey = key::local_user_info_cache_key(uid); + if (_local_user_cache) { + auto local = _local_user_cache->get(lkey); + if (local) { metrics::g_user_info_l1_hit_total << 1; return local; } + } + auto guard = _inflight_registry->acquire("user:" + uid); + std::unique_lock lk(*guard.mu); + if (_local_user_cache) if (auto local = _local_user_cache->get(lkey)) return local; + if (_user_info_cache) if (auto redis = _user_info_cache->get(uid)) { + metrics::g_user_info_l2_hit_total << 1; + _local_user_cache->set(lkey, *redis, randomized_ttl(std::chrono::seconds(45))); + return redis; + } + auto info = fetch_user_info_from_identity_(uid, rid, caller); + if (!info) return std::nullopt; + auto bytes = info->SerializeAsString(); + if (_user_info_cache) _user_info_cache->set(uid, bytes); + if (_local_user_cache) _local_user_cache->set(lkey, bytes, randomized_ttl(std::chrono::seconds(45))); + metrics::g_user_info_rpc_total << 1; + return bytes; +} +``` + +Call it before member resolution. Parse cached bytes into `chatnow::common::UserInfo`; corrupted values invalidate L1/L2 and return unavailable only if the subsequent RPC also fails. Remove the old unconditional async GetProfile call. + +- [ ] **Step 5: Invalidate after successful profile update** + +Construct `UserInfoCache` from Identity's existing `_redis_client`, inject it into `IdentityServiceImpl`, and immediately after `_mysql_user->update(user)` succeeds call `_user_info_cache->invalidate(auth.user_id)`. Cache deletion failure must not roll back the DB update. + +- [ ] **Step 6: Verify** + +Run: `cmake --build build -j2` + +Run on Linux: `cd tests && go test -tags=func ./func/... -run 'TestFN_CA_UserInfo' -v -count=1` + +Expected: all three tests PASS; 20 repeated sends cause at most one Identity RPC and profile update removes L2. + +- [ ] **Step 7: Commit** + +```bash +git add common/utils/redis_keys.hpp common/dao/data_redis.hpp common/infra/metrics.hpp transmite/source/transmite_server.h identity/source/identity_server.h tests/func/cache_test.go +git commit -m "feat: add UserInfo multilevel caching" +``` + +--- + +### Task 5: Complete TTL jitter and RedisMutex backoff + +**Files:** + +- Modify: `common/dao/data_redis.hpp` +- Modify: `common/utils/redis_mutex.hpp` +- Modify: `common/infra/metrics.hpp` +- Modify: `tests/func/cache_test.go` + +**Interfaces:** + +- Existing DAO method signatures remain unchanged. +- UnackedPush's paired keys receive one shared randomized TTL sample per operation. + +- [ ] **Step 1: Write failing FN-CA-04 TTL test** + +Use unique users/devices, exercise login/status/code/device/unacked API paths, read TTLs through `verify.RedisTTL`, and require each TTL to lie within `[0.8*base-2s, 1.2*base+2s]`. Collect at least 20 keys and require at least two distinct TTL values. + +Run on Linux: `cd tests && go test -tags=func ./func/... -run TestFN_CA_TTLJitter -v -count=1` + +Expected before implementation: FAIL because Session/Status/Codes/DeviceSet/UnackedPush use fixed or absent TTLs. + +- [ ] **Step 2: Apply jitter to every missing path** + +Use `randomized_ttl(ttl)` in Session append/touch, Status append/touch, Codes append, and any remaining fixed cache expiration found by `rg '_c->(set|expire).*ttl' common/dao/data_redis.hpp`. + +In `DeviceSet::add`, call `expire(key, randomized_ttl(kSessionTtl))`; add `touch(uid, ttl=kSessionTtl)` and invoke it from device activity paths. + +In `UnackedPush::push` and `bump_score`, calculate exactly once: + +```cpp +const auto effective_ttl = randomized_ttl(ttl); +_c->expire(k, effective_ttl); +_c->expire(ik, effective_ttl); +``` + +- [ ] **Step 3: Implement bounded exponential lock retry** + +Replace the fixed 5ms sleep in `RedisMutex::try_lock` with: + +```cpp +int backoff_ms = 5; +while (std::chrono::steady_clock::now() < deadline) { + if (_redis->set(_key, _token, std::chrono::milliseconds(_ttl_ms), + sw::redis::UpdateType::NOT_EXIST)) { + _locked = true; + return true; + } + metrics::g_redis_mutex_retry_total << 1; + std::uniform_int_distribution jitter(0, backoff_ms / 2); + auto delay = std::chrono::milliseconds(backoff_ms + jitter(rng_())); + auto remaining = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()); + if (remaining <= std::chrono::milliseconds::zero()) break; + std::this_thread::sleep_for(std::min(delay, remaining)); + backoff_ms = std::min(backoff_ms * 2, 20); +} +``` + +Add a private thread-local RNG accessor; preserve SET NX PX and Lua CAS unlock unchanged. + +- [ ] **Step 4: Verify** + +Run: `cmake --build build -j2` + +Run on Linux: `cd tests && go test -tags=func ./func/... -run TestFN_CA_TTLJitter -v -count=1` + +Expected: PASS with in-range, non-identical TTL samples. + +- [ ] **Step 5: Commit** + +```bash +git add common/dao/data_redis.hpp common/utils/redis_mutex.hpp common/infra/metrics.hpp tests/func/cache_test.go +git commit -m "perf: jitter cache TTLs and lock retries" +``` + +--- + +### Task 6: PF-09 performance baseline and complete verification + +**Files:** + +- Create: `tests/perf/cache_test.go` +- Modify: `tests/Makefile` +- Modify: `docs/superpowers/specs/2026-07-15-cache-resilience-completion-design.md` only if implementation constraints require a documented correction. + +**Interfaces:** + +- Produces: `BenchmarkPF09_UserInfoCache` with cold/L2/L1 sub-benchmarks and explicit RPC-reduction metric. + +- [ ] **Step 1: Write PF-09 benchmark** + +Create a perf-tag benchmark that prepares 20 conversations, runs parallel sends from repeated senders, records elapsed latency samples, and calls: + +```go +b.ReportMetric(float64(successes)/elapsed.Seconds(), "msg/s") +b.ReportMetric(float64(p95.Microseconds()), "p95-us") +b.ReportMetric(100*(1-float64(rpcDelta)/float64(successes)), "rpc-reduction-%") +``` + +Fail when RPC reduction is below 95%, steady-state throughput is below 5000 msg/s in the target Linux environment, or the checked-in baseline throughput regresses by more than 10%. + +- [ ] **Step 2: Run static and compile verification** + +Run: + +```bash +git diff --check +cmake -S . -B build +cmake --build build -j2 +cd tests +gofmt -w pkg/chaos/redis.go pkg/verify/redis.go func/cache_test.go reliability/*.go perf/cache_test.go +go vet -tags=func ./func/... ./pkg/... +go vet -tags=reliability ./reliability/... ./pkg/... +go vet -tags=perf ./perf/... ./pkg/... +go test -run '^$' ./... +``` + +Expected: zero formatting errors, all production targets compile, and all Go packages compile. + +- [ ] **Step 3: Run Linux full-stack validation** + +Run: + +```bash +cd tests +make test-func +make test-reliability +go test -tags=perf ./perf/... -run '^$' -bench BenchmarkPF09_UserInfoCache -benchmem -count=3 -benchtime=10s +``` + +Expected: FN-CA-01..05 and RL-05 PASS; PF-09 reports ≥5000 msg/s, p95, and ≥95% RPC reduction. + +- [ ] **Step 4: Review the issue acceptance matrix** + +Confirm with evidence: + +- #35: RL-05 shows fast circuit rejection and automatic recovery. +- #37: FN-CA-01..03 and PF-09 show L2 behavior and ≥95% RPC reduction. +- #38: FN-CA-04 shows jitter and DeviceSet TTL. +- #45: contention metrics show bounded jittered retry. +- #48 cache/rate-limit scope: RL-05 shows local rejection during Redis outage. + +- [ ] **Step 5: Commit** + +```bash +git add tests/perf/cache_test.go tests/Makefile docs/superpowers/specs/2026-07-15-cache-resilience-completion-design.md +git commit -m "test: add cache resilience performance baseline" +``` + +- [ ] **Step 6: Invoke verification-before-completion and requesting-code-review** + +Run the complete verification commands again with fresh output, inspect the full diff against `origin/3.0-dev`, and do not claim completion until both reviews find no unresolved correctness, availability, concurrency, or test-framework issue. diff --git a/docs/superpowers/plans/2026-07-16-pr57-ci-artifacts-native-regressions.md b/docs/superpowers/plans/2026-07-16-pr57-ci-artifacts-native-regressions.md new file mode 100644 index 0000000..026fa22 --- /dev/null +++ b/docs/superpowers/plans/2026-07-16-pr57-ci-artifacts-native-regressions.md @@ -0,0 +1,97 @@ +# PR #57 CI Artifacts and Go Regressions Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Produce runnable service artifacts once per workflow and automatically execute cache regressions through the existing Go full-stack framework. + +**Architecture:** A repository-owned, cached native builder feeds one artifact producer job. Consumer jobs download a validated Compose layout. Regression coverage uses real service boundaries and existing Go test targets; temporary C++ tests are removed. + +**Tech Stack:** GitHub Actions, Docker BuildKit, Ubuntu 24.04, CMake, Bash, Go testing, Docker Compose, Redis Cluster. + +## Global Constraints + +- Build all nine service binaries exactly once per workflow run. +- Do not depend on an unpublished or mutable external builder image. +- Missing native dependencies, binaries, shared libraries, or tests must fail closed; required work may not skip. +- `reliability` and `perf-cache` must depend on and download the same immutable artifact before Compose startup. +- All automated regression tests use the existing Go framework; do not add CTest, gtest, or production-only timing hooks. +- RL-05 and PF-09 remain Go full-stack gates using their existing Make targets. +- Every full-stack job ends with one `docker compose down -v` step guarded by `if: always()`. + +--- + +### Task 1: Reproducible native builder and Compose artifact packager + +**Files:** +- Create: `docker/ci/Dockerfile` +- Create: `docker/ci/dependencies.lock` +- Create: `scripts/package_compose_artifacts.sh` +- Create: `scripts/validate_compose_artifacts.sh` +- Create: `tests/pkg/contracts/artifacts_test.go` + +**Interfaces:** +- Produces: directory `compose-artifacts//{build,depends}` and `compose-artifacts/MANIFEST.sha256`. +- Services: `conversation gateway identity media message presence push relationship transmite`. + +- [ ] Write Go contract tests that fail unless the builder uses a digest/tag lock file, the packager enumerates all nine services, rejects missing binaries and unresolved `ldd` output, and the validator verifies the manifest plus executable/shared-library closure. +- [ ] Run `cd tests && go test ./pkg/contracts -run 'TestComposeArtifact' -count=1` and confirm it fails because the builder and scripts do not exist. +- [ ] Add the Ubuntu 24.04 builder definition. Install distribution dependencies and build non-distribution dependencies at exact immutable revisions from `docker/ci/dependencies.lock`; configure `/usr/local` through `ldconfig`. Do not use `latest`, an unpinned branch, or a pre-existing local image. +- [ ] Implement the packager with `set -euo pipefail`, explicit service enumeration, root-build source paths `build//_server`, executable checks, `ldd` closure copying, and deterministic `sha256sum` manifest generation. +- [ ] Implement the validator with the same explicit service list, `sha256sum --check`, executable checks, and an isolated `ldd` check for every binary. +- [ ] Run the focused Go contract test and shell syntax checks; expect zero failures. +- [ ] Build the builder image and run CMake plus packaging in it; expect nine binaries and a valid manifest. +- [ ] Commit with message `build(ci): package reproducible service artifacts`. + +### Task 2: Move the two regressions into the Go test framework + +**Files:** +- Modify: `common/test/CMakeLists.txt` +- Delete: `common/test/test_user_info_generation_fence.cc` +- Delete: `common/test/test_unacked_pending_ledger.cc` +- Modify: `tests/func/cache_test.go` +- Modify: `tests/pkg/client/http.go` +- Modify: `tests/pkg/contracts/ci_gates_test.go` + +**Interfaces:** +- Produces a direct protobuf HTTP client for the Push service test boundary. +- Produces Go cache tests executed by `make test-func`. +- Consumes the existing Redis Cluster and WebSocket helpers. + +- [ ] Write a failing Go contract test proving the temporary C++ files/target are still present and the Go Unacked business regression is absent. +- [ ] Add a failing Go functional test that establishes a Push route, invokes PushToUser twice with one `user_seq` and different payloads, verifies one ZSET identity/latest HASH payload, sends WebSocket ACK, and verifies both indexes are removed. +- [ ] Add the smallest direct protobuf HTTP helper needed to call the Push service without bypassing production serialization. +- [ ] Strengthen the existing UserInfo invalidation Go scenario so its comments and assertions explicitly cover bounded stale-data publication and latest Redis repopulation without a timing-only production hook. +- [ ] Remove both temporary C++ regression files and remove the Unacked CMake executable block. +- [ ] Run the focused Go contract/helper tests and Go compile/vet checks; expect zero failures. +- [ ] Commit with message `test(cache): move regressions into Go framework`. + +### Task 3: Wire the artifact producer and consumers into CI + +**Files:** +- Modify: `.github/workflows/ci.yml` +- Modify: `tests/pkg/contracts/ci_gates_test.go` + +**Interfaces:** +- Producer job: `service-artifacts`. +- Artifact name: `compose-service-artifacts`. +- Consumers: `reliability`, `perf-cache`. + +- [ ] Extend workflow contract tests to require one producer; BuildKit GHA caching; builder build, package, validate, and upload in order; consumer `needs`, download, validate, Compose start, Go gate, and teardown in order. +- [ ] Run `cd tests && go test ./pkg/contracts -count=1`; confirm the new assertions fail against the old workflow. +- [ ] Add `service-artifacts` to the workflow using `docker/build-push-action` cache-to/cache-from `type=gha`, then execute the native build and packaging inside that image and upload `compose-artifacts` with `actions/upload-artifact`. +- [ ] Make both consumers depend on the producer, download with `actions/download-artifact`, restore service-context directories, validate before Compose, and run the existing Go gates after stack readiness. +- [ ] Preserve PF-09 rate-limit overrides and strict final teardown behavior. +- [ ] Run the full contract suite, YAML parse, and `docker compose config --quiet`; expect zero failures. +- [ ] Commit with message `ci: distribute service artifacts to cache gates`. + +### Task 4: End-to-end verification and PR update + +**Files:** +- Modify only if verification exposes a defect in Tasks 1-3. + +- [ ] Run `git diff --check` and inspect the complete branch diff. +- [ ] Run `cd tests && PATH="$(go env GOPATH)/bin:$PATH" make proto` followed by `go test ./...`, tagged vet commands, and the PF-09 race helper command. +- [ ] Run static builder checks, package/validator behavior tests, and all Go contract/helper suites; do not require a local nine-service native build. +- [ ] Verify workflow ordering and that the Go functional/reliability/performance targets are the only regression entry points. +- [ ] Push the branch and inspect the new GitHub Actions run. Do not claim green if an external failure remains. +- [ ] Fetch review threads again and report which new threads are addressed. Do not reply to or resolve GitHub threads without explicit user authorization. diff --git a/docs/superpowers/plans/2026-07-16-pr57-review-fixes.md b/docs/superpowers/plans/2026-07-16-pr57-review-fixes.md new file mode 100644 index 0000000..c285363 --- /dev/null +++ b/docs/superpowers/plans/2026-07-16-pr57-review-fixes.md @@ -0,0 +1,180 @@ +# PR #57 Review Fixes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve all three requested-change threads with explicit cache fencing, an identity-stable Unacked pending ledger, and executable RL-05/PF-09 CI gates. + +**Architecture:** UserInfo cache fills expose committed/conflict/unavailable outcomes and publish L1 only when the outcome permits it. Unacked uses a ZSET of stable `user_seq` members plus a HASH of payloads, maintained atomically by Lua. Reliability and cache performance run as separate full-stack workflow jobs. + +**Tech Stack:** C++17, redis-plus-plus, Redis Cluster Lua, Go 1.24 tests, Docker Compose, GitHub Actions. + +## Global Constraints + +- Treat `seq_id`, `user_seq`, and `message_id` as distinct domains. +- Do not implement issue #58 in this change. +- Do not read or migrate the former `user_seq:payload` ZSET layout. +- Use the new Go test framework and preserve default production rate limits. +- Follow red-green-refactor and commit each task independently. + +--- + +### Task 1: Make UserInfo generation fencing explicit + +**Files:** +- Modify: `common/dao/data_redis.hpp` +- Modify: `transmite/source/transmite_server.h` +- Create: `common/test/test_user_info_generation_fence.cc` + +**Interfaces:** +- Produce: `enum class GenerationWriteResult { Committed, Conflict, Unavailable }`. +- Produce: `UserInfoCache::set_if_generation(...) -> GenerationWriteResult`. +- Preserve: `batch_set(...) -> size_t`, counting only committed writes. + +- [ ] **Step 1: Write the failing policy test** + +Create a focused test that asserts `Committed` permits L1 publication, +`Conflict` denies it, and `Unavailable` permits only the short-lived fallback. +It must also assert that a generation-aware fill cannot treat `Conflict` as +Redis unavailability. + +- [ ] **Step 2: Run the test and verify RED** + +Run the repository's existing C++ test compile command for +`test_user_info_generation_fence.cc`. Expected: compilation fails because +`GenerationWriteResult` and the publication policy do not exist. + +- [ ] **Step 3: Implement the three-state result** + +Return `Conflict` only when Lua executes and returns zero. Return `Unavailable` +for a missing Redis client, `RedisCircuitOpen`, or another Redis exception. +Update batch writes to count only `Committed` entries. + +- [ ] **Step 4: Enforce the policy in Transmite** + +Store the CAS result. Write L1 after `Committed`; skip L1 after `Conflict`; +write the existing randomized 45-second L1 fallback only when generation could +not be observed or the fenced write returns `Unavailable`. Always return the +Identity response to the current caller. + +- [ ] **Step 5: Verify GREEN and regressions** + +Run the new test, the existing cache harness, C++ syntax compilation for +`transmite_server.h`, and `git diff --check`. Expected: all exit zero. + +- [ ] **Step 6: Commit** + +Commit message: `fix(cache): fence UserInfo L1 publication`. + +### Task 2: Redesign Unacked as a stable pending ledger + +**Files:** +- Modify: `common/dao/data_redis.hpp` +- Create: `common/test/test_unacked_pending_ledger.cc` +- Modify: `tests/func/cache_test.go` only if the new framework needs black-box coverage. + +**Interfaces:** +- Preserve: `push`, `ack`, `peek_due`, and `bump_score` C++ signatures. +- Change Redis ZSET member to decimal `user_seq` only. +- Keep HASH field as decimal `user_seq` and value as `payload_b64`. + +- [ ] **Step 1: Write the failing real-Redis tests** + +Cover: pushing the same sequence with payload A then B leaves `ZCARD == 1` and +returns B; ACK removes both keys' entries; a ZSET-only orphan and a HASH-only +orphan are removed by due-read; bump updates only complete entries. Use the +existing Redis Cluster test setup and unique uid/device keys. + +- [ ] **Step 2: Run the tests and verify RED** + +Expected failures: duplicate ZSET members for changed payload, payload parsing +from the member string, and orphan entries remaining. + +- [ ] **Step 3: Replace push and ACK scripts** + +Push atomically executes `ZADD key score user_seq`, `HSET index user_seq payload`, +and paired expiry. ACK atomically executes `ZREM key user_seq` and +`HDEL index user_seq` without depending on payload contents. + +- [ ] **Step 4: Make due-read and bump atomic and self-healing** + +Implement due-read as Lua returning a flat sequence/payload array. For every due +sequence, return it only if the HASH payload exists; otherwise remove the ZSET +member. Remove HASH-only entries encountered by the bounded consistency pass. +Bump scores only when the HASH contains the sequence; otherwise remove the ZSET +member. Renew the paired TTL sample in mutating scripts. + +- [ ] **Step 5: Verify GREEN and regressions** + +Run the new real-Redis suite against the six-node cluster, the existing +DeviceSet/OnlineRoute/Unacked cluster suite, RedisMutex syntax validation, and +`git diff --check`. Expected: all exit zero and no orphan remains. + +- [ ] **Step 6: Commit** + +Commit message: `fix(push): make unacked retries identity-idempotent`. + +### Task 3: Wire RL-05 and PF-09 into CI + +**Files:** +- Modify: `.github/workflows/ci.yml` +- Modify: `docker-compose.yml` +- Modify: `tests/Makefile` +- Create: `tests/pkg/contracts/ci_gates_test.go` + +**Interfaces:** +- Produce workflow job `reliability` invoking `make test-reliability`. +- Produce scheduled workflow job `perf-cache` invoking `make test-perf-cache-gate`. +- Consume Compose variables `TRANSMITE_RATE_LIMIT_USER_MAX` and + `TRANSMITE_RATE_LIMIT_SESSION_MAX` with defaults 600 and 3000. + +- [ ] **Step 1: Write failing workflow contract tests** + +Parse `.github/workflows/ci.yml` and `docker-compose.yml` from Go. Assert that +RL-05 is invoked on pull requests, PF-09 invokes the non-skipping gate on a +schedule, PF-09 supplies both high-limit variables, and Compose maps those +variables to the Transmite flags while retaining defaults 600/3000. + +- [ ] **Step 2: Run contract tests and verify RED** + +Run `go test ./pkg/contracts -run TestCIGates -count=1`. Expected: failure because +the jobs and Compose overrides are absent. + +- [ ] **Step 3: Add dedicated jobs and rate-limit configuration** + +Give each job checkout, Go/protoc setup, full-stack startup, service wait, +protobuf generation, dependency download, its exact Make target, and +`if: always()` teardown. PF-09 sets limits above its generated ten-second load; +normal Compose startup uses 600/3000 defaults. + +- [ ] **Step 4: Verify workflow contracts and test discovery** + +Run the contract test, parse the workflow with a YAML parser, run `make -n +test-reliability` and `make -n test-perf-cache-gate`, and compile the reliability +and perf tagged packages. Expected: no skip-only target is used and all commands +exit zero. + +- [ ] **Step 5: Commit** + +Commit message: `ci: enforce cache reliability and performance gates`. + +### Task 4: Final review and PR update + +**Files:** +- Modify only files required by review findings. + +- [ ] **Step 1: Run branch verification** + +Run fresh protobuf generation, default Go tests, tagged vet/compile checks, +targeted race tests, Redis Cluster harnesses, workflow contract tests, YAML +parsing, and `git diff --check`. + +- [ ] **Step 2: Review the complete branch diff** + +Check requested-change compliance, concurrency correctness, Redis Cluster hash +slot safety, failure handling, and test claims. Fix every Critical or Important +finding and rerun its covering tests. + +- [ ] **Step 3: Push and update GitHub threads** + +Push the reviewed commits. Reply to each inline thread with the concrete fix and +verification, then resolve only threads whose requested behavior is fully met. diff --git a/docs/superpowers/specs/2026-05-18-cache-infrastructure-redesign.md b/docs/superpowers/specs/2026-05-18-cache-infrastructure-redesign.md index 2a8f274..41e846f 100644 --- a/docs/superpowers/specs/2026-05-18-cache-infrastructure-redesign.md +++ b/docs/superpowers/specs/2026-05-18-cache-infrastructure-redesign.md @@ -457,6 +457,21 @@ if (members.size() == 1 && members[0] == "__sentinel__") { } ``` +### 4.2.1 UserInfo 回填代际栅栏 + +UserInfo miss 读取必须同时取得该 uid 的 `observed_generation`，并在调用 +Identity 之前固定下来；RPC 返回后的正值或空标记回填都必须携带这个令牌， +禁止在数据源读取完成后重新读取 generation。单 uid 使用同槽 Lua 比较 +generation 后写入，避免并发资料更新被旧快照覆盖。 + +批量路径按 64 个虚拟 bucket 分组，每组用一次同槽 MGET 同时读取 +`value + generation`。`batch_get` 对每个 miss 返回独立的 observed token， +`batch_set` 接受 `{serialized, observed_generation}`，每个 bucket 用一次 Lua +逐项 CAS 回填。Lua 在任何写入前完整校验所有 key 类型和参数，避免后段 +确定性错误造成同 bucket 部分回填（Redis OOM 等运行时错误不提供事务回滚）。 +单批最多 2000 项，正值和空标记均使用随机 TTL；Redis +不可用或令牌缺失时只返回源数据，不写 L2。 + ### 4.3 防雪崩（Cache Avalanche） **已有设计**：`randomized_ttl()`，见 `2026-05-13-cache-strategy-redesign.md` §2。 @@ -472,6 +487,9 @@ if (members.size() == 1 && members[0] == "__sentinel__") { | L1 Members | 8s | 6.4-9.6s | | L1 UserInfo | 45s | 36-54s | +Push OnlineRoute L1 通过 gflag/Builder 注入，生产默认保持 2s；仅 Compose +可靠性测试栈配置为 30s，以便暂停唯一 Push 实例时稳定复现 Redis 故障。 + --- ## 5. 服务层改动 diff --git a/docs/superpowers/specs/2026-07-15-cache-resilience-completion-design.md b/docs/superpowers/specs/2026-07-15-cache-resilience-completion-design.md new file mode 100644 index 0000000..6d3c186 --- /dev/null +++ b/docs/superpowers/specs/2026-07-15-cache-resilience-completion-design.md @@ -0,0 +1,367 @@ +# ChatNow 缓存韧性补全设计 + +> **日期**: 2026-07-15 +> **基线**: `origin/3.0-dev@1cad99a` +> **关联 Issue**: #35、#37、#38、#45、#48 +> **测试架构**: PR #49 / #54 定义的纯 Go 五层测试体系 + +## 1. 目标与范围 + +本设计补全 ChatNow 3.0 缓存链路在 Redis 故障、高并发缓存回源和集中 +过期场景下的韧性。目标规模沿用现有架构约束：峰值 5000 msg/s、单服务 +2–8 个实例、最大 2000 人群。 + +验收目标： + +- Redis 熔断打开后，调用在 10–50ms 内快速短路，不再逐请求等待 2 秒。 +- Redis 故障期间限流仍生效，不允许无界 fail-open。 +- 缓存型数据可跳过 Redis 回源 RPC/DB；Redis 真相源失败时明确返回不可用。 +- UserInfo 热路径对 Identity RPC 的回源次数降低至少 95%。 +- 同一冷 key 的高并发请求只触发一次进程内回源，不形成 thundering herd。 +- 固定 TTL key 全面应用抖动，避免服务重启后的集中失效。 +- RedisMutex 竞争采用带 jitter 的有界指数退避，降低热点轮询压力。 + +范围包含： + +- `RedisClient` 进程内熔断与快速失败。 +- `RateLimiter` 本地分片令牌桶降级。 +- UserInfo L1/L2/RPC cache-aside 读写路径。 +- Session、Status、Codes、DeviceSet、UnackedPush 等 TTL 抖动补全。 +- RedisMutex 指数退避与 jitter。 +- 与新 Go 测试框架一致的功能、可靠性和性能验证。 +- 与鉴权模块共享 Redis 熔断状态的接口边界。 + +不包含： + +- Redis Cluster 拓扑、代理、服务网格或外部熔断服务改造。 +- 用 DB 或本地状态伪造 SeqGen、UnackedPush 等 Redis 真相源。 +- 完整 JWT 吊销策略重构；#48 的鉴权安全策略单独处理。 +- 跨进程同步熔断状态或本地限流状态。 +- 与缓存问题无关的 DAO、RPC 和部署重构。 + +## 2. 设计原则 + +1. **故障域本地化**：每个进程独立熔断，Redis 故障不占满请求线程和连接池。 +2. **按数据语义降级**：缓存可回源，限流可本地化，真相源必须失败。 +3. **快速路径无锁**：熔断关闭时只做原子状态读取；本地限流按 key 分片。 +4. **标准 cache-aside**：缓存 miss 回源，成功后回填；资料变更后失效。 +5. **有界资源**：本地 bucket、缓存和等待时间均有硬上限。 +6. **不增加常驻后台组件**：恢复探测、bucket 清理都由请求惰性驱动。 +7. **低基数观测**：只记录状态和路径计数，不按 uid、ssid 或 key 打标签。 + +## 3. 总体架构 + +### 3.1 RedisClient 轻量熔断器 + +每个 `RedisClient` 持有一个共享 `RedisCircuitBreaker`。由同一客户端构造的 +Session、Members、RateLimiter、JwtStore 等 DAO 观察同一个状态。 + +状态只有三种： + +- `Closed`：允许请求。连续成功会清空失败计数。 +- `Open`：直接抛出 `RedisCircuitOpen`，不借连接、不访问网络。 +- `HalfOpen`：冷却时间到后，只允许一个请求探测，其余继续快速失败。 + +转换规则： + +1. `Closed` 下连续 3 次连接、I/O、超时或连接池等待异常后进入 `Open`。 +2. `Open` 固定保持 1 秒。 +3. 1 秒后用 CAS 选出一个 `HalfOpen` 探测请求。 +4. 探测成功立即回到 `Closed`；失败重新进入 `Open` 1 秒。 +5. Redis 命令参数、序列化或业务脚本错误不计入熔断失败。 + +固定 1 秒冷却避免引入自适应窗口和复杂配置，同时每个进程最多每秒产生一个 +恢复探测，对 Redis 故障节点负载可忽略。 + +Redis 运行时超时调整为： + +- `connect_timeout = 50ms` +- `socket_timeout = 50ms` +- `pool.wait_timeout = 20ms` + +Redis 部署在同机房；该上限远高于正常亚毫秒至数毫秒延迟，又能在故障时快速 +触发熔断。启动时 Redis Cluster 仍依次尝试所有 seed，不改变现有拓扑发现逻辑。 + +### 3.2 Redis 命令接入 + +`RedisClient` 的公开命令统一通过两个内部模板执行： + +- 有返回值的 `execute(command)`。 +- 无返回值的 `execute_void(command)`。 + +模板负责： + +1. 在借连接前调用 `breaker.before_call()`。 +2. 执行 standalone 或 cluster 命令。 +3. 成功时调用 `breaker.on_success()`。 +4. 仅对连接类异常调用 `breaker.on_failure()`，然后原样抛出。 + +调用方现有的 try/catch 继续负责业务降级。熔断器不返回虚假默认值，防止把 +“Redis 不可用”误解释成“key 不存在”。 + +Pipeline 由 RAII 包装器在 `exec()` 时报告成功或失败；创建 pipeline 本身不视为 +一次成功请求。SCAN 和 Lua 也经过相同入口。 + +### 3.3 按数据语义降级 + +| 数据类型 | Redis 不可用时行为 | +|---|---| +| Members/UserInfo/LastMessage 等缓存 | 跳过 L2，回源 RPC/DB | +| 缓存写入、失效 | 记录指标后忽略，由 TTL 和后续回填收敛 | +| RateLimiter | 使用进程内令牌桶 | +| 幂等 SETNX | 保留现有 DB 唯一约束兜底 | +| SeqGen、UnackedPush 等真相源 | 返回明确的服务不可用 | +| JWT 相关 DAO | 获得统一快速失败信号；具体 fail-open/fail-close 不在本次改变 | + +## 4. 本地限流降级 + +### 4.1 数据结构 + +`LocalRateLimiter` 由 `RateLimiter` 持有，使用 64 个固定 shard。每个 shard 包含： + +- 一个 mutex。 +- `unordered_map`。 +- bucket 数量计数。 + +`Bucket` 只保存 `tokens`、`last_refill` 和 `last_seen`。一次请求只哈希并锁定一个 +shard，不存在全局热锁。 + +### 4.2 算法和边界 + +- 容量、窗口和补充速率与 Redis Lua 令牌桶一致。 +- Redis 正常时只执行 Redis Lua，不双写本地 bucket。 +- Redis 熔断或调用失败时执行本地 bucket。 +- Redis 恢复后下一次请求自动回到 Redis，不迁移本地临时状态。 +- 全进程最多 65,536 个活跃 bucket。 +- bucket 超过两个窗口未访问时，在后续请求中惰性清理。 +- 达到硬上限且无法清理时，不为陌生 key 创建 bucket，直接拒绝请求。 + +故障期间无法维持严格跨实例额度，最坏上限为“实例数 × 单实例额度”，但流量 +始终有界。引入服务发现来动态切分额度会增加一致性和可用性耦合，因此不采用。 + +## 5. UserInfo 三级缓存 + +### 5.1 Key 与 TTL + +- L2 key：`im:user:{bucket}:uid`，`bucket = fnv1a(uid) % 64`。花括号中的 + bucket 是 Redis Cluster hash tag，使批量读取可按 64 个虚拟分片分组，同时避免 + 全部 UserInfo 聚集到一个 slot。 +- L2 value：序列化的 UserInfo protobuf。 +- L2 正值 TTL：1 小时，±20% 抖动。 +- L1 正值 TTL：45 秒，±20% 抖动。 +- 空值 sentinel TTL：5 秒，使用相同抖动函数。 + +### 5.2 单用户读路径 + +1. 查询 Transmite 现有 `LocalCache`。 +2. L1 命中时解析 protobuf 并返回。 +3. L1 miss 后进入现有 `InflightRegistry` 的 uid 维度 singleflight。 +4. 获得 leader 后 double-check L1。 +5. 查询 `UserInfoCache` L2。 +6. L2 命中则回填 L1；损坏值删除后按 miss 处理。 +7. L2 miss 或熔断时调用 Identity `GetProfile`。 +8. RPC 成功后写 L2 和 L1；确认用户不存在时写 5 秒 sentinel。 +9. RPC/DB 失败不写 sentinel，避免把依赖故障缓存成“不存在”。 + +Transmite 当前热路径只读取发送者一个 uid，不额外引入没有消费者的批量业务 +流程。 + +### 5.3 批量接口 + +`UserInfoCache` 提供 `batch_get` 和 `batch_set`，满足 Message、Conversation 后续 +批量读取场景： + +- 读取使用 MGET；写入逐 uid 读取 generation 后以同槽 Lua CAS 回填，避免公开 + `set`/`batch_set` 绕过失效 fence。这里不额外引入复杂的批量 Lua 协议。 +- Redis Cluster 按 64 个虚拟 bucket 分组；每组 key 共享 hash tag，可安全使用 + MGET/pipeline，避免 CROSSSLOT，也不依赖 redis-plus-plus 的内部连接池。 +- 返回命中 map 和 miss uid 列表，调用方可用现有批量 RPC 一次回源。 +- 单次批量大小限制为 2000，与最大群规模一致。 + +本次只把单用户 Transmite 路径接入该 DAO；批量消费者迁移不在范围内。 + +### 5.4 失效 + +Identity 在 DB 提交前 best-effort 推进 generation 并删除 L2，DB 成功后再次执行 +同一原子失效。夹在 pre-invalidate 与 commit 之间读取旧 DB 的回填会被 post +generation 删除或拒绝；任一 Redis 失效都只记录指标，不拒绝或回滚已经请求的资料 +更新。若 Redis 在整个更新期间都不可用，只能依赖 L1 最多约 54 秒、L2 一小时 TTL、 +告警和恢复后的后续失效收敛，这是 cache-aside 可用性优先的明确边界，不引入 DB +outbox。Transmite 无法读取 generation 时仍把成功 RPC 结果放入短 TTL L1，供本机 +singleflight followers 共享，但不写 L2；not-found 仍必须拿到 generation 才写 sentinel。 + +## 6. TTL 抖动补全 + +统一使用现有 `randomized_ttl(base)`，范围为基准值 ±20%，且永远返回正值。 + +需要补全的路径： + +- Session：append、touch。 +- Status：append、touch。 +- Codes：append。 +- DeviceSet：add 时设置与 Session 相同的 7 天 TTL；设备活动路径刷新 TTL。 +- UnackedPush：push、bump_score 涉及的 ZSET/HASH 使用同一个随机 TTL 样本，避免 + 两个索引因不同抖动提前分离。 +- 其他仍使用固定常量的缓存写入和续期点。 + +DeviceSet 不使用短在线 TTL。它的生命周期与登录设备接近；7 天 TTL 防止永久 +常驻，同时避免正常在线设备被几分钟级 TTL 误删。 + +滚动升级期间，带 hash tag 的 `im:dev:{uid}` 是新权威 key。读取会额外读取旧版 +`im:dev:uid`，合并成员并懒迁移到新 key；删除同时清理两边。旧 key 不再接收新写， +每次被观察时只续一个最长 24 小时的迁移 grace TTL，因此不会形成无限双写或永久 +兼容负担。跨 slot 的迁移分步执行是有意选择：迁移可重试，在线路由正确性仍由同 +slot 的新 key 与 `im:online:{uid}` 原子脚本保证。 + +## 7. RedisMutex 退避 + +`try_lock` 保持现有 `SET key token NX PX ttl` 和 Lua CAS 解锁协议，仅调整竞争等待： + +- 首次失败等待基准 5ms。 +- 后续基准依次为 10ms、20ms，之后保持 20ms 上限。 +- 每次加入 `[0, base/2]` 的 thread-local 随机 jitter。 +- sleep 不超过剩余 timeout。 +- 每次 `try_lock` 调用重置退避状态。 + +100ms 默认超时下最多产生少量 Redis 轮询，且竞争者不再同频唤醒。 + +## 8. 错误处理与恢复 + +- 熔断快速拒绝使用独立异常类型，便于 RateLimiter 精确进入本地降级。 +- DAO 不记录每次快速拒绝，只在熔断状态转换时写日志，避免 Redis 故障造成日志 + 风暴。 +- cache-aside 回源失败时返回原业务依赖错误，不把失败响应写入缓存。 +- half-open 探测由真实业务命令完成，不另起 ping 线程。 +- Redis 恢复不要求服务重启；第一个成功探测关闭熔断器。 +- Redis Cluster 的节点级 failover 仍由 redis-plus-plus 处理；应用熔断只覆盖客户端 + 已无法在时限内完成命令的场景。 + +## 9. 可观测性 + +在现有 bvar 指标中增加低基数计数器： + +- `redis_circuit_open_total` +- `redis_circuit_rejected_total` +- `redis_circuit_recovered_total` +- `redis_call_failure_total` +- `rate_limit_local_fallback_total` +- `rate_limit_local_rejected_total` +- `user_info_l1_hit_total` +- `user_info_l2_hit_total` +- `user_info_rpc_total` +- `redis_mutex_retry_total` + +状态转换日志包含 Closed/Open/HalfOpen 和异常分类，不包含 uid、ssid 或完整 Redis +key。现有 Prometheus Redis 告警继续使用；新增应用指标用于区分 Redis 故障和业务 +错误。 + +## 10. 测试设计 + +测试遵循 PR #49/#54：纯 Go、真实全栈、黑盒行为验证，测试代码是权威，不新增 +C++ gtest 或 Python 合约测试。 + +### 10.1 L2 功能测试 + +文件：`tests/func/cache_test.go`，build tag：`func`。 + +- `FN-CA-01`：首次 UserInfo 回源后存在 L2，重复请求不重复调用 Identity。 +- `FN-CA-02`：并发请求同一 uid，只发生一次回源。 +- `FN-CA-03`：资料修改后 L2 失效，下一次读取获得新值。 +- `FN-CA-04`：Session、Status、Codes、DeviceSet、UnackedPush TTL 位于抖动范围， + 多个样本不过期于同一秒。 +- `FN-CA-05`：Redis 正常时大量消息请求触发分布式限流。 + +Session 与 Status DAO 在当前 3.0-dev 没有生产调用点，不为测试增加无业务意义的 +endpoint。它们的 TTL 源码契约由独立 contract test 覆盖；Codes、DeviceSet、 +UnackedPush 等可达路径继续由新框架做黑盒验证。待业务重新接入前两者时，再把对应 +断言提升为黑盒测试。 + +### 10.2 可靠性测试 + +文件：`tests/reliability/redis_failover_test.go`，build tag：`reliability`。 + +`RL-05` 使用 `tests/pkg/chaos` 停止和恢复 Redis： + +1. 故障前创建唯一用户和业务数据并预热缓存。 +2. 停止 Redis，触发各目标服务熔断。 +3. 验证缓存型接口可回源，稳定故障期请求延迟不超过 50ms。 +4. 验证突发请求出现本地限流拒绝。 +5. 验证依赖 SeqGen 的路径明确返回不可用。 +6. 恢复 Redis，验证 half-open 成功并自动恢复正常路径。 +7. Push 专项通过真实 WS、Rabbit 和容器 pause 编排验证：Redis 故障时 Unacked 写入 + 失败必须 NackRequeue 且不得先向 WS 投递；恢复后重投、持久化并按 at-least-once + 语义最终送达。 + +### 10.3 L4 性能测试 + +文件：`tests/perf/cache_test.go`，build tag：`perf`。 + +- `PF-09` 分别记录冷缓存、L2 命中和 L1 命中的吞吐与 p95。 +- 目标负载为 5000 msg/s，报告分配量和每请求耗时。 +- 同一 uid 的热路径 Identity RPC 降幅至少 95%。 +- 200 个并发请求访问同一冷 key 时，只允许一次进程内 RPC 回源。 +- nightly 基线吞吐下降超过 10% 时失败。 +- 性能门禁仅在 `PF09_RUN_FULLSTACK=1` 的 Linux 完整服务环境启用；普通编译发现 + 明确 Skip，不把缺少业务栈伪装成性能通过。cold、L2 各使用 20 个 fresh sender， + 每个 sender 在计时区间只请求一次；L2 在计时前向真实 Redis Cluster 原子写入从 + Identity 读取的 UserInfo。L1 使用另外 20 个 sender，通过真实发送路径预热后固定 + 压测 10 秒；预热按声明的 Transmite 实例数连续发送，利用 Gateway 的 round-robin + 为每个实例填充 L1，保证三个计时区间不会因 key 复用或跨实例路由相互转化。 +- 多 Transmite 实例压测通过 `PF09_TRANSMITE_VARS_URLS` 提供所有 bvar 地址并求和， + 并用 `PF09_EXPECTED_TRANSMITE_INSTANCES` 校验实例数。前后快照同时记录 pid、uptime、 + 启动时刻估值和 L1/L2/RPC counter，拒绝实例重启、counter 回绕、遗漏和求和溢出； + 每阶段 counter 必须精确符合对应缓存路径。门禁固定 `-benchtime=1x`，内部 10 秒时长 + 不可由环境变量修改；使用代码库内 5000 msg/s 基线，基线可向上调整但不可降低。 + +### 10.4 测试辅助设施 + +- `tests/pkg/verify/redis.go`：读取 key、TTL 和 bvar 指标，不包含业务断言。 +- `tests/pkg/chaos/redis.go`：Redis stop/start/wait 封装。 +- 所有用例使用唯一 ID 隔离，可在 PR #49 合并后直接接入 CleanupAll 和 CI。 +- macOS 执行 C++ 构建、Go build/vet/gofmt；故障注入和性能测试在 Linux Docker/CI + 执行。 + +## 11. 文件边界 + +计划新增： + +- `common/utils/redis_circuit_breaker.hpp`：熔断状态机。 +- `common/utils/local_rate_limiter.hpp`：分片本地令牌桶。 +- `tests/func/cache_test.go`。 +- `tests/reliability/redis_failover_test.go`。 +- `tests/perf/cache_test.go`。 +- `tests/pkg/verify/redis.go`。 +- `tests/pkg/chaos/redis.go`。 + +计划修改： + +- `common/dao/data_redis.hpp`：RedisClient 接入、UserInfoCache、TTL、RateLimiter。 +- `common/utils/redis_mutex.hpp`：指数退避和 jitter。 +- `common/infra/metrics.hpp`：低基数指标。 +- `common/utils/redis_keys.hpp`：UserInfo L2 key。 +- `transmite/source/transmite_server.h`：UserInfo 三级缓存读路径。 +- `identity/source/identity_server.h`：资料变更后失效 UserInfo L2。 +- 必要的服务 builder/config 文件：构造并注入共享 DAO，不新增外部依赖。 + +每个新增组件只有一个职责；不将熔断、限流和业务缓存合并成大型基础设施类。 + +## 12. 发布与回滚 + +1. 先合入无行为变化的指标和组件。 +2. 接入 RedisClient 熔断与 RateLimiter 降级。 +3. 接入 UserInfo cache-aside 和资料失效。 +4. 补全 TTL 与锁退避。 +5. Linux CI 运行 func、reliability 和 perf；观察熔断误触发与 RPC 降幅。 + +回滚可以按组件提交逐步进行。熔断器和本地限流没有外部持久状态；关闭相关接入 +即可恢复原行为。UserInfo L2 key 使用独立前缀，回滚后由 TTL 自动清理。 + +## 13. Issue 覆盖 + +| Issue | 解决点 | +|---|---| +| #35 | RedisClient 熔断、50ms 超时、缓存回源语义 | +| #37 | UserInfo L1/L2/RPC、singleflight、批量 DAO | +| #38 | 固定 TTL 全面抖动、DeviceSet 生命周期 | +| #45 | RedisMutex 指数退避与 jitter | +| #48 | 统一熔断接口、本地限流降级；JWT 策略明确排除 | diff --git a/docs/superpowers/specs/2026-07-16-pr57-ci-artifacts-native-regressions-design.md b/docs/superpowers/specs/2026-07-16-pr57-ci-artifacts-native-regressions-design.md new file mode 100644 index 0000000..26ff0f5 --- /dev/null +++ b/docs/superpowers/specs/2026-07-16-pr57-ci-artifacts-native-regressions-design.md @@ -0,0 +1,94 @@ +# PR #57 CI Artifacts and Go Regressions Design + +## Scope + +Address the two review threads added after commit `f633006`: + +1. make the RL-05 and PF-09 jobs start a runnable service stack from a clean + GitHub runner; +2. execute generation-fence and Unacked-ledger business regressions through the + repository's Go test framework. + +The change must not hide a missing native toolchain, silently skip a test, or +duplicate a full service build in every gate job. + +## Service artifact pipeline + +A single producer job builds the native services in a reproducible Linux +builder and packages the exact directory contract consumed by Compose: + +```text +compose-artifacts/ + identity/build/identity_server + identity/depends/*.so* + ... + transmite/build/transmite_server + transmite/depends/*.so* +``` + +The producer uses a repository-owned builder definition with pinned dependency +revisions and GitHub Actions layer caching. It performs one root CMake build, +then a packaging script copies `build//_server` into each +service build context and obtains the non-system shared-library closure from +`ldd`. Packaging fails if any of the nine binaries is missing or if `ldd` +reports an unresolved dependency. + +The producer uploads one immutable artifact for the workflow run. The +`reliability` and `perf-cache` consumers declare `needs: service-artifacts`, +download it before `docker compose up`, validate its manifest, and then run the +existing Go gates. A failed producer blocks consumers instead of producing a +misleading integration-test failure. + +## Go regression coverage + +The unified test architecture is authoritative: tests are Go, exercise +HTTP/protobuf or WebSocket service boundaries, and run through targets in +`tests/Makefile`. This PR does not introduce CTest or retain temporary C++ +regression executables. + +The Unacked regression calls the real Push service twice with the same +`user_seq` and different notification payloads after establishing an online +device route. It verifies through Redis Cluster that the ZSET retains one +stable identity and the HASH contains the second payload, then sends a real +WebSocket ACK and verifies both indexes are removed. + +The UserInfo regression remains a full-stack cache-consistency scenario. It +invalidates shared cache state through UpdateProfile, drives the Transmite +lookup path, and verifies that the repopulated Redis value contains the latest +profile after the documented bounded process-local L1 lifetime. The test +asserts externally observable freshness; it does not add a production timing +hook solely to force an internal CAS interleaving. + +RL-05 and PF-09 remain the authoritative system gates. + +## Contract enforcement + +The existing Go workflow contract tests will require: + +- exactly one service-artifact producer; +- build, package, validate, and upload steps in order; +- both gate jobs to depend on the producer and download/validate before + Compose startup; +- the Go cache and Unacked regressions to execute through an existing Make + target against the stack; +- no `continue-on-error`, conditional bypass, or skip-capable dependency + detection on required steps; +- teardown to remain the unique final step with `if: always()`. + +## Failure behavior + +- Missing compiler dependency: builder construction or CMake configuration + fails. +- Missing binary: packaging fails before upload. +- Unresolved shared library: artifact validation fails before upload and again + after download. +- Redis Cluster unavailable: the Go Unacked regression fails; it is never + reported as skipped. +- Gate failure: job fails and teardown still runs. + +## Non-goals + +- no refactor of all nine runtime Dockerfiles into multi-stage builds; +- no externally managed or mutable CI image dependency; +- no CTest or revival of the old C++ test suite; +- no change to cache, ACK, or message-watermark business semantics. diff --git a/docs/superpowers/specs/2026-07-16-pr57-review-fixes-design.md b/docs/superpowers/specs/2026-07-16-pr57-review-fixes-design.md new file mode 100644 index 0000000..055c304 --- /dev/null +++ b/docs/superpowers/specs/2026-07-16-pr57-review-fixes-design.md @@ -0,0 +1,71 @@ +# PR #57 Review Fixes Design + +## Scope + +Address the three unresolved requested-change threads on PR #57 without +expanding into the delivery-ACK redesign tracked by issue #58: + +1. prevent a failed UserInfo generation CAS from publishing stale data to L1; +2. make per-device Unacked storage idempotent by identity rather than payload; +3. turn RL-05 and PF-09 into real CI gates. + +The project is pre-release. The Unacked Redis layout may change without a +compatibility reader or migration. + +## UserInfo generation fencing + +`UserInfoCache::set_if_generation` returns a three-state result: + +- `Committed`: Redis generation matched and L2 was written; +- `Conflict`: Redis was reachable but the generation no longer matched; +- `Unavailable`: no Redis client, an open circuit, or a Redis exception. + +Transmite may publish an Identity result to L1 after `Committed`. It must not +publish after `Conflict`. After `Unavailable`, it may publish only to the +existing short-lived L1 fallback so singleflight followers share the successful +Identity response while Redis is down. The current request may return its +Identity result in all three states. + +## Unacked Pending Ledger + +The two Redis keys have separate responsibilities and share a cluster hash tag: + +```text +ZSET im:unack:{uid:device} member=user_seq, score=next_retry_at +HASH im:unack:idx:{uid:device} field=user_seq, value=payload_b64 +``` + +Lua scripts atomically maintain both keys: + +- push uses `ZADD` and `HSET`; retrying the same `user_seq` replaces its score + and payload without creating another member; +- peek selects due sequence IDs and returns payloads from the HASH in one + script; incomplete entries are removed from whichever side remains; +- bump updates scores only for IDs that still have payloads and removes ZSET + orphans; +- ack always removes the sequence ID from both structures. + +There is no parser or migration for the former `user_seq:payload` member format. + +## CI gates + +RL-05 runs in a dedicated `reliability` job for pull requests and schedules. +PF-09 runs in a dedicated scheduled `perf-cache` job through +`make test-perf-cache-gate`. The Compose Transmite command accepts environment +overrides for user and session rate limits; PF-09 supplies limits above its +generated load while normal Compose defaults remain 600 and 3000 per minute. +Each full-stack job owns setup and `if: always()` teardown. + +## Verification + +- generation conflict, committed, and unavailable policy tests; +- real Redis Unacked overwrite, due-read, orphan repair, bump, and ACK tests; +- workflow/Compose contract tests proving both targets and rate-limit overrides + are wired; +- existing Go vet/compile checks and the cache/reliability helper suites. + +## Non-goals + +- no compatibility with old Unacked Redis data; +- no change to `last_read_seq`, `last_ack_seq`, or issue #58; +- no Redis Streams, new broker, or additional cache tier. diff --git a/gateway/Dockerfile b/gateway/Dockerfile index e059354..9a1473e 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/gateway_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 -CMD /im/bin/gateway_server -flagfile=/im/conf/gateway_server.conf \ No newline at end of file +CMD /im/bin/gateway_server -flagfile=/im/conf/gateway_server.conf diff --git a/identity/Dockerfile b/identity/Dockerfile index 94a6007..c26649a 100644 --- a/identity/Dockerfile +++ b/identity/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/identity_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y ca-certificates netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 -CMD /im/bin/identity_server -flagfile=/im/conf/identity_server.conf \ No newline at end of file +CMD /im/bin/identity_server -flagfile=/im/conf/identity_server.conf diff --git a/identity/source/identity_server.h b/identity/source/identity_server.h index c104564..43ccb85 100644 --- a/identity/source/identity_server.h +++ b/identity/source/identity_server.h @@ -39,6 +39,7 @@ class IdentityServiceImpl : public ::chatnow::identity::IdentityService IdentityServiceImpl(const std::shared_ptr &mysql_client, const std::shared_ptr &es_client, const RedisClient::ptr &redis_client, + const UserInfoCache::ptr &user_info_cache, const std::shared_ptr &mail_client, const std::shared_ptr &jwt_codec, const std::shared_ptr &jwt_store, @@ -48,6 +49,7 @@ class IdentityServiceImpl : public ::chatnow::identity::IdentityService : _mysql_user(std::make_shared(mysql_client)), _es_user(std::make_shared(es_client)), _redis_codes(std::make_shared(redis_client)), + _user_info_cache(user_info_cache), _mail_client(mail_client), _jwt_codec(jwt_codec), _jwt_store(jwt_store), @@ -474,10 +476,17 @@ class IdentityServiceImpl : public ::chatnow::identity::IdentityService if (request->has_phone()) { user->phone(request->phone()); } + // Pre-invalidate fences a reader that races before the DB commit. + // It is best-effort: profile writes remain available during a Redis outage. + if (_user_info_cache) (void)_user_info_cache->invalidate(auth.user_id); if (!_mysql_user->update(user)) { throw ServiceError(::chatnow::error::kSystemInternalError, "db update failed"); } + // A reader between pre-invalidate and commit may have cached the old DB + // row under the new generation. Advance once more after commit so that + // fill is rejected/deleted without rolling back an already durable DB write. + if (_user_info_cache) (void)_user_info_cache->invalidate(auth.user_id); std::string ob_payload = outbox_payload_upsert_( user->user_id(), user->mail(), user->phone(), user->nickname(), user->description(), @@ -532,6 +541,7 @@ class IdentityServiceImpl : public ::chatnow::identity::IdentityService std::shared_ptr _mysql_user; std::shared_ptr _es_user; std::shared_ptr _redis_codes; + UserInfoCache::ptr _user_info_cache; std::shared_ptr _mail_client; std::shared_ptr _jwt_codec; std::shared_ptr _jwt_store; @@ -701,6 +711,7 @@ class IdentityServerBuilder _redis_client = std::make_shared(redis); } _es_outbox = std::make_shared(_redis_client, key::es_outbox_key("identity")); + _user_info_cache = std::make_shared(_redis_client); } /* brief: 加载 JWT 配置并构造 codec / store（必须在 make_redis_object 之后） */ void make_jwt_object(const std::string &auth_config_path) { @@ -781,7 +792,7 @@ class IdentityServerBuilder auto es_reaper_client = ESClientFactory::create(_es_hosts); IdentityServiceImpl *identity_service = new IdentityServiceImpl( - _mysql_client, _es_client, _redis_client, _mail_client, + _mysql_client, _es_client, _redis_client, _user_info_cache, _mail_client, _jwt_codec, _jwt_store, _media_public_url_prefix, _es_outbox, es_reaper_client); _service_impl = identity_service; @@ -826,6 +837,7 @@ class IdentityServerBuilder std::shared_ptr _mysql_client; std::string _redis_seeds; RedisClient::ptr _redis_client; + UserInfoCache::ptr _user_info_cache; std::shared_ptr _mail_client; std::string _media_public_url_prefix; diff --git a/media/CMakeLists.txt b/media/CMakeLists.txt index 28250ca..38855f8 100644 --- a/media/CMakeLists.txt +++ b/media/CMakeLists.txt @@ -62,6 +62,7 @@ set(src_files ${CMAKE_CURRENT_SOURCE_DIR}/source/download_handler.hpp ${CMAKE_CURRENT_SOURCE_DIR}/source/speech_handler.hpp ${CMAKE_CURRENT_SOURCE_DIR}/source/cleanup_worker.hpp) +find_package(jsoncpp CONFIG REQUIRED) add_executable(${target} ${src_files} ${proto_srcs} ${odb_srcs}) target_link_libraries(${target} @@ -71,7 +72,7 @@ target_link_libraries(${target} -lodb-mysql -lodb -lodb-boost -lredis++ -lhiredis -laws-cpp-sdk-s3 -laws-cpp-sdk-core - /usr/local/lib/libjsoncpp.so.19) + jsoncpp_lib) # 5. 测试可执行（gtest 单元 + 集成） set(test_files @@ -86,7 +87,7 @@ set(test_files # -lodb-mysql -lodb -lodb-boost # -lredis++ -lhiredis # -laws-cpp-sdk-s3 -laws-cpp-sdk-core -# /usr/local/lib/libjsoncpp.so.19) +# jsoncpp_lib) # 6. 头文件搜索路径 target_include_directories(${target} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/media/Dockerfile b/media/Dockerfile index ea9a789..ae3c167 100644 --- a/media/Dockerfile +++ b/media/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/media_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 -CMD /im/bin/media_server -flagfile=/im/conf/media_server.conf \ No newline at end of file +CMD /im/bin/media_server -flagfile=/im/conf/media_server.conf diff --git a/message/Dockerfile b/message/Dockerfile index 0973192..f01876a 100644 --- a/message/Dockerfile +++ b/message/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/message_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 -CMD /im/bin/message_server -flagfile=/im/conf/message_server.conf \ No newline at end of file +CMD /im/bin/message_server -flagfile=/im/conf/message_server.conf diff --git a/message/source/message_server.h b/message/source/message_server.h index a0f1bae..e3b2a32 100644 --- a/message/source/message_server.h +++ b/message/source/message_server.h @@ -450,23 +450,13 @@ class MessageServiceImpl : public chatnow::message::MessageService { brpc::ClosureGuard done_guard(done); auto* cntl = static_cast(base_cntl); HANDLE_RPC(cntl, req, rsp, { - if (req->message_id() == 0) { + if (req->seq_id() == 0) { throw ::chatnow::ServiceError(::chatnow::error::kSystemInvalidArgument, - "message_id required"); + "seq_id required"); } - auto msg = _mysql_msg->select_by_id( - static_cast(req->message_id())); - if (!msg || msg->session_id() != req->conversation_id()) { - throw ::chatnow::ServiceError(::chatnow::error::kMessageNotFound, - "ack message not found"); - } - uint64_t session_seq = resolve_ack_session_seq(msg->seq_id()); - if (session_seq == 0) - throw ::chatnow::ServiceError(::chatnow::error::kSystemInvalidArgument, - "message seq_id required"); require_member_(req->conversation_id(), auth.user_id); bool ok = _mysql_member->update_last_ack_seq( - req->conversation_id(), auth.user_id, session_seq); + req->conversation_id(), auth.user_id, req->seq_id()); if (!ok) throw ::chatnow::ServiceError(::chatnow::error::kSystemInternalError, "update last_ack_seq failed"); diff --git a/presence/Dockerfile b/presence/Dockerfile index bd43bb1..71af26a 100644 --- a/presence/Dockerfile +++ b/presence/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/presence_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 CMD /im/bin/presence_server -flagfile=/im/conf/presence_server.conf diff --git a/presence/source/presence_server.h b/presence/source/presence_server.h index 220d67d..ef02b5d 100644 --- a/presence/source/presence_server.h +++ b/presence/source/presence_server.h @@ -13,6 +13,7 @@ #include "error/error_codes.hpp" #include "error/service_error.hpp" #include "utils/brpc_closure.hpp" +#include "utils/random_ttl.hpp" #include "common/types.pb.h" #include "common/error.pb.h" @@ -292,7 +293,8 @@ class PresenceServiceImpl : public PresenceService { std::chrono::system_clock::now().time_since_epoch()).count(); _redis->sadd("im:presence:typing:" + conv_id, auth.user_id + ":" + std::to_string(now_ms)); - _redis->expire("im:presence:typing:" + conv_id, std::chrono::seconds(5)); + _redis->expire("im:presence:typing:" + conv_id, + randomized_ttl(std::chrono::seconds(5))); } else { std::vector members; _redis->smembers("im:presence:typing:" + conv_id, diff --git a/proto/message/message_service.proto b/proto/message/message_service.proto index b7daa39..b017ef2 100644 --- a/proto/message/message_service.proto +++ b/proto/message/message_service.proto @@ -150,6 +150,7 @@ message SelectByClientMsgIdRsp { message UpdateReadAckReq { string request_id = 1; string conversation_id = 2; - uint64 message_id = 3; + // 会话内单调递增的送达确认水位；服务端只允许向前推进。 + uint64 seq_id = 3; } message UpdateReadAckRsp { chatnow.common.ResponseHeader header = 1; } diff --git a/proto/push/notify.proto b/proto/push/notify.proto index dfa0cc0..9eb9a19 100644 --- a/proto/push/notify.proto +++ b/proto/push/notify.proto @@ -34,13 +34,15 @@ message NotifyClientAuth { } // Client ACKs a delivered push item. user_seq identifies the per-device -// unacked entry; message_id is the only source used to advance conversation ACK. +// unacked entry and message_id identifies the delivered item. seq_id is an +// optional conversation watermark; non-message pushes leave it as zero. message NotifyMsgPushAck { string user_id = 1; string device_id = 2; int64 message_id = 3; uint64 user_seq = 4; string conversation_id = 5; + uint64 seq_id = 6; } message NotifyHeartbeat { diff --git a/push/Dockerfile b/push/Dockerfile index 2586f36..103eb1c 100644 --- a/push/Dockerfile +++ b/push/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/push_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 CMD /im/bin/push_server -flagfile=/im/conf/push_server.conf diff --git a/push/source/push_server.cc b/push/source/push_server.cc index 78acdce..17247ff 100644 --- a/push/source/push_server.cc +++ b/push/source/push_server.cc @@ -35,6 +35,7 @@ DEFINE_string(mq_push_binding_key, "push", "推送绑定键"); // M5: 心跳触发未 ack 重传的可调参数 DEFINE_int32(resend_batch, 50, "心跳触发未 ack 重传的批量上限"); DEFINE_int32(resend_max_age_sec, 5, "未 ack 项入队后等待多少秒视为可重传"); +DEFINE_int32(route_l1_ttl_sec, 2, "Push 在线路由 L1 TTL（1-300 秒）"); // JWT — 统一从 auth.json 加载（与 identity/gateway 共享密钥源） DEFINE_string(auth_config, "/im/conf/auth.json", "JWT 鉴权配置文件路径(JSON)"); @@ -55,6 +56,7 @@ int main(int argc, char *argv[]) psb.make_discovery_object(FLAGS_registry_host, FLAGS_base_service, FLAGS_message_service, FLAGS_push_service); psb.make_reg_object(FLAGS_registry_host, FLAGS_base_service + FLAGS_instance_name, FLAGS_access_host); psb.set_resend_params(FLAGS_resend_batch, FLAGS_resend_max_age_sec); + psb.set_route_l1_ttl(FLAGS_route_l1_ttl_sec); psb.set_etcd_client(std::make_shared(FLAGS_registry_host)); psb.set_push_service_dir(FLAGS_base_service + FLAGS_push_service); psb.make_cross_reaper_election(); diff --git a/push/source/push_server.h b/push/source/push_server.h index 7c8afa2..7c46c12 100644 --- a/push/source/push_server.h +++ b/push/source/push_server.h @@ -41,6 +41,8 @@ #include #include #include +#include +#include #include namespace chatnow::push { @@ -64,7 +66,8 @@ class PushServiceImpl : public PushService const ServiceManager::ptr &channels, LeaderElection::ptr cross_reaper_election = nullptr, LocalCache::ptr local_route_cache = nullptr, - InflightRegistry::ptr inflight_registry = nullptr) + InflightRegistry::ptr inflight_registry = nullptr, + std::chrono::seconds route_l1_ttl = std::chrono::seconds(2)) : _connections(connections), _jwt_codec(jwt_codec), _redis(redis), @@ -76,7 +79,12 @@ class PushServiceImpl : public PushService _mm_channels(channels), _cross_reaper_election(std::move(cross_reaper_election)), _local_route_cache(std::move(local_route_cache)), - _inflight_registry(std::move(inflight_registry)) {} + _inflight_registry(std::move(inflight_registry)), + _route_l1_ttl(route_l1_ttl) { + if (_route_l1_ttl.count() < 1 || _route_l1_ttl.count() > 300) { + throw std::invalid_argument("Push route L1 TTL must be within 1..300 seconds"); + } + } void set_resend_params(long batch, long max_age_sec) { _resend_batch = batch; @@ -124,24 +132,36 @@ class PushServiceImpl : public PushService payload = notify.SerializeAsString(); } - int delivered = 0; auto route = resolve_route(request->user_id()); - for (const auto &did : route.device_ids) { - if (filter_devices && target_dids.find(did) == target_dids.end()) continue; - if (_local_send(request->user_id(), did, payload) > 0) ++delivered; - } - - if (request->has_user_seq() && _unacked) { + // Persist before delivery. On failure the RPC is retryable and no + // client has observed a payload without durable retransmit state. + if (request->has_user_seq()) { std::string payload_b64 = _utils_base64_encode(payload); long long now_ts = static_cast(time(nullptr)); for (const auto &did : route.device_ids) { if (filter_devices && target_dids.find(did) == target_dids.end()) continue; - _unacked->push(request->user_id(), did, - request->user_seq(), payload_b64, now_ts); + persist_unacked_(request->user_id(), did, request->user_seq(), + payload_b64, now_ts); } } + int delivered = 0; + for (const auto &did : route.device_ids) { + if (filter_devices && target_dids.find(did) == target_dids.end()) continue; + if (_local_send(request->user_id(), did, payload) > 0) ++delivered; + } + response->set_online_device_count(delivered); + } catch (const RedisCircuitOpen& e) { + response->mutable_header()->set_success(false); + response->mutable_header()->set_error_code(::chatnow::error::kSystemUnavailable); + response->mutable_header()->set_error_message("unacked persistence unavailable"); + cntl->SetFailed(e.what()); + } catch (const sw::redis::Error& e) { + response->mutable_header()->set_success(false); + response->mutable_header()->set_error_code(::chatnow::error::kSystemUnavailable); + response->mutable_header()->set_error_message("unacked persistence unavailable"); + cntl->SetFailed(e.what()); } catch (const ::chatnow::ServiceError& e) { response->mutable_header()->set_success(false); response->mutable_header()->set_error_code(e.code()); @@ -192,14 +212,24 @@ class PushServiceImpl : public PushService } for (const auto &did : route.device_ids) { - if (_local_send(uid, did, payload) > 0) ++total; - if (it != uid2seq.end() && _unacked) { - _unacked->push(uid, did, it->second, - _utils_base64_encode(payload), now_ts); + if (it != uid2seq.end()) { + persist_unacked_(uid, did, it->second, + _utils_base64_encode(payload), now_ts); } + if (_local_send(uid, did, payload) > 0) ++total; } } response->set_online_count(total); + } catch (const RedisCircuitOpen& e) { + response->mutable_header()->set_success(false); + response->mutable_header()->set_error_code(::chatnow::error::kSystemUnavailable); + response->mutable_header()->set_error_message("unacked persistence unavailable"); + cntl->SetFailed(e.what()); + } catch (const sw::redis::Error& e) { + response->mutable_header()->set_success(false); + response->mutable_header()->set_error_code(::chatnow::error::kSystemUnavailable); + response->mutable_header()->set_error_message("unacked persistence unavailable"); + cntl->SetFailed(e.what()); } catch (const ::chatnow::ServiceError& e) { response->mutable_header()->set_success(false); response->mutable_header()->set_error_code(e.code()); @@ -239,7 +269,16 @@ class PushServiceImpl : public PushService std::vector remote_uids; remote_uids.reserve(internal_msg.member_id_list_size()); for (const auto &uid : internal_msg.member_id_list()) { - auto route = resolve_route(uid); + RouteEntry route; + try { + route = resolve_route(uid); + } catch (const RedisCircuitOpen &e) { + LOG_WARN("Push-Consumer: route circuit unavailable: {}", e.what()); + return requeue_(); + } catch (const sw::redis::Error &e) { + LOG_WARN("Push-Consumer: route Redis unavailable: {}", e.what()); + return requeue_(); + } if (route.device_ids.empty()) { remote_uids.push_back(uid); continue; } auto itu = uid2seq.find(uid); @@ -259,11 +298,23 @@ class PushServiceImpl : public PushService std::string inst = (it != route.device_to_instance.end()) ? it->second : ""; if (inst == _instance_id) { if (itu != uid2seq.end()) { - if (_local_send(uid, did, user_payload) > 0) any_local = true; - if (_unacked) { - _unacked->push(uid, did, itu->second, - _utils_base64_encode(user_payload), now_ts); + try { + persist_unacked_(uid, did, itu->second, + _utils_base64_encode(user_payload), now_ts); + } catch (const RedisCircuitOpen &e) { + LOG_WARN("Push-Consumer: unacked circuit unavailable: {}", e.what()); + return requeue_(); + } catch (const sw::redis::Error &e) { + LOG_WARN("Push-Consumer: unacked Redis unavailable: {}", e.what()); + return requeue_(); + } catch (const ::chatnow::ServiceError &e) { + LOG_WARN("Push-Consumer: unacked unavailable: {}", e.what()); + return requeue_(); + } catch (const std::exception &e) { + LOG_WARN("Push-Consumer: unacked persistence failed: {}", e.what()); + return requeue_(); } + if (_local_send(uid, did, user_payload) > 0) any_local = true; } else { // 大群读扩散：无 user_seq，仅下发 _local_send(uid, did, notify_template.SerializeAsString()); @@ -279,7 +330,14 @@ class PushServiceImpl : public PushService // 2) 跨实例：按 Push 实例 ID 分组 std::unordered_map> peer_to_uids; for (const auto &uid : remote_uids) { - auto route = resolve_route(uid); + RouteEntry route; + try { + route = resolve_route(uid); + } catch (const RedisCircuitOpen &) { + return requeue_(); + } catch (const sw::redis::Error &) { + return requeue_(); + } for (const auto &did : route.device_ids) { auto it = route.device_to_instance.find(did); std::string peer = (it != route.device_to_instance.end()) ? it->second : ""; @@ -288,45 +346,37 @@ class PushServiceImpl : public PushService } } - // 3) 每个对端一次 PushBatch（异步 brpc::DoNothing） - std::string internal_b64 = _utils_base64_encode(internal_msg.SerializeAsString()); + // 3) 每个对端一次 PushBatch。等待持久化结果后才能 ACK MQ；若后续 + // peer 失败，MQ 重投可能重复前面的下发，但 Unacked ZADD/HSET 以 seq + // 幂等覆盖，提供明确的 at-least-once 语义。 for (auto &kv : peer_to_uids) { const std::string &peer = kv.first; std::vector uids(kv.second.begin(), kv.second.end()); auto channel = _mm_channels->choose(peer); if (!channel) { LOG_WARN("Push-Consumer: 对端 {} 不可达", peer); - for (const auto &u : uids) - if (_online_route) _online_route->unbind(u, "", peer); - if (_cross_outbox) { - _cross_outbox->enqueue(internal_b64, uids, peer, now_ts); - } - continue; + return requeue_(); } PushService_Stub stub(channel.get()); - auto *closure = new SelfDeleteRpcClosure(); - closure->req.set_request_id(msg_info.client_msg_id()); - for (const auto &u : uids) closure->req.add_user_id_list(u); - closure->req.mutable_notify()->CopyFrom(notify_template); + PushBatchReq req; + PushBatchRsp rsp; + brpc::Controller cntl; + req.set_request_id(msg_info.client_msg_id()); + for (const auto &u : uids) req.add_user_id_list(u); + req.mutable_notify()->CopyFrom(notify_template); for (const auto &u : uids) { auto it = uid2seq.find(u); if (it == uid2seq.end()) continue; - auto *p = closure->req.add_user_seqs(); + auto *p = req.add_user_seqs(); p->set_user_id(u); p->set_user_seq(it->second); } - std::string peer_id = peer; - closure->on_done = [peer_id, uids, outbox = _cross_outbox, - online = _online_route, internal_b64, now_ts] - (brpc::Controller *c, const PushBatchRsp &) { - if (c->Failed()) { - LOG_WARN("PushBatch 跨实例失败 peer={}: {}", peer_id, c->ErrorText()); - for (const auto &u : uids) - if (online) online->unbind(u, "", peer_id); - if (outbox) outbox->enqueue(internal_b64, uids, peer_id, now_ts); - } - }; - stub.PushBatch(&closure->cntl, &closure->req, &closure->rsp, closure); + stub.PushBatch(&cntl, &req, &rsp, nullptr); + if (cntl.Failed() || !rsp.header().success()) { + LOG_WARN("PushBatch persistence failed peer={}: {} {}", peer, + cntl.ErrorText(), rsp.header().error_message()); + return requeue_(); + } } return ConsumeAction::Ack; } @@ -338,7 +388,7 @@ class PushServiceImpl : public PushService const auto &ack = notify.msg_push_ack(); if (!is_valid_push_ack_ids(ack.user_seq(), ack.message_id()) || ack.user_id().empty() || - ack.conversation_id().empty() || ack.device_id().empty()) { + ack.device_id().empty()) { LOG_WARN("MSG_PUSH_ACK: invalid fields uid={} did={} seq={} message_id={}", ack.user_id(), ack.device_id(), ack.user_seq(), ack.message_id()); return; @@ -356,7 +406,11 @@ class PushServiceImpl : public PushService } if (_unacked) _unacked->ack(ack.user_id(), ack.device_id(), ack.user_seq()); - // 异步上报 UpdateReadAck（无入站 RPC context，需手动设置 auth metadata） + // Non-message pushes still ACK their Unacked entry but carry no + // conversation delivery ACK watermark. + if (!(ack.seq_id() > 0 && !ack.conversation_id().empty())) return; + + // 异步上报会话 seq_id 水位（无入站 RPC context，需手动设置 auth metadata） auto channel = _mm_channels->choose(_message_service_name); if (!channel) { LOG_WARN("UpdateReadAck: message service 不可达 uid={}", ack.user_id()); @@ -368,7 +422,7 @@ class PushServiceImpl : public PushService chatnow::message::UpdateReadAckRsp>(); closure->req.set_request_id(ack.user_id()); closure->req.set_conversation_id(ack.conversation_id()); - closure->req.set_message_id(static_cast(ack.message_id())); + closure->req.set_seq_id(ack.seq_id()); // 手动设置 auth metadata：WS handler 无入站 RPC context，需自行构造 RpcMetadata ::chatnow::rpc::RpcMetadata meta; meta.set_user_id(conn_uid); @@ -377,10 +431,10 @@ class PushServiceImpl : public PushService std::string data; meta.SerializeToString(&data); closure->cntl.request_attachment().append(data); - closure->on_done = [uid = ack.user_id(), mid = ack.message_id()] + closure->on_done = [uid = ack.user_id(), seq = ack.seq_id()] (brpc::Controller *c, const chatnow::message::UpdateReadAckRsp &r) { if (c->Failed()) { - LOG_WARN("UpdateReadAck RPC 失败 uid={} message_id={}: {}", uid, mid, c->ErrorText()); + LOG_WARN("UpdateReadAck RPC 失败 uid={} seq_id={}: {}", uid, seq, c->ErrorText()); } }; stub.UpdateReadAck(&closure->cntl, &closure->req, &closure->rsp, closure); @@ -428,6 +482,27 @@ class PushServiceImpl : public PushService } private: + void persist_unacked_(const std::string &uid, const std::string &did, + unsigned long user_seq, const std::string &payload_b64, + long long score_ts) { + if (!_unacked) { + metrics::g_push_unacked_persist_failure_total << 1; + throw ::chatnow::ServiceError(::chatnow::error::kSystemUnavailable, + "unacked persistence unavailable"); + } + try { + _unacked->push(uid, did, user_seq, payload_b64, score_ts); + } catch (...) { + metrics::g_push_unacked_persist_failure_total << 1; + throw; + } + } + + static ConsumeAction requeue_() { + metrics::g_push_message_requeue_total << 1; + return ConsumeAction::NackRequeue; + } + void _handle_client_auth_(const NotifyClientAuth &auth, server_t::connection_ptr conn) { if (auth.access_token().empty() || auth.device_id().empty()) { @@ -503,12 +578,13 @@ class PushServiceImpl : public PushService void _write_presence_online_(const std::string &uid, const std::string &did) { try { std::string k = key::presence_device_key(uid, did); + const auto effective_ttl = randomized_ttl(std::chrono::seconds(kPresenceTtlSec)); auto pipe = _redis->pipeline(); pipe.hset(k, "state", "ONLINE"); pipe.hset(k, "last_active_at_ms", std::to_string( std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count())); - pipe.expire(k, std::chrono::seconds(kPresenceTtlSec)); + pipe.expire(k, effective_ttl); pipe.exec(); } catch (std::exception &e) { LOG_WARN("Presence write failed uid={} did={}: {}", uid, did, e.what()); @@ -518,12 +594,13 @@ class PushServiceImpl : public PushService void _write_presence_offline_(const std::string &uid, const std::string &did) { try { std::string k = key::presence_device_key(uid, did); + const auto effective_ttl = randomized_ttl(std::chrono::seconds(kPresenceTtlSec)); auto pipe = _redis->pipeline(); pipe.hset(k, "state", "OFFLINE"); pipe.hset(k, "last_active_at_ms", std::to_string( std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count())); - pipe.expire(k, std::chrono::seconds(kPresenceTtlSec)); + pipe.expire(k, effective_ttl); pipe.exec(); } catch (std::exception &e) { LOG_WARN("Presence offline write failed uid={} did={}: {}", uid, did, e.what()); @@ -533,7 +610,7 @@ class PushServiceImpl : public PushService void _refresh_presence_ttl_(const std::string &uid, const std::string &did) { try { std::string k = key::presence_device_key(uid, did); - _redis->expire(k, std::chrono::seconds(kPresenceTtlSec)); + _redis->expire(k, randomized_ttl(std::chrono::seconds(kPresenceTtlSec))); } catch (std::exception &e) { LOG_WARN("Presence TTL refresh failed uid={} did={}: {}", uid, did, e.what()); } @@ -625,14 +702,14 @@ class PushServiceImpl : public PushService // L2 Redis: hgetall + build RouteEntry RouteEntry route; - auto dmap = _online_route->device_instances_map(uid); + auto dmap = _online_route->device_instances_map_strict(uid); route.device_ids.reserve(dmap.size()); for (const auto &[did, inst] : dmap) { route.device_ids.push_back(did); route.device_to_instance[did] = inst; } if (_local_route_cache) { - _local_route_cache->set(cache_key, route, randomized_ttl(std::chrono::seconds(2))); + _local_route_cache->set(cache_key, route, randomized_ttl(_route_l1_ttl)); } lk.unlock(); @@ -809,7 +886,7 @@ class PushServiceImpl : public PushService continue; } - std::vector> stale_entries; + std::vector> stale_entries; // Cluster mode: for_each traverses all nodes via RedisClient::scan(). long long cursor = 0; do { @@ -824,14 +901,14 @@ class PushServiceImpl : public PushService for (const auto &[did, instance] : device_map) { if (std::find(online_instances.begin(), online_instances.end(), instance) == online_instances.end()) { - stale_entries.emplace_back(uid, did); + stale_entries.emplace_back(uid, did, instance); } } } } while (cursor != 0); - for (const auto &[uid, did] : stale_entries) { - _online_route->unbind(uid, did, ""); + for (const auto &[uid, did, instance] : stale_entries) { + _online_route->unbind(uid, did, instance); if (_local_route_cache) _local_route_cache->invalidate(key::local_route_cache_key(uid)); } if (!stale_entries.empty()) @@ -913,6 +990,7 @@ class PushServiceImpl : public PushService LeaderElection::ptr _cross_reaper_election; LocalCache::ptr _local_route_cache; InflightRegistry::ptr _inflight_registry; + std::chrono::seconds _route_l1_ttl{2}; std::mutex _dummy_mu_; }; @@ -1116,6 +1194,12 @@ class PushServerBuilder _resend_batch = batch; _resend_max_age_sec = max_age_sec; } + void set_route_l1_ttl(int ttl_sec) { + if (ttl_sec < 1 || ttl_sec > 300) { + throw std::invalid_argument("Push route L1 TTL must be within 1..300 seconds"); + } + _route_l1_ttl = std::chrono::seconds(ttl_sec); + } void set_reaper_owner(const std::string &owner) { _reaper_owner = owner; } void set_etcd_client(std::shared_ptr etcd) { _etcd_client = etcd; } @@ -1154,7 +1238,7 @@ class PushServerBuilder _push_service = new PushServiceImpl( _connections, _jwt_codec, _redis_client, _online_route, _unacked, _cross_outbox, _instance_id, _message_service_name, _mm_channels, - _cross_reaper_election, _local_route_cache, _inflight_registry); + _cross_reaper_election, _local_route_cache, _inflight_registry, _route_l1_ttl); _push_service->set_resend_params(_resend_batch, _resend_max_age_sec); int ret = _rpc_server->AddService(_push_service, brpc::ServiceOwnership::SERVER_OWNS_SERVICE); if (ret == -1) { LOG_ERROR("Push: AddService 失败"); abort(); } @@ -1242,6 +1326,7 @@ class PushServerBuilder LeaderElection::ptr _cross_reaper_election; LocalCache::ptr _local_route_cache; InflightRegistry::ptr _inflight_registry; + std::chrono::seconds _route_l1_ttl{2}; std::string _push_service_dir; LeaderElection::ptr _stale_reaper_election; std::thread _stale_reaper_thread; diff --git a/relationship/Dockerfile b/relationship/Dockerfile index a561296..58f61ad 100644 --- a/relationship/Dockerfile +++ b/relationship/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/relationship_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 -CMD /im/bin/relationship_server -flagfile=/im/conf/relationship_server.conf \ No newline at end of file +CMD /im/bin/relationship_server -flagfile=/im/conf/relationship_server.conf diff --git a/scripts/compose_artifact_core_abi.sh b/scripts/compose_artifact_core_abi.sh new file mode 100644 index 0000000..30f88c9 --- /dev/null +++ b/scripts/compose_artifact_core_abi.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Libraries guaranteed by the pinned Ubuntu base image. Everything else must +# travel in the service-private artifact closure. +is_core_system_abi() { + case "$(basename "$1")" in + ld-linux*.so.*|ld-musl-*.so.*|libc.so.*|libm.so.*|libpthread.so.*|librt.so.*|libdl.so.*|libgcc_s.so.*|libstdc++.so.*|libanl.so.*|libBrokenLocale.so.*|libcrypt.so.*|libnss_*.so.*|libresolv.so.*|libutil.so.*) + return 0 + ;; + *) + return 1 + ;; + esac +} + +is_system_library_path() { + case "$1" in + /lib/*|/lib64/*|/usr/lib/*) return 0 ;; + *) return 1 ;; + esac +} diff --git a/scripts/package_compose_artifacts.sh b/scripts/package_compose_artifacts.sh new file mode 100755 index 0000000..8681c45 --- /dev/null +++ b/scripts/package_compose_artifacts.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=compose_artifact_core_abi.sh +. "$script_dir/compose_artifact_core_abi.sh" + +services=(conversation gateway identity media message presence push relationship transmite) +build_root="${1:-build}" +artifact_root="${2:-compose-artifacts}" +ldd_command="${LDD:-ldd}" +sha256sum_command="${SHA256SUM:-sha256sum}" + +rm -rf "$artifact_root" +mkdir -p "$artifact_root" + +for service in "${services[@]}"; do + binary="$build_root/$service/${service}_server" + [[ -x "$binary" ]] || { + echo "missing executable service binary: $binary" >&2 + exit 1 + } + + service_root="$artifact_root/$service" + depends_dir="$service_root/depends" + mkdir -p "$service_root/build" "$depends_dir" + cp -p "$binary" "$service_root/build/${service}_server" + + ldd_output="$("$ldd_command" "$binary" 2>&1)" || { + echo "ldd failed for $binary: $ldd_output" >&2 + exit 1 + } + if grep -Fq "not found" <<<"$ldd_output"; then + echo "unresolved shared library for $binary:" >&2 + echo "$ldd_output" >&2 + exit 1 + fi + + while IFS= read -r library; do + [[ -n "$library" ]] || continue + library_name="$(basename "$library")" + if is_core_system_abi "$library_name"; then + if ! is_system_library_path "$library"; then + echo "core system ABI resolved outside pinned runtime paths for $binary: $library" >&2 + exit 1 + fi + continue + fi + resolved_library="$(realpath "$library")" || { + echo "unable to resolve shared library for $binary: $library" >&2 + exit 1 + } + destination="$depends_dir/$library_name" + if [[ -e "$destination" ]]; then + if ! cmp -s "$resolved_library" "$destination"; then + echo "conflicting shared libraries share artifact basename for $binary: $library_name" >&2 + exit 1 + fi + continue + fi + cp -L "$resolved_library" "$destination" + done < <(awk ' + /=> \/[^ ]+/ { print $3; next } + /^[[:space:]]*\/[^ ]+/ { print $1 } + ' <<<"$ldd_output" | LC_ALL=C sort -u) +done + +( + cd "$artifact_root" + find . \( -type f -o -type l \) ! -name MANIFEST.sha256 -print0 \ + | LC_ALL=C sort -z \ + | xargs -0 "$sha256sum_command" > MANIFEST.sha256 +) diff --git a/scripts/validate_compose_artifacts.sh b/scripts/validate_compose_artifacts.sh new file mode 100755 index 0000000..4106e02 --- /dev/null +++ b/scripts/validate_compose_artifacts.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=compose_artifact_core_abi.sh +. "$script_dir/compose_artifact_core_abi.sh" + +services=(conversation gateway identity media message presence push relationship transmite) +artifact_root="${1:-compose-artifacts}" +if [[ ! -d "$artifact_root" ]]; then + echo "missing artifact root directory: $artifact_root" >&2 + exit 1 +fi +artifact_root="$(cd "$artifact_root" && pwd -P)" +manifest="$artifact_root/MANIFEST.sha256" +ldd_command="${LDD:-ldd}" +sha256sum_command="${SHA256SUM:-sha256sum}" + +if [[ ! -f "$manifest" ]]; then + echo "missing artifact manifest: $manifest" >&2 + exit 1 +fi + +actual_manifest="$(mktemp)" +trap 'rm -f "$actual_manifest"' EXIT +( + cd "$artifact_root" + find . \( -type f -o -type l \) ! -name MANIFEST.sha256 -print0 \ + | LC_ALL=C sort -z \ + | xargs -0 "$sha256sum_command" > "$actual_manifest" + if ! cmp -s MANIFEST.sha256 "$actual_manifest"; then + echo "manifest mismatch or unlisted artifact file" >&2 + exit 1 + fi + # sha256sum --check is retained as an explicit integrity gate. + "$sha256sum_command" --check MANIFEST.sha256 +) + +for service in "${services[@]}"; do + binary="$artifact_root/$service/build/${service}_server" + depends_dir="$artifact_root/$service/depends" + [[ -x "$binary" ]] || { + echo "missing executable service binary: $binary" >&2 + exit 1 + } + if [[ ! -d "$depends_dir" ]]; then + echo "missing shared-library directory: $depends_dir" >&2 + exit 1 + fi + depends_dir_real="$(cd "$depends_dir" && pwd -P)" + + while IFS= read -r packaged_library; do + library_name="$(basename "$packaged_library")" + if is_core_system_abi "$library_name"; then + echo "packaged runtime loader or system ABI library for $binary: $packaged_library" >&2 + exit 1 + fi + done < <(find "$depends_dir" -mindepth 1 -maxdepth 1 \( -type f -o -type l \) -print | LC_ALL=C sort) + + ldd_output="$(env -i PATH=/usr/bin:/bin LD_LIBRARY_PATH="$depends_dir" "$ldd_command" "$binary" 2>&1)" || { + echo "isolated ldd failed for $binary: $ldd_output" >&2 + exit 1 + } + if grep -Fq "not found" <<<"$ldd_output"; then + echo "unresolved shared library for $binary:" >&2 + echo "$ldd_output" >&2 + exit 1 + fi + + while IFS=$'\t' read -r entry_kind library; do + [[ -n "$library" ]] || continue + library_name="$(basename "$library")" + if [[ ! -f "$library" ]]; then + echo "shared library is outside packaged closure for $binary: $library" >&2 + exit 1 + fi + resolved_library="$(realpath "$library")" + case "$resolved_library" in + "$depends_dir_real"/*) ;; + *) + if ! is_core_system_abi "$library_name" || ! is_system_library_path "$resolved_library"; then + echo "non-core library is outside packaged closure for $binary: $library" >&2 + exit 1 + fi + ;; + esac + done < <(awk ' + /=> \/[^ ]+/ { print "dependency\t" $3; next } + /^[[:space:]]*\/[^ ]+/ { print "loader\t" $1 } + ' <<<"$ldd_output" | LC_ALL=C sort -u) +done diff --git a/tests/Makefile b/tests/Makefile index 686f3e3..619f47a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,7 @@ -.PHONY: proto test-bvt test-func test-scenario test-perf test-agent-policy clean deps +.PHONY: proto test-bvt test-func test-reliability test-scenario test-perf test-perf-cache test-perf-cache-gate test-agent-policy clean deps + +PF09_BASELINE_MSG_PER_SEC ?= 5000 +PF09_EXPECTED_TRANSMITE_INSTANCES ?= 1 # Generate Go protobuf from proto/ definitions # NOTE: protoc-gen-go >= v1.35 requires go_package option or M flags to derive @@ -35,6 +38,10 @@ test-bvt: test-func: go test -tags=func ./func/... -v -count=1 +# Run reliability/chaos tests (requires the Docker Compose stack) +test-reliability: + go test -tags=reliability ./reliability/... -v -count=1 -timeout=180s + # Run scenario tests only (L3) test-scenario: go test -tags=func ./func/... -run TestScenario -v -count=1 @@ -43,6 +50,21 @@ test-scenario: test-perf: go test -tags=perf ./perf/... -bench=. -benchmem -count=3 -benchtime=10s +# Compile/discover PF-09 locally. It skips unless the explicit full-stack gate is enabled. +test-perf-cache: + go test -v -tags=perf ./perf/... -run '^$$' -bench '^BenchmarkPF09_UserInfoCache$$' -benchtime=1x + +# Real Linux full-stack gate. The deployment must raise Transmite's user/session +# rate limits above the generated load; any rate_limited response fails PF-09. +# Multi-instance stacks must set PF09_TRANSMITE_VARS_URLS to every bvar endpoint, +# separated by commas, so the RPC-reduction snapshot cannot omit an instance. +test-perf-cache-gate: + @test "$$(uname -s)" = Linux || (echo "PF-09 gate requires Linux" >&2; exit 1) + PF09_RUN_FULLSTACK=1 PF09_BASELINE_MSG_PER_SEC=$(PF09_BASELINE_MSG_PER_SEC) \ + PF09_EXPECTED_TRANSMITE_INSTANCES=$(PF09_EXPECTED_TRANSMITE_INSTANCES) \ + go test -tags=perf ./perf/... -run '^$$' -bench '^BenchmarkPF09_UserInfoCache$$' \ + -benchmem -count=3 -benchtime=1x + # Run repository Agent policy validator tests test-agent-policy: go test ./pkg/agentpolicy ./cmd/agent-policy -count=1 diff --git a/tests/config.yaml b/tests/config.yaml index a0b5541..e65dad1 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -19,3 +19,11 @@ database: log: level: "debug" + +infra: + redis_container: "redis-node1" + push_container: "push_server-service" + push_vars: "http://localhost:10008" + push_route_l1_ttl_sec: 30 + compose_dir: ".." + transmite_vars: "http://localhost:10004" diff --git a/tests/func/cache_test.go b/tests/func/cache_test.go new file mode 100644 index 0000000..e5195c9 --- /dev/null +++ b/tests/func/cache_test.go @@ -0,0 +1,545 @@ +//go:build func + +package func_test + +import ( + "bytes" + "encoding/base64" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" + + "chatnow-tests/pkg/client" + "chatnow-tests/pkg/fixture" + "chatnow-tests/pkg/verify" + common "chatnow-tests/proto/chatnow/common" + identity "chatnow-tests/proto/chatnow/identity" + msg "chatnow-tests/proto/chatnow/message" + push "chatnow-tests/proto/chatnow/push" + transmite "chatnow-tests/proto/chatnow/transmite" +) + +func userInfoBucket(uid string) uint32 { + hash := uint32(2166136261) + for i := 0; i < len(uid); i++ { + hash ^= uint32(uid[i]) + hash *= 16777619 + } + return hash % 64 +} + +func userInfoRedisKey(uid string) string { + return fmt.Sprintf("im:user:{%d}:%s", userInfoBucket(uid), uid) +} + +func redisRaw(t testing.TB, key string) []byte { + t.Helper() + container := HTTP.Config().Infra.RedisContainer + if container == "" { + container = "redis-node1" + } + out, err := exec.Command("docker", "exec", container, "redis-cli", "-c", "--raw", "GET", key).Output() + require.NoError(t, err) + return bytes.TrimSuffix(out, []byte("\n")) +} + +func sendCacheTestMessageResult(user *client.HTTPClient, convID, suffix string) error { + rsp := &transmite.SendMessageRsp{} + err := user.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), + ConversationId: convID, + Content: &msg.MessageContent{ + Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: "user-info-cache-" + suffix}}, + }, + ClientMsgId: client.NewRequestID(), + }, rsp) + if err != nil { + return err + } + if !rsp.GetHeader().GetSuccess() { + return fmt.Errorf("send failed: %s", rsp.GetHeader().GetErrorMessage()) + } + return nil +} + +func sendCacheTestMessage(t testing.TB, user *client.HTTPClient, convID, suffix string) { + t.Helper() + require.NoError(t, sendCacheTestMessageResult(user, convID, suffix)) +} + +type cacheTestWebSocket = client.WebSocket + +func openCacheTestWebSocket(t testing.TB) *cacheTestWebSocket { + t.Helper() + ws, err := client.OpenWebSocket(HTTP.Config()) + require.NoError(t, err) + return ws +} + +func writeCacheTestBinary(t testing.TB, ws *cacheTestWebSocket, payload []byte) { + t.Helper() + require.NoError(t, ws.WriteBinary(payload)) +} + +func appendProtoString(dst []byte, field protowire.Number, value string) []byte { + dst = protowire.AppendTag(dst, field, protowire.BytesType) + return protowire.AppendString(dst, value) +} + +func cacheTestAuthNotify(accessToken, deviceID string) []byte { + return client.PushAuthNotify(accessToken, deviceID) +} + +func cacheTestHeartbeatNotify(uid string) []byte { + var heartbeat []byte + heartbeat = appendProtoString(heartbeat, 1, uid) + var notify []byte + notify = protowire.AppendTag(notify, 2, protowire.VarintType) + notify = protowire.AppendVarint(notify, 51) // CLIENT_HEARTBEAT + notify = protowire.AppendTag(notify, 9, protowire.BytesType) + return protowire.AppendBytes(notify, heartbeat) +} + +func requireJitteredRedisTTL(t testing.TB, key string, base time.Duration) time.Duration { + t.Helper() + ttl := verify.RedisTTL(t, key) + require.GreaterOrEqual(t, ttl, base*8/10-2*time.Second, key) + require.LessOrEqual(t, ttl, base*12/10+2*time.Second, key) + return ttl +} + +var _ func(...string) (string, error) = verify.RedisCLIResult + +func readRepoSource(t testing.TB, relativePath string) string { + t.Helper() + _, currentFile, _, ok := runtime.Caller(0) + require.True(t, ok, "locate cache_test.go") + repoRoot := filepath.Clean(filepath.Join(filepath.Dir(currentFile), "..", "..")) + content, err := os.ReadFile(filepath.Join(repoRoot, filepath.FromSlash(relativePath))) + require.NoError(t, err, relativePath) + return string(content) +} + +func normalizedSourceSection(source, start, end string) (string, error) { + startAt := strings.Index(source, start) + if startAt < 0 { + return "", fmt.Errorf("missing source section %q", start) + } + endAt := strings.Index(source[startAt+len(start):], end) + if endAt < 0 { + return "", fmt.Errorf("missing end marker %q for %q", end, start) + } + section := source[startAt : startAt+len(start)+endAt] + return strings.Join(strings.Fields(section), " "), nil +} + +func requireSourceFragments(sectionName, section string, fragments ...string) error { + for _, fragment := range fragments { + normalized := strings.Join(strings.Fields(fragment), " ") + if !strings.Contains(section, normalized) { + return fmt.Errorf("%s missing contract fragment %q", sectionName, normalized) + } + } + return nil +} + +func checkCacheTTLSourceContracts(source string) error { + sections := make(map[string]string) + markers := [][3]string{ + {"Session", "class Session", "class Status"}, + {"Status", "class Status", "class Codes"}, + } + for _, marker := range markers { + section, err := normalizedSourceSection(source, marker[1], marker[2]) + if err != nil { + return err + } + sections[marker[0]] = section + } + + rules := []struct { + name string + fragments []string + }{ + {"Session", []string{ + "_c->set(key::kSession + ssid, uid, randomized_ttl(ttl))", + "_c->expire(key::kSession + ssid, randomized_ttl(ttl))", + }}, + {"Status", []string{ + "_c->set(key::kStatus + uid, \"1\", randomized_ttl(ttl))", + "_c->expire(key::kStatus + uid, randomized_ttl(ttl))", + }}, + } + for _, rule := range rules { + if err := requireSourceFragments(rule.name, sections[rule.name], rule.fragments...); err != nil { + return err + } + } + + return nil +} + +func requireCacheTTLSourceContracts(t testing.TB, source string) { + t.Helper() + require.NoError(t, checkCacheTTLSourceContracts(source)) +} + +func TestFN_CA_LegacyUnusedTTLSourceContract(t *testing.T) { + daoSource := readRepoSource(t, "common/dao/data_redis.hpp") + requireCacheTTLSourceContracts(t, daoSource) + + t.Run("rejects fixed Session TTL", func(t *testing.T) { + mutant := strings.Replace(daoSource, "randomized_ttl(ttl)", "ttl", 1) + require.Error(t, checkCacheTTLSourceContracts(mutant)) + }) +} + +type redisResultPredicate func(string) (bool, error) + +func requireEventuallyRedis(t testing.TB, timeout, interval time.Duration, message string, + predicate redisResultPredicate, args ...string) string { + t.Helper() + var mu sync.Mutex + var lastResult string + var lastErr error + matched := assert.Eventually(t, func() bool { + result, err := verify.RedisCLIResult(args...) + if err == nil { + var predicateMatch bool + predicateMatch, err = predicate(result) + mu.Lock() + lastResult, lastErr = result, err + mu.Unlock() + return predicateMatch && err == nil + } + mu.Lock() + lastResult, lastErr = result, err + mu.Unlock() + return false + }, timeout, interval, message) + + mu.Lock() + result, err := lastResult, lastErr + mu.Unlock() + require.NoError(t, err, "%s (last result %q)", message, result) + require.True(t, matched, "%s (last result %q)", message, result) + return result +} + +func redisEquals(expected string) redisResultPredicate { + return func(result string) (bool, error) { return result == expected, nil } +} + +// FN-CA-04 | cache expirations are bounded, varied, and paired unacked keys +// share one randomized sample per push operation. +func TestFN_CA_TTLJitter(t *testing.T) { + const sampleCount = 20 + const sessionTTL = 7 * 24 * time.Hour + connections := make([]*cacheTestWebSocket, 0, sampleCount) + t.Cleanup(func() { + for _, ws := range connections { + _ = ws.Close() + } + }) + + t.Log("Session and Status DAOs have no production call sites; the ignored Task 5 DAO harness covers append/touch") + + t.Run("Codes", func(t *testing.T) { + if os.Getenv("SMTP_HOST") == "" { + t.Skip("SMTP_HOST is not configured; only the Codes reachable-path subtest is skipped") + } + codeRsp := &identity.SendVerifyCodeRsp{} + require.NoError(t, HTTP.DoNoAuth("/service/identity/send_verify_code", + &identity.SendVerifyCodeReq{ + RequestId: client.NewRequestID(), + Destination: &identity.SendVerifyCodeReq_Email{ + Email: fmt.Sprintf("ttl-%s@example.com", strings.ToLower(client.NewRequestID())), + }, + }, codeRsp)) + require.True(t, codeRsp.GetHeader().GetSuccess(), codeRsp.GetHeader().GetErrorMessage()) + requireJitteredRedisTTL(t, "im:code:"+codeRsp.GetVerifyCodeId(), 5*time.Minute) + }) + + t.Run("DeviceSet", func(t *testing.T) { + deviceTTLs := make(map[time.Duration]struct{}, sampleCount) + var heartbeatBefore time.Duration + for i := 0; i < sampleCount; i++ { + user, _, _ := fixture.RegisterAndLogin(t, HTTP) + ws := openCacheTestWebSocket(t) + connections = append(connections, ws) + writeCacheTestBinary(t, ws, cacheTestAuthNotify(user.AccessToken, "default_device")) + + deviceKey := fmt.Sprintf("im:dev:{%s}", user.UserID) + requireEventuallyRedis(t, 5*time.Second, 50*time.Millisecond, + "DeviceSet key must exist", redisEquals("1"), "EXISTS", deviceKey) + if i == 0 { + verify.RedisCLI(t, "EXPIRE", deviceKey, "60") + heartbeatBefore = verify.RedisTTL(t, deviceKey) + writeCacheTestBinary(t, ws, cacheTestHeartbeatNotify(user.UserID)) + requireEventuallyRedis(t, 5*time.Second, 50*time.Millisecond, + "heartbeat must renew DeviceSet TTL", func(result string) (bool, error) { + seconds, err := strconv.ParseInt(result, 10, 64) + return time.Duration(seconds)*time.Second > heartbeatBefore+24*time.Hour, err + }, "TTL", deviceKey) + } else { + writeCacheTestBinary(t, ws, cacheTestHeartbeatNotify(user.UserID)) + } + deviceTTLs[requireJitteredRedisTTL(t, deviceKey, sessionTTL)] = struct{}{} + } + require.GreaterOrEqual(t, len(deviceTTLs), 2, + "20 independently randomized device TTLs must not all be identical") + }) + + t.Run("UnackedPush", func(t *testing.T) { + sender, recipient, convID := fixture.MakeFriends(t, HTTP) + recipientWS := openCacheTestWebSocket(t) + connections = append(connections, recipientWS) + writeCacheTestBinary(t, recipientWS, cacheTestAuthNotify(recipient.AccessToken, "default_device")) + deviceKey := fmt.Sprintf("im:dev:{%s}", recipient.UserID) + requireEventuallyRedis(t, 5*time.Second, 50*time.Millisecond, + "recipient DeviceSet key must exist", redisEquals("1"), "EXISTS", deviceKey) + + unackedKey := fmt.Sprintf("im:unack:{%s:default_device}", recipient.UserID) + unackedIndexKey := fmt.Sprintf("im:unack:idx:{%s:default_device}", recipient.UserID) + initial, err := strconv.Atoi(verify.RedisCLI(t, "ZCARD", unackedKey)) + require.NoError(t, err) + unackedTTLs := make(map[time.Duration]struct{}) + for i := 0; i < 12; i++ { + sendCacheTestMessage(t, sender, convID, fmt.Sprintf("ttl-jitter-%d", i)) + requireEventuallyRedis(t, 10*time.Second, 100*time.Millisecond, + "UnackedPush ZSET must receive the message", func(result string) (bool, error) { + count, err := strconv.Atoi(result) + return count >= initial+i+1, err + }, "ZCARD", unackedKey) + unackedTTL := requireJitteredRedisTTL(t, unackedKey, sessionTTL) + indexTTL := requireJitteredRedisTTL(t, unackedIndexKey, sessionTTL) + require.LessOrEqual(t, absDuration(unackedTTL-indexTTL), time.Second, + "paired unacked keys must share one TTL sample") + unackedTTLs[unackedTTL] = struct{}{} + } + require.GreaterOrEqual(t, len(unackedTTLs), 2, + "repeated UnackedPush operations must produce varied paired TTL samples") + }) +} + +func absDuration(value time.Duration) time.Duration { + if value < 0 { + return -value + } + return value +} + +func TestFN_CA_UnackedSameUserSeqLatestPayloadAndAck(t *testing.T) { + recipient, _, _ := fixture.RegisterAndLogin(t, HTTP) + ws := openCacheTestWebSocket(t) + t.Cleanup(func() { _ = ws.Close() }) + const deviceID = "default_device" + writeCacheTestBinary(t, ws, cacheTestAuthNotify(recipient.AccessToken, deviceID)) + + deviceKey := fmt.Sprintf("im:dev:{%s}", recipient.UserID) + requireEventuallyRedis(t, 5*time.Second, 50*time.Millisecond, + "Push route must exist before direct PushToUser calls", redisEquals("1"), + "EXISTS", deviceKey) + + userSeq := uint64(time.Now().UnixNano()) + unackedKey := fmt.Sprintf("im:unack:{%s:%s}", recipient.UserID, deviceID) + unackedIndexKey := fmt.Sprintf("im:unack:idx:{%s:%s}", recipient.UserID, deviceID) + verify.RedisCLI(t, "DEL", unackedKey, unackedIndexKey) + + first := &push.NotifyMessage{ + NotifyEventId: proto.String("unacked-first-" + client.NewRequestID()), + NotifyType: push.NotifyType_TYPING_NOTIFY, + NotifyRemarks: &push.NotifyMessage_Typing{Typing: &push.NotifyTyping{ + UserId: recipient.UserID, ConversationId: "unacked-contract", IsTyping: false, + }}, + } + second := proto.Clone(first).(*push.NotifyMessage) + second.NotifyEventId = proto.String("unacked-second-" + client.NewRequestID()) + second.GetTyping().IsTyping = true + + for _, notify := range []*push.NotifyMessage{first, second} { + rsp := &push.PushToUserRsp{} + require.NoError(t, HTTP.DoProtobufURL( + HTTP.Config().Infra.PushVars+"/chatnow.push.PushService/PushToUser", + &push.PushToUserReq{ + RequestId: client.NewRequestID(), UserId: recipient.UserID, + Notify: notify, UserSeq: proto.Uint64(userSeq), TargetDeviceIds: []string{deviceID}, + }, rsp)) + require.True(t, rsp.GetHeader().GetSuccess(), rsp.GetHeader().GetErrorMessage()) + require.Equal(t, int32(1), rsp.GetOnlineDeviceCount()) + } + + seq := strconv.FormatUint(userSeq, 10) + requireEventuallyRedis(t, 5*time.Second, 50*time.Millisecond, + "same user_seq must retain one stable ZSET identity", redisEquals("1"), + "ZCARD", unackedKey) + require.Equal(t, seq, verify.RedisCLI(t, "ZRANGE", unackedKey, "0", "-1")) + require.Equal(t, "1", verify.RedisCLI(t, "HLEN", unackedIndexKey)) + + encodedLatest := verify.RedisCLI(t, "HGET", unackedIndexKey, seq) + latestBytes, err := base64.StdEncoding.DecodeString(encodedLatest) + require.NoError(t, err) + latest := &push.NotifyMessage{} + require.NoError(t, proto.Unmarshal(latestBytes, latest)) + require.True(t, proto.Equal(second, latest), + "same user_seq must replace the HASH payload with the second notification") + + ackBytes, err := proto.Marshal(&push.NotifyMessage{ + NotifyType: push.NotifyType_MSG_PUSH_ACK, + NotifyRemarks: &push.NotifyMessage_MsgPushAck{MsgPushAck: &push.NotifyMsgPushAck{ + UserId: recipient.UserID, DeviceId: deviceID, MessageId: 1, + UserSeq: userSeq, ConversationId: "unacked-contract", SeqId: 0, // typing push has no conversation watermark + }}, + }) + require.NoError(t, err) + writeCacheTestBinary(t, ws, ackBytes) + requireEventuallyRedis(t, 5*time.Second, 50*time.Millisecond, + "WebSocket ACK must remove the ZSET identity", redisEquals("0"), + "EXISTS", unackedKey) + requireEventuallyRedis(t, 5*time.Second, 50*time.Millisecond, + "WebSocket ACK must remove the HASH payload index", redisEquals("0"), + "EXISTS", unackedIndexKey) +} + +// FN-CA-05 | healthy Redis applies the distributed message rate limit. +func TestFN_CA_RateLimit(t *testing.T) { + user, peer, convID := fixture.MakeFriends(t, HTTP) + _ = peer + + rateLimited := 0 + for i := 0; i < 650; i++ { + rsp := &transmite.SendMessageRsp{} + err := user.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), + ConversationId: convID, + Content: &msg.MessageContent{ + Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: fmt.Sprintf("rate-limit-%d", i)}}, + }, + ClientMsgId: client.NewRequestID(), + }, rsp) + require.NoError(t, err) + if rsp.GetHeader().GetErrorMessage() == "rate_limited" { + rateLimited++ + } + } + + require.Greater(t, rateLimited, 0, "healthy Redis must enforce the distributed rate limit") +} + +func TestFN_CA_UserInfoL2AvoidsRepeatedRPC(t *testing.T) { + user, _, convID := fixture.MakeFriends(t, HTTP) + verify.RedisCLI(t, "DEL", userInfoRedisKey(user.UserID)) + endpoints := transmiteBVarEndpoints(t) + before := sumBVar(t, endpoints, "user_info_rpc_total") + + for i := 0; i < 20; i++ { + sendCacheTestMessage(t, user, convID, fmt.Sprintf("repeat-%d", i)) + } + + after := sumBVar(t, endpoints, "user_info_rpc_total") + require.LessOrEqual(t, after-before, int64(len(endpoints))) + require.Equal(t, "1", verify.RedisCLI(t, "EXISTS", userInfoRedisKey(user.UserID))) +} + +func TestFN_CA_UserInfoSingleflight(t *testing.T) { + user, _, convID := fixture.MakeFriends(t, HTTP) + verify.RedisCLI(t, "DEL", userInfoRedisKey(user.UserID)) + endpoints := transmiteBVarEndpoints(t) + before := sumBVar(t, endpoints, "user_info_rpc_total") + + start := make(chan struct{}) + results := make(chan error, 200) + var wg sync.WaitGroup + for i := 0; i < 200; i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + <-start + results <- sendCacheTestMessageResult(user, convID, fmt.Sprintf("flight-%d", i)) + }() + } + close(start) + wg.Wait() + close(results) + for err := range results { + require.NoError(t, err) + } + + after := sumBVar(t, endpoints, "user_info_rpc_total") + require.LessOrEqual(t, after-before, int64(len(endpoints)), + "each Transmite process may originate at most one Identity RPC") +} + +func transmiteBVarEndpoints(t testing.TB) []string { + t.Helper() + parts := strings.Split(HTTP.Config().Infra.TransmiteVars, ",") + endpoints := make([]string, 0, len(parts)) + for _, part := range parts { + endpoint := strings.TrimRight(strings.TrimSpace(part), "/") + if endpoint == "" { + t.Fatal("empty Transmite bvar endpoint") + } + endpoints = append(endpoints, endpoint) + } + return endpoints +} + +func sumBVar(t testing.TB, endpoints []string, name string) int64 { + t.Helper() + var total int64 + for _, endpoint := range endpoints { + total += verify.BVar(t, endpoint, name) + } + return total +} + +func TestFN_CA_UserInfoInvalidatedAfterProfileUpdate(t *testing.T) { + user, _, convID := fixture.MakeFriends(t, HTTP) + key := userInfoRedisKey(user.UserID) + verify.RedisCLI(t, "DEL", key) + sendCacheTestMessage(t, user, convID, "warm") + require.Equal(t, "1", verify.RedisCLI(t, "EXISTS", key)) + warmInfo := &common.UserInfo{} + require.NoError(t, proto.Unmarshal(redisRaw(t, key), warmInfo)) + require.NotEmpty(t, warmInfo.GetNickname()) + + newNickname := fmt.Sprintf("cache_%d", time.Now().UnixNano()%1_000_000_000) + require.NotEqual(t, warmInfo.GetNickname(), newNickname) + updateRsp := &identity.UpdateProfileRsp{} + require.NoError(t, user.DoAuth("/service/identity/update_profile", &identity.UpdateProfileReq{ + RequestId: client.NewRequestID(), + Nickname: &newNickname, + }, updateRsp)) + require.True(t, updateRsp.GetHeader().GetSuccess(), updateRsp.GetHeader().GetErrorMessage()) + require.Equal(t, newNickname, updateRsp.GetUserInfo().GetNickname()) + require.Equal(t, "0", verify.RedisCLI(t, "EXISTS", key)) + + // UpdateProfile invalidates shared L2 immediately. A Transmite process may + // still publish its bounded stale L1 value until the documented 45s lifetime + // expires; this waits at the business boundary instead of forcing an internal + // generation/CAS interleaving with a production timing hook. + time.Sleep(55 * time.Second) + sendCacheTestMessage(t, user, convID, "after-update") + // The next Transmite lookup must repopulate Redis from Identity with the + // latest profile, proving stale publication cannot survive the L1 bound. + serialized := redisRaw(t, key) + info := &common.UserInfo{} + require.NoError(t, proto.Unmarshal(serialized, info)) + require.Equal(t, newNickname, info.GetNickname()) + require.NotEqual(t, warmInfo.GetNickname(), info.GetNickname()) +} diff --git a/tests/func/consistency_test.go b/tests/func/consistency_test.go index 1094581..a28dc4f 100644 --- a/tests/func/consistency_test.go +++ b/tests/func/consistency_test.go @@ -73,7 +73,7 @@ func TestFN_DC_UnreadCount(t *testing.T) { dbV.UnreadCount(t, bob.UserID, convID, 3) // bob sync 消息后，HTTP 响应也应显示 unread=3 - // （sync 不会清未读，需要 UpdateReadAck 才清） + // （sync 与送达 ACK 都不会清未读；用户已读需调用 Conversation.MarkRead） // 这里只验证 DB 一致性，不测 HTTP（HTTP 测试在 FN-MS 中覆盖） } diff --git a/tests/func/message_test.go b/tests/func/message_test.go index 30e9b91..de0ec4f 100644 --- a/tests/func/message_test.go +++ b/tests/func/message_test.go @@ -4,6 +4,7 @@ package func_test import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -443,10 +444,13 @@ func TestFN_MS_SelectByClientMsgId_NotFound(t *testing.T) { assert.Nil(t, rsp.Message, "不存在的 client_msg_id 应返回 nil message") } -// FN-MS (untested) | P0 | UpdateReadAck 更新 last_read_msg_id +// FN-MS | P0 | UpdateReadAck advances the conversation delivery ACK watermark. func TestFN_MS_UpdateReadAck_Success(t *testing.T) { alice, bob, convID := fixture.MakeFriends(t, HTTP) - _, seqID := fixture.SendTextMessage(t, alice, convID, "ack-test-msg") + messageID, seqID := fixture.SendTextMessage(t, alice, convID, "ack-test-msg") + verifier := verify.NewDBVerifier(Cfg.Database.MySQLDSN) + defer verifier.Close() + verifier.WaitMessageExists(t, messageID, 10*time.Second) req := &msg.UpdateReadAckReq{ RequestId: client.NewRequestID(), @@ -458,17 +462,19 @@ func TestFN_MS_UpdateReadAck_Success(t *testing.T) { require.NoError(t, err) require.True(t, rsp.Header.Success, "update_read_ack 失败: %s", rsp.Header.ErrorMessage) - // 直查 DB 验证 last_read_seq 更新 - verifier := verify.NewDBVerifier(Cfg.Database.MySQLDSN) - defer verifier.Close() - verifier.LastReadSeq(t, bob.UserID, convID, seqID) + // 直查 DB 验证 last_ack_seq 更新。 + verifier.LastAckSeq(t, bob.UserID, convID, seqID) } -// FN-MS (untested) | P0 | UpdateReadAck 幂等（重复 ACK 不回退） +// FN-MS | P0 | UpdateReadAck is idempotent and never moves backwards. func TestFN_MS_UpdateReadAck_Idempotent(t *testing.T) { alice, bob, convID := fixture.MakeFriends(t, HTTP) - _, seq1 := fixture.SendTextMessage(t, alice, convID, "ack-idempotent-1") - _, seq2 := fixture.SendTextMessage(t, alice, convID, "ack-idempotent-2") + messageID1, seq1 := fixture.SendTextMessage(t, alice, convID, "ack-idempotent-1") + messageID2, seq2 := fixture.SendTextMessage(t, alice, convID, "ack-idempotent-2") + verifier := verify.NewDBVerifier(Cfg.Database.MySQLDSN) + defer verifier.Close() + verifier.WaitMessageExists(t, messageID1, 10*time.Second) + verifier.WaitMessageExists(t, messageID2, 10*time.Second) // ACK 到 seq2 ackReq := &msg.UpdateReadAckReq{ @@ -476,18 +482,22 @@ func TestFN_MS_UpdateReadAck_Idempotent(t *testing.T) { ConversationId: convID, SeqId: seq2, } - require.NoError(t, bob.DoAuth("/service/message/update_read_ack", ackReq, &msg.UpdateReadAckRsp{})) + ackRsp := &msg.UpdateReadAckRsp{} + require.NoError(t, bob.DoAuth("/service/message/update_read_ack", ackReq, ackRsp)) + require.True(t, ackRsp.GetHeader().GetSuccess(), + "newer delivery ACK watermark failed: %s", ackRsp.GetHeader().GetErrorMessage()) - // 再 ACK 到 seq1（小于 seq2），last_read_seq 不应回退 + // 再 ACK 较早消息，last_ack_seq 不应回退。 ackReq2 := &msg.UpdateReadAckReq{ RequestId: client.NewRequestID(), ConversationId: convID, SeqId: seq1, } - require.NoError(t, bob.DoAuth("/service/message/update_read_ack", ackReq2, &msg.UpdateReadAckRsp{})) + ackRsp2 := &msg.UpdateReadAckRsp{} + require.NoError(t, bob.DoAuth("/service/message/update_read_ack", ackReq2, ackRsp2)) + require.True(t, ackRsp2.GetHeader().GetSuccess(), + "backward delivery ACK watermark must be idempotent: %s", ackRsp2.GetHeader().GetErrorMessage()) - // 直查 DB 验证 last_read_seq 仍为 seq2 - verifier := verify.NewDBVerifier(Cfg.Database.MySQLDSN) - defer verifier.Close() - verifier.LastReadSeq(t, bob.UserID, convID, seq2) + // 直查 DB 验证 last_ack_seq 仍为 seq2。 + verifier.LastAckSeq(t, bob.UserID, convID, seq2) } diff --git a/tests/func/scenarios_test.go b/tests/func/scenarios_test.go index d80f852..63823af 100644 --- a/tests/func/scenarios_test.go +++ b/tests/func/scenarios_test.go @@ -582,7 +582,7 @@ func TestScenario_LargeGroupFanOut(t *testing.T) { // --------------------------------------------------------------------------- // Scenario 9: Unread Count Consistency（未读数跨服务一致性） -// SC-09 | P0 | scenario | 未读数跨服务跨设备一致：发消息 unread+1 -> UpdateReadAck -> unread=0 +// SC-09 | P0 | scenario | 未读数跨服务跨设备一致：发消息 unread+1 -> MarkRead -> unread=0 // --------------------------------------------------------------------------- func TestScenario_UnreadCountConsistency(t *testing.T) { @@ -613,20 +613,21 @@ func TestScenario_UnreadCountConsistency(t *testing.T) { defer dbV.Close() dbV.UnreadCount(t, b.UserID, convID, 3) - // Step 4: b UpdateReadAck（读到最后一条 seq） - ackReq := &msg.UpdateReadAckReq{ + // Step 4: b MarkRead（读到最后一条 seq） + markReadReq := &conversation.MarkReadReq{ RequestId: client.NewRequestID(), ConversationId: convID, - SeqId: lastSeq, + LastReadSeq: lastSeq, } - require.NoError(t, b.DoAuth("/service/message/update_read_ack", ackReq, &msg.UpdateReadAckRsp{})) + require.NoError(t, b.DoAuth( + "/service/conversation/mark_read", markReadReq, &conversation.MarkReadRsp{})) // Step 5: b 再次 ListConversations，unread_count=0 listRsp2 := &conversation.ListConversationsRsp{} require.NoError(t, b.DoAuth("/service/conversation/list", listReq, listRsp2)) for _, c := range listRsp2.Conversations { if c.ConversationId == convID { - assert.Equal(t, uint64(0), c.Self.UnreadCount, "read ack 后未读数应清零") + assert.Equal(t, uint64(0), c.Self.UnreadCount, "MarkRead 后未读数应清零") } } diff --git a/tests/perf/cache_helpers_test.go b/tests/perf/cache_helpers_test.go new file mode 100644 index 0000000..de7359e --- /dev/null +++ b/tests/perf/cache_helpers_test.go @@ -0,0 +1,161 @@ +//go:build perf + +package perf_test + +import ( + "fmt" + "math" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "chatnow-tests/pkg/client" +) + +func TestPF09PhaseConversationsRequireUniqueKeys(t *testing.T) { + conversations := make([]pf09Conversation, pf09ConversationCount) + for i := range conversations { + conversations[i] = pf09Conversation{ + sender: &client.HTTPClient{UserID: "user-" + string(rune('a'+i))}, + id: "conversation-" + string(rune('a'+i)), + } + } + if err := validatePF09Conversations(conversations); err != nil { + t.Fatalf("valid phase plan rejected: %v", err) + } + conversations[19].sender.UserID = conversations[0].sender.UserID + if err := validatePF09Conversations(conversations); err == nil || !strings.Contains(err.Error(), "duplicate user-info key") { + t.Fatalf("duplicate phase key not rejected: %v", err) + } +} + +func TestPF09EndpointsRejectEmptyDuplicateAndWrongCount(t *testing.T) { + if _, err := parsePF09Endpoints("http://a,http://b", 2); err != nil { + t.Fatalf("valid endpoints rejected: %v", err) + } + for _, test := range []struct { + raw string + expected int + }{ + {raw: "", expected: 1}, + {raw: "http://a,", expected: 1}, + {raw: "http://a/,http://a", expected: 2}, + {raw: "http://a", expected: 2}, + } { + if _, err := parsePF09Endpoints(test.raw, test.expected); err == nil { + t.Errorf("invalid endpoints accepted: raw=%q expected=%d", test.raw, test.expected) + } + } +} + +func TestPF09SnapshotDeltaRejectsRestartRewindMissingAndOverflow(t *testing.T) { + before := pf09Snapshot{ + "http://a": {pid: 10, uptimeSeconds: 100, counters: map[string]int64{ + pf09RPCMetric: 10, pf09L1Metric: 20, pf09L2Metric: 30, + }}, + } + after := pf09Snapshot{ + "http://a": {pid: 10, uptimeSeconds: 101, counters: map[string]int64{ + pf09RPCMetric: 11, pf09L1Metric: 22, pf09L2Metric: 33, + }}, + } + delta, err := pf09SnapshotDelta(before, after) + if err != nil { + t.Fatalf("valid snapshot rejected: %v", err) + } + if delta[pf09RPCMetric] != 1 || delta[pf09L1Metric] != 2 || delta[pf09L2Metric] != 3 { + t.Fatalf("wrong deltas: %#v", delta) + } + + cases := []pf09Snapshot{ + {}, + {"http://a": {pid: 11, uptimeSeconds: 1, counters: after["http://a"].counters}}, + {"http://a": {pid: 10, uptimeSeconds: 99, counters: after["http://a"].counters}}, + {"http://a": {pid: 10, uptimeSeconds: 101, counters: map[string]int64{ + pf09RPCMetric: 9, pf09L1Metric: 22, pf09L2Metric: 33, + }}}, + } + for i, invalid := range cases { + if _, err := pf09SnapshotDelta(before, invalid); err == nil { + t.Errorf("invalid snapshot %d accepted", i) + } + } + + overflowBefore := pf09Snapshot{ + "http://a": {pid: 1, uptimeSeconds: 1, counters: map[string]int64{pf09RPCMetric: 0, pf09L1Metric: 0, pf09L2Metric: 0}}, + "http://b": {pid: 2, uptimeSeconds: 1, counters: map[string]int64{pf09RPCMetric: 0, pf09L1Metric: 0, pf09L2Metric: 0}}, + } + overflowAfter := pf09Snapshot{ + "http://a": {pid: 1, uptimeSeconds: 2, counters: map[string]int64{pf09RPCMetric: math.MaxInt64, pf09L1Metric: 0, pf09L2Metric: 0}}, + "http://b": {pid: 2, uptimeSeconds: 2, counters: map[string]int64{pf09RPCMetric: 1, pf09L1Metric: 0, pf09L2Metric: 0}}, + } + if _, err := pf09SnapshotDelta(overflowBefore, overflowAfter); err == nil { + t.Fatal("overflowing counter delta accepted") + } +} + +func TestPF09SnapshotRequiresProcessIdentityFields(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + values := map[string]string{ + "/vars/pid": "42", + "/vars/user_info_rpc_total": "1", + "/vars/user_info_l1_hit_total": "2", + "/vars/user_info_l2_hit_total": "3", + } + value, exists := values[r.URL.Path] + if !exists { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprint(w, value) + })) + defer server.Close() + if _, err := capturePF09Snapshot([]string{server.URL}); err == nil || !strings.Contains(err.Error(), "process uptime") { + t.Fatalf("missing process_uptime did not hard fail: %v", err) + } +} + +func TestPF09PhaseCountersProveStableCacheState(t *testing.T) { + valid := []struct { + state pf09CacheState + count uint64 + deltas map[string]int64 + }{ + {pf09Cold, 20, map[string]int64{pf09RPCMetric: 20, pf09L1Metric: 0, pf09L2Metric: 0}}, + {pf09L2, 20, map[string]int64{pf09RPCMetric: 0, pf09L1Metric: 0, pf09L2Metric: 20}}, + {pf09L1, 20, map[string]int64{pf09RPCMetric: 0, pf09L1Metric: 20, pf09L2Metric: 0}}, + {pf09Stampede, 200, map[string]int64{pf09RPCMetric: 2, pf09L1Metric: 198, pf09L2Metric: 0}}, + } + for _, test := range valid { + if err := validatePF09PhaseCounters(test.state, test.count, 2, test.deltas); err != nil { + t.Errorf("valid %s counters rejected: %v", pf09StateName(test.state), err) + } + contaminated := make(map[string]int64, len(test.deltas)) + for metric, value := range test.deltas { + contaminated[metric] = value + } + contaminated[pf09RPCMetric]++ + if err := validatePF09PhaseCounters(test.state, test.count, 2, contaminated); err == nil { + t.Errorf("contaminated %s counters accepted", pf09StateName(test.state)) + } + } +} + +func TestPF09GateDurationCannotBeBypassed(t *testing.T) { + t.Setenv("PF09_GATE_MIN_DURATION", "24h") + if pf09SteadyDuration != 10*time.Second { + t.Fatalf("steady duration changed through environment: %s", pf09SteadyDuration) + } + if err := validatePF09IterationCount(1); err != nil { + t.Fatalf("one-shot invocation rejected: %v", err) + } + if err := validatePF09IterationCount(2); err == nil { + t.Fatal("calibrated invocation accepted; -benchtime=1x is mandatory") + } + if _, present := os.LookupEnv("PF09_GATE_MIN_DURATION"); !present { + t.Fatal("test precondition lost") + } +} diff --git a/tests/perf/cache_test.go b/tests/perf/cache_test.go new file mode 100644 index 0000000..1ada70e --- /dev/null +++ b/tests/perf/cache_test.go @@ -0,0 +1,657 @@ +//go:build perf + +package perf_test + +import ( + "bytes" + "fmt" + "io" + "math" + "net/http" + "os" + "os/exec" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "chatnow-tests/pkg/client" + "chatnow-tests/pkg/fixture" + identity "chatnow-tests/proto/chatnow/identity" + msg "chatnow-tests/proto/chatnow/message" + transmite "chatnow-tests/proto/chatnow/transmite" + "google.golang.org/protobuf/proto" +) + +const ( + pf09ConversationCount = 20 + pf09StampedeConcurrency = 200 + pf09MaxLatencySamples = 65_536 + pf09MinThroughput = 5_000.0 + pf09CheckedInBaseline = 5_000.0 + pf09MinRPCReduction = 95.0 + pf09MaxBaselineRegression = 10.0 + pf09SteadyDuration = 10 * time.Second + pf09SnapshotTimeout = 2 * time.Second + pf09StartEstimateTolerance = 2 * time.Second + + pf09RPCMetric = "user_info_rpc_total" + pf09L1Metric = "user_info_l1_hit_total" + pf09L2Metric = "user_info_l2_hit_total" +) + +var pf09CounterMetrics = [...]string{pf09RPCMetric, pf09L1Metric, pf09L2Metric} + +type pf09CacheState uint8 + +const ( + pf09Cold pf09CacheState = iota + pf09L2 + pf09L1 + pf09Stampede +) + +type pf09Conversation struct { + sender *client.HTTPClient + id string +} + +type pf09InstanceSnapshot struct { + pid int64 + uptimeSeconds float64 + startedAtEstimateNano int64 + counters map[string]int64 +} + +type pf09Snapshot map[string]pf09InstanceSnapshot + +type pf09PhaseResult struct { + elapsed time.Duration + successes uint64 + latencies []int64 + err error +} + +// BenchmarkPF09_UserInfoCache is intentionally a one-shot benchmark harness. +// cold/L2 each execute one concurrent operation for 20 unique sender keys; L1 +// executes a fixed ten-second steady workload over 20 prewarmed senders. This +// prevents Go's adaptive benchmark calibration from changing the cache state. +func BenchmarkPF09_UserInfoCache(b *testing.B) { + if os.Getenv("PF09_RUN_FULLSTACK") != "1" { + b.Skip("PF-09 requires the full service stack; use make test-perf-cache-gate on Linux") + } + if runtime.GOOS != "linux" { + b.Fatalf("PF-09 gate is calibrated for Linux, got %s", runtime.GOOS) + } + if err := validatePF09IterationCount(b.N); err != nil { + b.Fatal(err) + } + + baseline := pf09Baseline(b) + endpoints := pf09ConfiguredEndpoints(b) + for _, phase := range []struct { + name string + state pf09CacheState + }{ + {name: "cold", state: pf09Cold}, + {name: "L2", state: pf09L2}, + {name: "L1", state: pf09L1}, + {name: "stampede", state: pf09Stampede}, + } { + phase := phase + b.Run(phase.name, func(b *testing.B) { + if err := validatePF09IterationCount(b.N); err != nil { + b.Fatal(err) + } + conversations := preparePF09Conversations(b, phase.state, len(endpoints)) + if err := validatePF09Conversations(conversations); err != nil { + b.Fatalf("PF-09 %s phase plan: %v", phase.name, err) + } + runPF09Phase(b, phase.state, conversations, endpoints, baseline) + }) + } +} + +func validatePF09IterationCount(n int) error { + if n != 1 { + return fmt.Errorf("PF-09 requires -benchtime=1x to preserve phase state, got b.N=%d", n) + } + return nil +} + +func preparePF09Conversations(b *testing.B, state pf09CacheState, instanceCount int) []pf09Conversation { + b.Helper() + b.StopTimer() + if state == pf09Stampede { + sender, peer, conversationID := fixture.MakeFriends(b, HTTP) + _ = peer + verifyPF09DeleteL2(b, sender.UserID) + conversations := make([]pf09Conversation, pf09StampedeConcurrency) + for i := range conversations { + conversations[i] = pf09Conversation{sender: sender, id: conversationID} + } + return conversations + } + conversations := make([]pf09Conversation, pf09ConversationCount) + for i := range conversations { + sender, peer, conversationID := fixture.MakeFriends(b, HTTP) + _ = peer + conversations[i] = pf09Conversation{sender: sender, id: conversationID} + if state == pf09L2 || state == pf09L1 { + seedPF09L2(b, sender) + } + } + if state == pf09L1 { + // Gateway dispatch is round-robin. Consecutive instanceCount sends per + // sender warm that sender in every declared Transmite process. + for i, conversation := range conversations { + for instance := 0; instance < instanceCount; instance++ { + sequence := uint64(i*instanceCount + instance) + if err := sendPF09Message(conversation, sequence); err != nil { + b.Fatalf("warm PF-09 L1: %v", err) + } + } + } + } + return conversations +} + +func validatePF09Conversations(conversations []pf09Conversation) error { + if len(conversations) == pf09StampedeConcurrency { + first := conversations[0] + for _, conversation := range conversations { + if conversation.sender == nil || conversation.sender.UserID != first.sender.UserID || conversation.id != first.id { + return fmt.Errorf("stampede phase must target one cold key") + } + } + return nil + } + if len(conversations) != pf09ConversationCount { + return fmt.Errorf("want %d conversations, got %d", pf09ConversationCount, len(conversations)) + } + keys := make(map[string]struct{}, len(conversations)) + conversationIDs := make(map[string]struct{}, len(conversations)) + for _, conversation := range conversations { + if conversation.sender == nil || conversation.sender.UserID == "" || conversation.id == "" { + return fmt.Errorf("empty sender or conversation") + } + key := pf09UserInfoKey(conversation.sender.UserID) + if _, exists := keys[key]; exists { + return fmt.Errorf("duplicate user-info key %q", key) + } + keys[key] = struct{}{} + if _, exists := conversationIDs[conversation.id]; exists { + return fmt.Errorf("duplicate conversation %q", conversation.id) + } + conversationIDs[conversation.id] = struct{}{} + } + return nil +} + +func verifyPF09DeleteL2(b *testing.B, uid string) { + b.Helper() + cmd := exec.Command("docker", "exec", HTTP.Config().Infra.RedisContainer, + "redis-cli", "-c", "DEL", pf09UserInfoKey(uid)) + if out, err := cmd.CombinedOutput(); err != nil { + b.Fatalf("clear PF-09 L2: %v: %s", err, strings.TrimSpace(string(out))) + } +} + +func seedPF09L2(b *testing.B, sender *client.HTTPClient) { + b.Helper() + rsp := &identity.GetProfileRsp{} + if err := sender.DoAuth("/service/identity/get_profile", &identity.GetProfileReq{ + RequestId: client.NewRequestID(), + }, rsp); err != nil { + b.Fatalf("read PF-09 profile: %v", err) + } + if !rsp.GetHeader().GetSuccess() || rsp.GetUserInfo() == nil { + b.Fatalf("read PF-09 profile: %s", rsp.GetHeader().GetErrorMessage()) + } + value, err := proto.Marshal(rsp.GetUserInfo()) + if err != nil { + b.Fatalf("marshal PF-09 profile: %v", err) + } + key := pf09UserInfoKey(sender.UserID) + const seedScript = "return redis.call('SET',KEYS[1],ARGV[2],'EX',ARGV[1])" + set := exec.Command( + "docker", "exec", "-i", HTTP.Config().Infra.RedisContainer, + "redis-cli", "-c", "-x", "EVAL", seedScript, "1", key, "3600", + ) + set.Stdin = bytes.NewReader(value) + if out, err := set.CombinedOutput(); err != nil || strings.TrimSpace(string(out)) != "OK" { + b.Fatalf("seed PF-09 L2 %s: %v: %s", key, err, strings.TrimSpace(string(out))) + } +} + +func runPF09Phase( + b *testing.B, + state pf09CacheState, + conversations []pf09Conversation, + endpoints []string, + baseline float64, +) { + b.Helper() + before, err := capturePF09Snapshot(endpoints) + if err != nil { + b.Fatalf("PF-09 before snapshot: %v", err) + } + + var memoryBefore runtime.MemStats + runtime.ReadMemStats(&memoryBefore) + b.ResetTimer() + var result pf09PhaseResult + if state == pf09L1 { + result = runPF09Steady(conversations) + } else { + result = runPF09SingleUse(conversations) + } + b.StopTimer() + var memoryAfter runtime.MemStats + runtime.ReadMemStats(&memoryAfter) + if result.err != nil { + b.Fatalf("PF-09 %s send failed: %v", pf09StateName(state), result.err) + } + if result.successes == 0 || result.elapsed <= 0 || len(result.latencies) == 0 { + b.Fatalf("PF-09 %s produced no measurable successful requests", pf09StateName(state)) + } + + after, err := capturePF09Snapshot(endpoints) + if err != nil { + b.Fatalf("PF-09 after snapshot: %v", err) + } + deltas, err := pf09SnapshotDelta(before, after) + if err != nil { + b.Fatalf("PF-09 snapshot integrity: %v", err) + } + if err := validatePF09PhaseCounters(state, result.successes, len(endpoints), deltas); err != nil { + b.Error(err) + } + + sort.Slice(result.latencies, func(i, j int) bool { return result.latencies[i] < result.latencies[j] }) + p95Index := int(math.Ceil(float64(len(result.latencies))*0.95)) - 1 + throughput := float64(result.successes) / result.elapsed.Seconds() + p95Micros := float64(result.latencies[p95Index]) / float64(time.Microsecond) + rpcReduction := 100 * (1 - float64(deltas[pf09RPCMetric])/float64(result.successes)) + baselineRegression := 100 * (1 - throughput/baseline) + b.ReportMetric(throughput, "msg/s") + b.ReportMetric(p95Micros, "p95-us") + b.ReportMetric(rpcReduction, "rpc-reduction-%") + b.ReportMetric(baselineRegression, "baseline-regression-%") + b.ReportMetric(float64(memoryAfter.Mallocs-memoryBefore.Mallocs)/float64(result.successes), "allocs/req") + b.ReportMetric(float64(memoryAfter.TotalAlloc-memoryBefore.TotalAlloc)/float64(result.successes), "bytes/req") + + if state == pf09L1 { + if result.elapsed < pf09SteadyDuration { + b.Errorf("PF-09 L1 duration %s is below fixed gate duration %s", result.elapsed, pf09SteadyDuration) + } + if throughput < pf09MinThroughput { + b.Errorf("PF-09 L1 throughput %.2f msg/s is below %.2f msg/s", throughput, pf09MinThroughput) + } + if baselineRegression > pf09MaxBaselineRegression { + b.Errorf("PF-09 L1 throughput regressed %.2f%% from baseline %.2f msg/s", baselineRegression, baseline) + } + } +} + +func runPF09SingleUse(conversations []pf09Conversation) pf09PhaseResult { + start := make(chan struct{}) + results := make(chan struct { + latency int64 + err error + }, len(conversations)) + var wg sync.WaitGroup + for i, conversation := range conversations { + i, conversation := i, conversation + wg.Add(1) + go func() { + defer wg.Done() + <-start + requestStarted := time.Now() + err := sendPF09Message(conversation, uint64(i)) + results <- struct { + latency int64 + err error + }{time.Since(requestStarted).Nanoseconds(), err} + }() + } + started := time.Now() + close(start) + wg.Wait() + elapsed := time.Since(started) + close(results) + result := pf09PhaseResult{elapsed: elapsed, latencies: make([]int64, 0, len(conversations))} + for item := range results { + if item.err != nil && result.err == nil { + result.err = item.err + } + if item.err == nil { + result.successes++ + result.latencies = append(result.latencies, item.latency) + } + } + return result +} + +func runPF09Steady(conversations []pf09Conversation) pf09PhaseResult { + workers := max(pf09ConversationCount, runtime.GOMAXPROCS(0)*4) + perWorkerSamples := max(1, pf09MaxLatencySamples/workers) + start := make(chan struct{}) + deadline := time.Time{} + var sequence atomic.Uint64 + var successes atomic.Uint64 + var stopped atomic.Bool + var firstError error + var errorMu sync.Mutex + latencies := make([]int64, 0, pf09MaxLatencySamples) + var latencyMu sync.Mutex + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + worker := worker + wg.Add(1) + go func() { + defer wg.Done() + local := make([]int64, 0, perWorkerSamples) + var seen uint64 + randomState := uint64(worker+1) | 1 + <-start + for !stopped.Load() && time.Now().Before(deadline) { + id := sequence.Add(1) - 1 + conversation := conversations[id%uint64(len(conversations))] + requestStarted := time.Now() + err := sendPF09Message(conversation, id) + latency := time.Since(requestStarted).Nanoseconds() + if err != nil { + errorMu.Lock() + if firstError == nil { + firstError = err + stopped.Store(true) + } + errorMu.Unlock() + break + } + successes.Add(1) + seen++ + local, randomState = pf09ReservoirAdd(local, perWorkerSamples, seen, randomState, latency) + } + latencyMu.Lock() + remaining := pf09MaxLatencySamples - len(latencies) + if len(local) > remaining { + local = local[:remaining] + } + latencies = append(latencies, local...) + latencyMu.Unlock() + }() + } + started := time.Now() + deadline = started.Add(pf09SteadyDuration) + close(start) + wg.Wait() + return pf09PhaseResult{ + elapsed: time.Since(started), + successes: successes.Load(), + latencies: latencies, + err: firstError, + } +} + +func pf09ReservoirAdd(samples []int64, capacity int, seen, state uint64, value int64) ([]int64, uint64) { + if len(samples) < capacity { + return append(samples, value), state + } + state ^= state << 13 + state ^= state >> 7 + state ^= state << 17 + if replacement := state % seen; replacement < uint64(len(samples)) { + samples[replacement] = value + } + return samples, state +} + +func validatePF09PhaseCounters(state pf09CacheState, successes uint64, instanceCount int, deltas map[string]int64) error { + if successes > math.MaxInt64 { + return fmt.Errorf("PF-09 success count overflows int64") + } + want := int64(successes) + switch state { + case pf09Cold: + if deltas[pf09RPCMetric] != want || deltas[pf09L1Metric] != 0 || deltas[pf09L2Metric] != 0 { + return fmt.Errorf("PF-09 cold state contaminated: success=%d rpc=%d l1=%d l2=%d", want, deltas[pf09RPCMetric], deltas[pf09L1Metric], deltas[pf09L2Metric]) + } + case pf09L2: + if deltas[pf09L2Metric] != want || deltas[pf09L1Metric] != 0 || deltas[pf09RPCMetric] != 0 { + return fmt.Errorf("PF-09 L2 state contaminated: success=%d rpc=%d l1=%d l2=%d", want, deltas[pf09RPCMetric], deltas[pf09L1Metric], deltas[pf09L2Metric]) + } + case pf09L1: + if deltas[pf09L1Metric] != want || deltas[pf09L2Metric] != 0 || deltas[pf09RPCMetric] != 0 { + return fmt.Errorf("PF-09 L1 state contaminated: success=%d rpc=%d l1=%d l2=%d", want, deltas[pf09RPCMetric], deltas[pf09L1Metric], deltas[pf09L2Metric]) + } + if 100*(1-float64(deltas[pf09RPCMetric])/float64(want)) < pf09MinRPCReduction { + return fmt.Errorf("PF-09 L1 RPC reduction below %.0f%%", pf09MinRPCReduction) + } + case pf09Stampede: + rpc := deltas[pf09RPCMetric] + if rpc < 1 || rpc > int64(instanceCount) || + rpc+deltas[pf09L1Metric]+deltas[pf09L2Metric] != want { + return fmt.Errorf("PF-09 stampede invariant failed: success=%d instances=%d rpc=%d l1=%d l2=%d", + want, instanceCount, rpc, deltas[pf09L1Metric], deltas[pf09L2Metric]) + } + default: + return fmt.Errorf("unknown PF-09 state %d", state) + } + return nil +} + +func pf09ConfiguredEndpoints(b *testing.B) []string { + b.Helper() + raw := os.Getenv("PF09_TRANSMITE_VARS_URLS") + if raw == "" { + raw = HTTP.Config().Infra.TransmiteVars + } + value := os.Getenv("PF09_EXPECTED_TRANSMITE_INSTANCES") + if value == "" { + b.Fatal("PF09_EXPECTED_TRANSMITE_INSTANCES is required by the full-stack gate") + } + expected, err := strconv.Atoi(value) + if err != nil || expected <= 0 { + b.Fatalf("PF09_EXPECTED_TRANSMITE_INSTANCES must be positive, got %q", value) + } + endpoints, err := parsePF09Endpoints(raw, expected) + if err != nil { + b.Fatal(err) + } + return endpoints +} + +func parsePF09Endpoints(raw string, expected int) ([]string, error) { + parts := strings.Split(raw, ",") + endpoints := make([]string, 0, len(parts)) + seen := make(map[string]struct{}, len(parts)) + for _, part := range parts { + endpoint := strings.TrimRight(strings.TrimSpace(part), "/") + if endpoint == "" { + return nil, fmt.Errorf("PF-09 contains an empty Transmite bvar endpoint") + } + if _, duplicate := seen[endpoint]; duplicate { + return nil, fmt.Errorf("PF-09 duplicate Transmite bvar endpoint %q", endpoint) + } + seen[endpoint] = struct{}{} + endpoints = append(endpoints, endpoint) + } + if len(endpoints) != expected { + return nil, fmt.Errorf("PF-09 expected %d Transmite instances, got %d", expected, len(endpoints)) + } + return endpoints, nil +} + +func capturePF09Snapshot(endpoints []string) (pf09Snapshot, error) { + snapshot := make(pf09Snapshot, len(endpoints)) + for _, endpoint := range endpoints { + pid, err := readPF09Int(endpoint, "pid") + if err != nil || pid <= 0 { + return nil, fmt.Errorf("%s process identity: pid=%d err=%w", endpoint, pid, err) + } + uptime, err := readPF09Float(endpoint, "process_uptime") + if err != nil || uptime < 0 { + return nil, fmt.Errorf("%s process uptime: uptime=%f err=%w", endpoint, uptime, err) + } + instance := pf09InstanceSnapshot{ + pid: pid, + uptimeSeconds: uptime, + startedAtEstimateNano: time.Now().Add(-time.Duration(uptime * float64(time.Second))).UnixNano(), + counters: make(map[string]int64, len(pf09CounterMetrics)), + } + for _, metric := range pf09CounterMetrics { + value, err := readPF09Int(endpoint, metric) + if err != nil || value < 0 { + return nil, fmt.Errorf("%s %s: value=%d err=%w", endpoint, metric, value, err) + } + instance.counters[metric] = value + } + snapshot[endpoint] = instance + } + return snapshot, nil +} + +func readPF09Int(endpoint, name string) (int64, error) { + raw, err := readPF09BVar(endpoint, name) + if err != nil { + return 0, err + } + return strconv.ParseInt(strings.Trim(raw, "\""), 10, 64) +} + +func readPF09Float(endpoint, name string) (float64, error) { + raw, err := readPF09BVar(endpoint, name) + if err != nil { + return 0, err + } + value, err := strconv.ParseFloat(strings.Trim(raw, "\""), 64) + if err != nil || math.IsNaN(value) || math.IsInf(value, 0) { + return 0, fmt.Errorf("parse %s=%q", name, raw) + } + return value, nil +} + +func readPF09BVar(endpoint, name string) (string, error) { + client := &http.Client{Timeout: pf09SnapshotTimeout} + rsp, err := client.Get(endpoint + "/vars/" + name) + if err != nil { + return "", err + } + defer rsp.Body.Close() + body, err := io.ReadAll(io.LimitReader(rsp.Body, 4096)) + if err != nil { + return "", err + } + if rsp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP %d: %s", rsp.StatusCode, strings.TrimSpace(string(body))) + } + value := strings.TrimSpace(string(body)) + if value == "" { + return "", fmt.Errorf("empty bvar %s", name) + } + return value, nil +} + +func pf09SnapshotDelta(before, after pf09Snapshot) (map[string]int64, error) { + if len(before) == 0 || len(before) != len(after) { + return nil, fmt.Errorf("snapshot endpoint set changed: before=%d after=%d", len(before), len(after)) + } + totals := make(map[string]int64, len(pf09CounterMetrics)) + for endpoint, old := range before { + current, exists := after[endpoint] + if !exists { + return nil, fmt.Errorf("snapshot endpoint %s disappeared", endpoint) + } + if old.pid != current.pid { + return nil, fmt.Errorf("snapshot endpoint %s restarted: pid %d -> %d", endpoint, old.pid, current.pid) + } + if current.uptimeSeconds < old.uptimeSeconds { + return nil, fmt.Errorf("snapshot endpoint %s uptime rewound: %f -> %f", endpoint, old.uptimeSeconds, current.uptimeSeconds) + } + startDrift := time.Duration(current.startedAtEstimateNano - old.startedAtEstimateNano) + if startDrift < -pf09StartEstimateTolerance || startDrift > pf09StartEstimateTolerance { + return nil, fmt.Errorf("snapshot endpoint %s process start estimate changed by %s", endpoint, startDrift) + } + for _, metric := range pf09CounterMetrics { + oldValue, oldOK := old.counters[metric] + newValue, newOK := current.counters[metric] + if !oldOK || !newOK { + return nil, fmt.Errorf("snapshot endpoint %s missing %s", endpoint, metric) + } + if newValue < oldValue { + return nil, fmt.Errorf("snapshot endpoint %s counter %s rewound: %d -> %d", endpoint, metric, oldValue, newValue) + } + delta := newValue - oldValue + if delta > math.MaxInt64-totals[metric] { + return nil, fmt.Errorf("snapshot counter %s delta overflow", metric) + } + totals[metric] += delta + } + } + return totals, nil +} + +func sendPF09Message(conversation pf09Conversation, sequence uint64) error { + rsp := &transmite.SendMessageRsp{} + err := conversation.sender.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), + ConversationId: conversation.id, + Content: &msg.MessageContent{ + Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: fmt.Sprintf("pf09-%d", sequence)}}, + }, + ClientMsgId: client.NewRequestID(), + }, rsp) + if err != nil { + return err + } + if !rsp.GetHeader().GetSuccess() { + return fmt.Errorf("code=%d message=%s", rsp.GetHeader().GetErrorCode(), rsp.GetHeader().GetErrorMessage()) + } + return nil +} + +func pf09Baseline(b *testing.B) float64 { + b.Helper() + value := pf09CheckedInBaseline + if raw := os.Getenv("PF09_BASELINE_MSG_PER_SEC"); raw != "" { + parsed, err := strconv.ParseFloat(raw, 64) + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) || parsed < pf09CheckedInBaseline { + b.Fatalf("PF09_BASELINE_MSG_PER_SEC must be finite and >= %.0f, got %q", pf09CheckedInBaseline, raw) + } + value = parsed + } + return value +} + +func pf09UserInfoKey(uid string) string { + const offset32 = uint32(2166136261) + const prime32 = uint32(16777619) + hash := offset32 + for i := 0; i < len(uid); i++ { + hash ^= uint32(uid[i]) + hash *= prime32 + } + return fmt.Sprintf("im:user:{%d}:%s", hash%64, uid) +} + +func pf09StateName(state pf09CacheState) string { + switch state { + case pf09Cold: + return "cold" + case pf09L2: + return "L2" + case pf09L1: + return "L1" + case pf09Stampede: + return "stampede" + default: + return "unknown" + } +} diff --git a/tests/perf/setup_test.go b/tests/perf/setup_test.go index d2318dc..4e73894 100644 --- a/tests/perf/setup_test.go +++ b/tests/perf/setup_test.go @@ -12,7 +12,7 @@ import ( var HTTP *client.HTTPClient func TestMain(m *testing.M) { - cfg := client.LoadConfig("") + cfg := client.LoadConfig("../config.yaml") HTTP = client.NewHTTPClient(cfg) os.Exit(m.Run()) } diff --git a/tests/pkg/chaos/redis.go b/tests/pkg/chaos/redis.go new file mode 100644 index 0000000..d136862 --- /dev/null +++ b/tests/pkg/chaos/redis.go @@ -0,0 +1,59 @@ +package chaos + +import ( + "os/exec" + "strings" + "testing" + "time" + + "chatnow-tests/pkg/client" +) + +var redisServices = []string{ + "redis-node1", "redis-node2", "redis-node3", + "redis-node4", "redis-node5", "redis-node6", +} + +func compose(t testing.TB, cfg *client.Config, args ...string) { + t.Helper() + cmd := exec.Command("docker", append([]string{"compose"}, args...)...) + cmd.Dir = cfg.Infra.ComposeDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("docker compose %v: %v: %s", args, err, out) + } +} + +func StopRedisCluster(t testing.TB, cfg *client.Config) { + compose(t, cfg, append([]string{"stop"}, redisServices...)...) +} + +func StartRedisCluster(t testing.TB, cfg *client.Config) { + compose(t, cfg, append([]string{"start"}, redisServices...)...) +} + +func WaitRedisCluster(t testing.TB, cfg *client.Config, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + cmd := exec.Command("docker", "exec", cfg.Infra.RedisContainer, "redis-cli", "cluster", "info") + if out, err := cmd.Output(); err == nil && clusterHealthy(string(out)) { + return + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("redis cluster did not recover within %s", timeout) +} + +func clusterHealthy(info string) bool { + values := make(map[string]string) + for _, line := range strings.Split(strings.ReplaceAll(info, "\r", ""), "\n") { + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + values[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) + } + } + return values["cluster_state"] == "ok" && + values["cluster_slots_assigned"] == "16384" && + values["cluster_slots_ok"] == "16384" && + values["cluster_slots_fail"] == "0" +} diff --git a/tests/pkg/chaos/redis_test.go b/tests/pkg/chaos/redis_test.go new file mode 100644 index 0000000..709b432 --- /dev/null +++ b/tests/pkg/chaos/redis_test.go @@ -0,0 +1,19 @@ +package chaos + +import "testing" + +func TestClusterHealthyRequiresCompleteSlotCoverage(t *testing.T) { + healthy := "cluster_state:ok\r\ncluster_slots_assigned:16384\r\ncluster_slots_ok:16384\r\ncluster_slots_fail:0\r\n" + if !clusterHealthy(healthy) { + t.Fatal("complete healthy cluster rejected") + } + for _, unhealthy := range []string{ + "cluster_state:fail\ncluster_slots_assigned:16384\ncluster_slots_ok:16384\ncluster_slots_fail:0\n", + "cluster_state:ok\ncluster_slots_assigned:12000\ncluster_slots_ok:12000\ncluster_slots_fail:0\n", + "cluster_state:ok\ncluster_slots_assigned:16384\ncluster_slots_ok:16000\ncluster_slots_fail:384\n", + } { + if clusterHealthy(unhealthy) { + t.Fatalf("unhealthy cluster accepted: %q", unhealthy) + } + } +} diff --git a/tests/pkg/client/config.go b/tests/pkg/client/config.go index 6a25d5f..394e486 100644 --- a/tests/pkg/client/config.go +++ b/tests/pkg/client/config.go @@ -2,6 +2,7 @@ package client import ( "os" + "strconv" "gopkg.in/yaml.v3" ) @@ -11,6 +12,7 @@ type Config struct { Timeout TimeoutConfig `yaml:"timeout"` Database DatabaseConfig `yaml:"database"` Log LogConfig `yaml:"log"` + Infra InfraConfig `yaml:"infra"` } type TargetConfig struct { @@ -33,6 +35,15 @@ type LogConfig struct { Level string `yaml:"level"` } +type InfraConfig struct { + RedisContainer string `yaml:"redis_container"` + PushContainer string `yaml:"push_container"` + PushVars string `yaml:"push_vars"` + PushRouteL1TTLSec int `yaml:"push_route_l1_ttl_sec"` + ComposeDir string `yaml:"compose_dir"` + TransmiteVars string `yaml:"transmite_vars"` +} + func LoadConfig(path string) *Config { if path == "" { path = "config.yaml" @@ -58,5 +69,27 @@ func LoadConfig(path string) *Config { if v := os.Getenv("ES_URL"); v != "" { cfg.Database.ESURL = v } + if v := os.Getenv("REDIS_CONTAINER"); v != "" { + cfg.Infra.RedisContainer = v + } + if v := os.Getenv("PUSH_CONTAINER"); v != "" { + cfg.Infra.PushContainer = v + } + if v := os.Getenv("PUSH_VARS"); v != "" { + cfg.Infra.PushVars = v + } + if v := os.Getenv("PUSH_ROUTE_L1_TTL_SEC"); v != "" { + ttl, err := strconv.Atoi(v) + if err != nil { + panic("invalid PUSH_ROUTE_L1_TTL_SEC: " + err.Error()) + } + cfg.Infra.PushRouteL1TTLSec = ttl + } + if v := os.Getenv("COMPOSE_DIR"); v != "" { + cfg.Infra.ComposeDir = v + } + if v := os.Getenv("TRANSMITE_VARS"); v != "" { + cfg.Infra.TransmiteVars = v + } return cfg } diff --git a/tests/pkg/client/http.go b/tests/pkg/client/http.go index 44a8b61..167c39a 100644 --- a/tests/pkg/client/http.go +++ b/tests/pkg/client/http.go @@ -57,13 +57,23 @@ func (c *HTTPClient) DoWithTrace(path string, req proto.Message, resp proto.Mess return c.do(path, req, resp, accessToken, traceID) } +// DoProtobufURL sends a protobuf request directly to an absolute service URL. +func (c *HTTPClient) DoProtobufURL(endpoint string, req proto.Message, resp proto.Message) error { + _, err := c.doURL(endpoint, req, resp, "", "") + return err +} + func (c *HTTPClient) do(path string, req proto.Message, resp proto.Message, accessToken, traceID string) (http.Header, error) { + return c.doURL(c.baseURL+path, req, resp, accessToken, traceID) +} + +func (c *HTTPClient) doURL(endpoint string, req proto.Message, resp proto.Message, accessToken, traceID string) (http.Header, error) { body, err := proto.Marshal(req) if err != nil { return nil, fmt.Errorf("marshal request: %w", err) } - httpReq, err := http.NewRequest("POST", c.baseURL+path, bytes.NewReader(body)) + httpReq, err := http.NewRequest("POST", endpoint, bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("create request: %w", err) } diff --git a/tests/pkg/client/websocket.go b/tests/pkg/client/websocket.go new file mode 100644 index 0000000..b41c36f --- /dev/null +++ b/tests/pkg/client/websocket.go @@ -0,0 +1,205 @@ +package client + +import ( + "bufio" + "crypto/rand" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" + + "google.golang.org/protobuf/encoding/protowire" +) + +type WebSocket struct { + conn net.Conn + reader *bufio.Reader +} + +func OpenWebSocket(cfg *Config) (*WebSocket, error) { + addr := cfg.Target.WebsocketAddr + conn, err := net.DialTimeout("tcp", addr, 5*time.Second) + if err != nil { + return nil, err + } + fail := func(err error) (*WebSocket, error) { _ = conn.Close(); return nil, err } + if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + return fail(err) + } + keyBytes := make([]byte, 16) + if _, err := rand.Read(keyBytes); err != nil { + return fail(err) + } + req := &http.Request{ + Method: "GET", URL: &url.URL{Scheme: "http", Host: addr, Path: "/"}, Host: addr, + Header: http.Header{"Connection": {"Upgrade"}, "Upgrade": {"websocket"}, + "Sec-Websocket-Key": {base64.StdEncoding.EncodeToString(keyBytes)}, + "Sec-Websocket-Version": {"13"}}, + } + if err := req.Write(conn); err != nil { + return fail(err) + } + reader := bufio.NewReader(conn) + rsp, err := http.ReadResponse(reader, req) + if err != nil { + return fail(err) + } + if rsp.StatusCode != http.StatusSwitchingProtocols { + return fail(fmt.Errorf("websocket upgrade status %d", rsp.StatusCode)) + } + if err := conn.SetDeadline(time.Time{}); err != nil { + return fail(err) + } + return &WebSocket{conn: conn, reader: reader}, nil +} + +func (ws *WebSocket) Close() error { return ws.conn.Close() } + +func (ws *WebSocket) WriteBinary(payload []byte) error { + return ws.writeFrame(0x2, payload) +} + +func (ws *WebSocket) writeFrame(opcode byte, payload []byte) error { + frame := []byte{0x80 | opcode} + switch { + case len(payload) < 126: + frame = append(frame, 0x80|byte(len(payload))) + case len(payload) <= 65535: + frame = append(frame, 0x80|126, byte(len(payload)>>8), byte(len(payload))) + default: + frame = append(frame, 0x80|127) + var size [8]byte + binary.BigEndian.PutUint64(size[:], uint64(len(payload))) + frame = append(frame, size[:]...) + } + var mask [4]byte + if _, err := rand.Read(mask[:]); err != nil { + return err + } + frame = append(frame, mask[:]...) + for i, b := range payload { + frame = append(frame, b^mask[i%len(mask)]) + } + _, err := ws.conn.Write(frame) + return err +} + +func (ws *WebSocket) ReadFrame(timeout time.Duration) ([]byte, error) { + if err := ws.conn.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return nil, err + } + var message []byte + continuing := false + for { + fin, opcode, payload, err := ws.readRawFrame() + if err != nil { + if continuing { + _ = ws.Close() + } + return nil, err + } + switch opcode { + case 0x8: + _ = ws.Close() + return nil, io.EOF + case 0x9: + if err := ws.writeFrame(0xA, payload); err != nil { + return nil, err + } + continue + case 0xA: + continue + case 0x1, 0x2: + if continuing { + _ = ws.Close() + return nil, fmt.Errorf("new data frame during continuation") + } + message = append(message, payload...) + if fin { + return message, nil + } + continuing = true + case 0x0: + if !continuing { + _ = ws.Close() + return nil, fmt.Errorf("unexpected continuation frame") + } + message = append(message, payload...) + if fin { + return message, nil + } + default: + _ = ws.Close() + return nil, fmt.Errorf("unsupported websocket opcode %d", opcode) + } + } +} + +func (ws *WebSocket) readRawFrame() (bool, byte, []byte, error) { + var header [2]byte + n, err := io.ReadFull(ws.reader, header[:]) + if err != nil { + if n != 0 { + _ = ws.Close() + } + return false, 0, nil, err + } + fin, opcode := header[0]&0x80 != 0, header[0]&0x0f + masked := header[1]&0x80 != 0 + if masked { + _ = ws.Close() + return false, 0, nil, fmt.Errorf("masked websocket server frame") + } + lengthCode := header[1] & 0x7f + if opcode&0x8 != 0 && (!fin || lengthCode > 125) { + _ = ws.Close() + return false, 0, nil, fmt.Errorf("invalid websocket control frame") + } + length := uint64(lengthCode) + switch length { + case 126: + var size [2]byte + if _, err := io.ReadFull(ws.reader, size[:]); err != nil { + _ = ws.Close() + return false, 0, nil, err + } + length = uint64(binary.BigEndian.Uint16(size[:])) + case 127: + var size [8]byte + if _, err := io.ReadFull(ws.reader, size[:]); err != nil { + _ = ws.Close() + return false, 0, nil, err + } + length = binary.BigEndian.Uint64(size[:]) + } + if length > 1<<20 { + _ = ws.Close() + return false, 0, nil, fmt.Errorf("websocket frame too large: %d", length) + } + payload := make([]byte, length) + if _, err := io.ReadFull(ws.reader, payload); err != nil { + _ = ws.Close() + return false, 0, nil, err + } + return fin, opcode, payload, nil +} + +func PushAuthNotify(accessToken, deviceID string) []byte { + var auth []byte + auth = appendProtoString(auth, 1, accessToken) + auth = appendProtoString(auth, 2, deviceID) + var notify []byte + notify = protowire.AppendTag(notify, 2, protowire.VarintType) + notify = protowire.AppendVarint(notify, 49) + notify = protowire.AppendTag(notify, 10, protowire.BytesType) + return protowire.AppendBytes(notify, auth) +} + +func appendProtoString(dst []byte, field protowire.Number, value string) []byte { + dst = protowire.AppendTag(dst, field, protowire.BytesType) + return protowire.AppendString(dst, value) +} diff --git a/tests/pkg/client/websocket_test.go b/tests/pkg/client/websocket_test.go new file mode 100644 index 0000000..e933430 --- /dev/null +++ b/tests/pkg/client/websocket_test.go @@ -0,0 +1,140 @@ +package client + +import ( + "bufio" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +func serverFrame(fin bool, opcode byte, payload string) []byte { + first := opcode + if fin { + first |= 0x80 + } + return append([]byte{first, byte(len(payload))}, []byte(payload)...) +} + +func TestWebSocketPreservesFrameBufferedWithUpgrade(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hijacker := w.(http.Hijacker) + conn, rw, err := hijacker.Hijack() + if err != nil { + return + } + defer conn.Close() + _, _ = rw.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + _, _ = rw.Write(serverFrame(true, 2, "buffered")) + _ = rw.Flush() + time.Sleep(100 * time.Millisecond) + })) + defer server.Close() + u, _ := url.Parse(server.URL) + ws, err := OpenWebSocket(&Config{Target: TargetConfig{WebsocketAddr: u.Host}}) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + payload, err := ws.ReadFrame(time.Second) + if err != nil || string(payload) != "buffered" { + t.Fatalf("payload=%q err=%v", payload, err) + } +} + +func pipeWebSocket(t *testing.T, frames ...[]byte) *WebSocket { + t.Helper() + ws, serverConn := newPipeWebSocket(t) + go func() { + for _, frame := range frames { + _, _ = serverConn.Write(frame) + if len(frame) >= 2 && frame[0]&0x0f == 9 { + pong := make([]byte, 7) // FIN+PONG, masked one-byte payload. + _, _ = io.ReadFull(serverConn, pong) + } + } + }() + return ws +} + +func newPipeWebSocket(t *testing.T) (*WebSocket, net.Conn) { + t.Helper() + clientConn, serverConn := net.Pipe() + t.Cleanup(func() { _ = clientConn.Close(); _ = serverConn.Close() }) + return &WebSocket{conn: clientConn, reader: bufio.NewReader(clientConn)}, serverConn +} + +func requirePeerClosed(t *testing.T, peer net.Conn) { + t.Helper() + _ = peer.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)) + _, err := peer.Write(serverFrame(true, 2, "probe")) + if err == nil { + t.Fatal("websocket peer remained writable") + } + if timeout, ok := err.(net.Error); ok && timeout.Timeout() { + t.Fatalf("websocket peer was not closed: %v", err) + } +} + +func TestWebSocketSkipsPingBeforeData(t *testing.T) { + ws := pipeWebSocket(t, serverFrame(true, 9, "p"), serverFrame(true, 2, "data")) + payload, err := ws.ReadFrame(time.Second) + if err != nil || string(payload) != "data" { + t.Fatalf("payload=%q err=%v", payload, err) + } +} + +func TestWebSocketReassemblesFragmentsAcrossControlFrame(t *testing.T) { + ws := pipeWebSocket(t, + serverFrame(false, 2, "frag-"), + serverFrame(true, 9, "p"), + serverFrame(true, 0, "ment"), + ) + payload, err := ws.ReadFrame(time.Second) + if err != nil || string(payload) != "frag-ment" { + t.Fatalf("payload=%q err=%v", payload, err) + } +} + +func TestWebSocketTimeoutBetweenFragmentsClosesConnection(t *testing.T) { + ws, serverConn := newPipeWebSocket(t) + go func() { _, _ = serverConn.Write(serverFrame(false, 2, "partial")) }() + _, err := ws.ReadFrame(30 * time.Millisecond) + if timeout, ok := err.(net.Error); !ok || !timeout.Timeout() { + t.Fatalf("expected fragment timeout, got %v", err) + } + requirePeerClosed(t, serverConn) +} + +func TestWebSocketRejectsMaskedServerFrame(t *testing.T) { + ws, serverConn := newPipeWebSocket(t) + masked := []byte{0x82, 0x81, 1, 2, 3, 4, 'x' ^ 1} + go func() { _, _ = serverConn.Write(masked) }() + _, err := ws.ReadFrame(time.Second) + if err == nil || !strings.Contains(err.Error(), "masked") { + t.Fatalf("expected masked-server error, got %v", err) + } + requirePeerClosed(t, serverConn) +} + +func TestWebSocketRejectsInvalidControlFrames(t *testing.T) { + tests := map[string][]byte{ + "fragmented": serverFrame(false, 9, "p"), + "oversized": {0x89, 126, 0, 126}, + } + for name, frame := range tests { + t.Run(name, func(t *testing.T) { + ws, serverConn := newPipeWebSocket(t) + go func() { _, _ = serverConn.Write(frame) }() + _, err := ws.ReadFrame(time.Second) + if err == nil || !strings.Contains(err.Error(), "control") { + t.Fatalf("expected control-frame error, got %v", err) + } + requirePeerClosed(t, serverConn) + }) + } +} diff --git a/tests/pkg/contracts/artifacts_test.go b/tests/pkg/contracts/artifacts_test.go new file mode 100644 index 0000000..2256513 --- /dev/null +++ b/tests/pkg/contracts/artifacts_test.go @@ -0,0 +1,458 @@ +package contracts + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +var composeArtifactServices = []string{ + "conversation", "gateway", "identity", "media", "message", + "presence", "push", "relationship", "transmite", +} + +func TestComposeArtifactBuilderIsLocked(t *testing.T) { + root := repositoryRoot(t) + dockerfile := readContractFile(t, root, "docker/ci/Dockerfile") + lockfile := readContractFile(t, root, "docker/ci/dependencies.lock") + + imageLock := regexp.MustCompile(`(?m)^UBUNTU_IMAGE=(ubuntu:24\.04@sha256:[0-9a-f]{64})$`).FindStringSubmatch(lockfile) + require.Len(t, imageLock, 2, "the Ubuntu builder image must be locked by tag and digest") + require.Contains(t, dockerfile, "ARG UBUNTU_IMAGE="+imageLock[1]) + require.Contains(t, dockerfile, "FROM ${UBUNTU_IMAGE}") + require.Contains(t, dockerfile, "COPY docker/ci/dependencies.lock") + require.Contains(t, dockerfile, ". /tmp/dependencies.lock") + require.NotContains(t, strings.ToLower(dockerfile), ":latest") + require.NotRegexp(t, regexp.MustCompile(`(?m)git (clone|checkout).*(main|master)(\s|$)`), dockerfile) + require.Contains(t, dockerfile, "git -C /tmp/aws-sdk-cpp submodule update --init --recursive --depth 1") + require.NotContains(t, dockerfile, "submodule update --remote") + + for _, dependency := range []string{ + "BRPC_REV", "ETCD_CPP_APIV3_REV", "REDIS_PLUS_PLUS_REV", "CPR_REV", + "ELASTICLIENT_REV", "AMQP_CPP_REV", "AWS_SDK_CPP_REV", + } { + revision := regexp.MustCompile(`(?m)^` + dependency + `=([0-9a-f]{40})$`).FindStringSubmatch(lockfile) + require.Len(t, revision, 2, "%s must be an immutable 40-character Git revision", dependency) + require.Contains(t, dockerfile, `"$`+dependency+`"`, "%s must be consumed by the builder", dependency) + } + require.Contains(t, dockerfile, "ldconfig") + + for _, service := range composeArtifactServices { + runtimeDockerfile := readContractFile(t, root, service+"/Dockerfile") + require.Contains(t, runtimeDockerfile, "ARG UBUNTU_IMAGE="+imageLock[1], + "%s runtime must use the builder's exact Ubuntu digest", service) + require.Contains(t, runtimeDockerfile, "FROM ${UBUNTU_IMAGE}") + require.Contains(t, runtimeDockerfile, "COPY ./depends/ /im/depends/") + require.Contains(t, runtimeDockerfile, "LD_LIBRARY_PATH=/im/depends") + require.NotContains(t, runtimeDockerfile, "COPY ./depends/* /lib", + "%s must not overwrite distribution libraries", service) + } +} + +func TestMediaUsesDiscoveredJsonCppTarget(t *testing.T) { + root := repositoryRoot(t) + cmake := readContractFile(t, root, "media/CMakeLists.txt") + require.Contains(t, cmake, "find_package(jsoncpp CONFIG REQUIRED)") + require.Contains(t, cmake, "jsoncpp_lib") + require.NotContains(t, cmake, "/usr/local/lib/libjsoncpp.so.19") + require.NotRegexp(t, regexp.MustCompile(`/usr/(local/)?lib[^\s)]*libjsoncpp`), cmake) +} + +func TestComposeArtifactPackagerContract(t *testing.T) { + root := repositoryRoot(t) + scriptPath := filepath.Join(root, "scripts/package_compose_artifacts.sh") + script := readContractFile(t, root, "scripts/package_compose_artifacts.sh") + + require.Contains(t, script, "set -euo pipefail") + assertExplicitArtifactServices(t, script) + require.Contains(t, script, `build_root="${1:-build}"`) + require.Contains(t, script, `artifact_root="${2:-compose-artifacts}"`) + require.Contains(t, script, `"$build_root/$service/${service}_server"`) + require.Contains(t, script, `[[ -x "$binary" ]]`) + require.Contains(t, script, "not found") + require.Contains(t, script, "compose_artifact_core_abi.sh") + require.Contains(t, script, "is_core_system_abi") + require.Contains(t, script, "sha256sum") + require.Contains(t, script, "sort -z") + + t.Run("rejects missing service binary", func(t *testing.T) { + buildRoot := filepath.Join(t.TempDir(), "build") + result := exec.Command("bash", scriptPath, buildRoot, filepath.Join(t.TempDir(), "artifacts")) + output, err := result.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "missing executable service binary") + }) + + t.Run("rejects unresolved ldd dependency", func(t *testing.T) { + temp := t.TempDir() + buildRoot := filepath.Join(temp, "build") + for _, service := range composeArtifactServices { + binary := filepath.Join(buildRoot, service, service+"_server") + require.NoError(t, os.MkdirAll(filepath.Dir(binary), 0o755)) + require.NoError(t, os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755)) + } + binDir := filepath.Join(temp, "bin") + require.NoError(t, os.MkdirAll(binDir, 0o755)) + ldd := filepath.Join(binDir, "ldd") + require.NoError(t, os.WriteFile(ldd, []byte("#!/bin/sh\necho 'libmissing.so => not found'\n"), 0o755)) + + result := exec.Command("bash", scriptPath, buildRoot, filepath.Join(temp, "artifacts")) + result.Env = append(os.Environ(), "PATH="+binDir+":"+os.Getenv("PATH")) + output, err := result.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "unresolved shared library") + }) + + t.Run("rejects conflicting libraries with the same artifact basename", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + first := filepath.Join(fixture.temp, "first", "libcollision.so") + second := filepath.Join(fixture.temp, "second", "libcollision.so") + require.NoError(t, os.MkdirAll(filepath.Dir(first), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Dir(second), 0o755)) + require.NoError(t, os.WriteFile(first, []byte("first\n"), 0o644)) + require.NoError(t, os.WriteFile(second, []byte("second\n"), 0o644)) + fixture.lddPath = fixture.writeLDDLibraries(t, filepath.Join(fixture.tools, "ldd-collision"), first, second) + + command := exec.Command("bash", scriptPath, fixture.buildRoot, fixture.artifactRoot) + command.Env = append(os.Environ(), "LDD="+fixture.lddPath, "PATH="+fixture.tools+":"+os.Getenv("PATH")) + output, err := command.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "conflicting shared libraries share artifact basename") + }) +} + +func TestComposeArtifactValidatorContract(t *testing.T) { + root := repositoryRoot(t) + script := readContractFile(t, root, "scripts/validate_compose_artifacts.sh") + + require.Contains(t, script, "set -euo pipefail") + assertExplicitArtifactServices(t, script) + require.Contains(t, script, `artifact_root="${1:-compose-artifacts}"`) + require.Contains(t, script, "sha256sum --check") + require.Contains(t, script, `[[ -x "$binary" ]]`) + require.Contains(t, script, "env -i") + require.Contains(t, script, `LD_LIBRARY_PATH="$depends_dir"`) + require.Contains(t, script, "not found") + require.Contains(t, script, "depends_dir_real") + require.Contains(t, script, "resolved_library") + require.Contains(t, script, "packaged runtime loader or system ABI library") + require.Contains(t, script, "compose_artifact_core_abi.sh") +} + +func TestComposeArtifactScriptsEndToEnd(t *testing.T) { + root := repositoryRoot(t) + + t.Run("packages all services and validates manifest", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + + manifest := readContractFile(t, fixture.artifactRoot, "MANIFEST.sha256") + for _, service := range composeArtifactServices { + require.FileExists(t, filepath.Join(fixture.artifactRoot, service, "build", service+"_server")) + require.FileExists(t, filepath.Join(fixture.artifactRoot, service, "depends", "libfixture.so")) + require.Contains(t, manifest, "./"+service+"/build/"+service+"_server") + require.Contains(t, manifest, "./"+service+"/depends/libfixture.so") + } + + output, err := fixture.validate(t, fixture.lddPath) + require.NoError(t, err, "%s", output) + }) + + t.Run("packages non-core distribution libraries but not core ABI", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.lddPath = fixture.writeLDDWithSystemRuntime(t, filepath.Join(fixture.tools, "ldd-system")) + fixture.packageArtifacts(t) + + for _, service := range composeArtifactServices { + depends := filepath.Join(fixture.artifactRoot, service, "depends") + require.FileExists(t, filepath.Join(depends, "libfixture.so")) + require.FileExists(t, filepath.Join(depends, "libprotobuf.so.32")) + require.NoFileExists(t, filepath.Join(depends, "libc.so.6")) + require.NoFileExists(t, filepath.Join(depends, "libstdc++.so.6")) + require.NoFileExists(t, filepath.Join(depends, "ld-linux-x86-64.so.2")) + } + }) + + t.Run("rejects non-core system library fallback", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + hostLibrary := filepath.Join(fixture.temp, "system", "usr", "lib", "libprotobuf.so.32") + require.NoError(t, os.MkdirAll(filepath.Dir(hostLibrary), 0o755)) + require.NoError(t, os.WriteFile(hostLibrary, []byte("host protobuf\n"), 0o644)) + fallbackLDD := fixture.writeLDD(t, filepath.Join(fixture.tools, "ldd-protobuf-host"), hostLibrary) + + output, err := fixture.validate(t, fallbackLDD) + require.Error(t, err) + require.Contains(t, output, "non-core library is outside packaged closure") + }) + + t.Run("rejects injected packaged system ABI library", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + require.NoError(t, os.WriteFile( + filepath.Join(fixture.artifactRoot, "gateway", "depends", "libc.so.6"), + []byte("not glibc\n"), 0o644)) + fixture.rewriteManifest(t) + + output, err := fixture.validate(t, fixture.lddPath) + require.Error(t, err) + require.Contains(t, output, "packaged runtime loader or system ABI library") + }) + + t.Run("rejects injected packaged runtime loader", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + require.NoError(t, os.WriteFile( + filepath.Join(fixture.artifactRoot, "identity", "depends", "ld-linux-x86-64.so.2"), + []byte("not a loader\n"), 0o755)) + fixture.rewriteManifest(t) + + output, err := fixture.validate(t, fixture.lddPath) + require.Error(t, err) + require.Contains(t, output, "packaged runtime loader or system ABI library") + }) + + t.Run("rejects manifest tampering", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + tampered := filepath.Join(fixture.artifactRoot, "identity", "build", "identity_server") + file, err := os.OpenFile(tampered, os.O_APPEND|os.O_WRONLY, 0) + require.NoError(t, err) + _, err = file.WriteString("tampered\n") + require.NoError(t, err) + require.NoError(t, file.Close()) + + output, err := fixture.validate(t, fixture.lddPath) + require.Error(t, err) + require.Contains(t, output, "manifest") + }) + + t.Run("rejects missing packaged library", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + require.NoError(t, os.Remove(filepath.Join(fixture.artifactRoot, "media", "depends", "libfixture.so"))) + fixture.rewriteManifest(t) + + output, err := fixture.validate(t, fixture.lddPath) + require.Error(t, err) + require.Contains(t, output, "outside packaged closure") + }) + + t.Run("rejects host library fallback with matching basename", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + hostLibrary := filepath.Join(fixture.temp, "host", "libfixture.so") + require.NoError(t, os.MkdirAll(filepath.Dir(hostLibrary), 0o755)) + require.NoError(t, os.WriteFile(hostLibrary, []byte("host library\n"), 0o644)) + fallbackLDD := fixture.writeLDD(t, filepath.Join(fixture.tools, "ldd-host"), hostLibrary) + + output, err := fixture.validate(t, fallbackLDD) + require.Error(t, err) + require.Contains(t, output, "outside packaged closure") + }) + + t.Run("rejects packaged symlink escaping to host library", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + hostLibrary := filepath.Join(fixture.temp, "host", "libfixture.so") + require.NoError(t, os.MkdirAll(filepath.Dir(hostLibrary), 0o755)) + require.NoError(t, os.WriteFile(hostLibrary, []byte("host library\n"), 0o644)) + packagedLibrary := filepath.Join(fixture.artifactRoot, "push", "depends", "libfixture.so") + require.NoError(t, os.Remove(packagedLibrary)) + require.NoError(t, os.Symlink(hostLibrary, packagedLibrary)) + fixture.rewriteManifest(t) + + output, err := fixture.validate(t, fixture.lddPath) + require.Error(t, err) + require.Contains(t, output, "outside packaged closure") + }) + + t.Run("validates a relative artifact root", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + + output, err := fixture.validateRelative(t, fixture.lddPath) + require.NoError(t, err, "%s", output) + }) + + t.Run("rejects relative artifact root symlink escape", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + hostLibrary := filepath.Join(fixture.temp, "host", "libfixture.so") + require.NoError(t, os.MkdirAll(filepath.Dir(hostLibrary), 0o755)) + require.NoError(t, os.WriteFile(hostLibrary, []byte("host library\n"), 0o644)) + packagedLibrary := filepath.Join(fixture.artifactRoot, "relationship", "depends", "libfixture.so") + require.NoError(t, os.Remove(packagedLibrary)) + require.NoError(t, os.Symlink(hostLibrary, packagedLibrary)) + fixture.rewriteManifest(t) + + output, err := fixture.validateRelative(t, fixture.lddPath) + require.Error(t, err) + require.Contains(t, output, "outside packaged closure") + }) + + t.Run("rejects unlisted artifact file", func(t *testing.T) { + fixture := newArtifactFixture(t, root) + fixture.packageArtifacts(t) + require.NoError(t, os.WriteFile(filepath.Join(fixture.artifactRoot, "unlisted.txt"), []byte("extra\n"), 0o644)) + + output, err := fixture.validate(t, fixture.lddPath) + require.Error(t, err) + require.Contains(t, output, "unlisted artifact file") + }) +} + +type artifactFixture struct { + root string + temp string + buildRoot string + artifactRoot string + tools string + sha256sumPath string + lddPath string +} + +func newArtifactFixture(t *testing.T, root string) artifactFixture { + t.Helper() + temp := t.TempDir() + fixture := artifactFixture{ + root: root, + temp: temp, + buildRoot: filepath.Join(temp, "build"), + artifactRoot: filepath.Join(temp, "compose-artifacts"), + tools: filepath.Join(temp, "tools"), + } + require.NoError(t, os.MkdirAll(fixture.tools, 0o755)) + fixture.sha256sumPath = filepath.Join(fixture.tools, "sha256sum") + require.NoError(t, os.WriteFile(fixture.sha256sumPath, []byte(`#!/bin/sh +set -eu +checksum() { cksum "$1" | awk '{ print $1 ":" $2 }'; } +if [ "${1:-}" = "--check" ]; then + manifest="$2" + status=0 + while read -r expected file; do + file="${file# }" + actual="$(checksum "$file")" + if [ "$actual" != "$expected" ]; then + echo "$file: FAILED" >&2 + status=1 + fi + done < "$manifest" + exit "$status" +fi +for file in "$@"; do + printf '%s %s\n' "$(checksum "$file")" "$file" +done +`), 0o755)) + + fixture.lddPath = fixture.writeLDD(t, filepath.Join(fixture.tools, "ldd"), "") + for _, service := range composeArtifactServices { + binary := filepath.Join(fixture.buildRoot, service, service+"_server") + require.NoError(t, os.MkdirAll(filepath.Dir(binary), 0o755)) + require.NoError(t, os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755)) + } + return fixture +} + +func (fixture artifactFixture) writeLDD(t *testing.T, path, forcedLibrary string) string { + t.Helper() + script := "#!/bin/sh\nset -eu\n" + if forcedLibrary != "" { + script += "echo 'libfixture.so => " + forcedLibrary + " (0x1)'\n" + } else { + script += `if [ -n "${LD_LIBRARY_PATH:-}" ]; then + library="$LD_LIBRARY_PATH/libfixture.so" +else + library="` + filepath.Join(fixture.temp, "libfixture.so") + `" +fi +echo "libfixture.so => $library (0x1)" +` + } + require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) + if forcedLibrary == "" { + require.NoError(t, os.WriteFile(filepath.Join(fixture.temp, "libfixture.so"), []byte("fixture library\n"), 0o644)) + } + return path +} + +func (fixture artifactFixture) writeLDDWithSystemRuntime(t *testing.T, path string) string { + t.Helper() + protobuf := filepath.Join(fixture.temp, "system", "usr", "lib", "libprotobuf.so.32") + require.NoError(t, os.MkdirAll(filepath.Dir(protobuf), 0o755)) + require.NoError(t, os.WriteFile(protobuf, []byte("protobuf fixture\n"), 0o644)) + script := `#!/bin/sh +set -eu +echo "libfixture.so => ${LD_LIBRARY_PATH:-` + fixture.temp + `}/libfixture.so (0x1)" +echo "libprotobuf.so.32 => ` + protobuf + ` (0x2)" +echo "libstdc++.so.6 => /usr/lib/x86_64-linux-gnu/libstdc++.so.6 (0x2)" +echo "libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x3)" +echo "/lib64/ld-linux-x86-64.so.2 (0x4)" +` + require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) + return path +} + +func (fixture artifactFixture) writeLDDLibraries(t *testing.T, path string, libraries ...string) string { + t.Helper() + var script strings.Builder + script.WriteString("#!/bin/sh\nset -eu\n") + for _, library := range libraries { + fmt.Fprintf(&script, "echo 'libfixture.so => %s (0x1)'\n", library) + } + require.NoError(t, os.WriteFile(path, []byte(script.String()), 0o755)) + return path +} + +func (fixture artifactFixture) packageArtifacts(t *testing.T) { + t.Helper() + command := exec.Command("bash", filepath.Join(fixture.root, "scripts/package_compose_artifacts.sh"), fixture.buildRoot, fixture.artifactRoot) + command.Env = append(os.Environ(), "PATH="+fixture.tools+":"+os.Getenv("PATH"), "LDD="+fixture.lddPath) + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) +} + +func (fixture artifactFixture) validate(t *testing.T, lddPath string) (string, error) { + t.Helper() + command := exec.Command("bash", filepath.Join(fixture.root, "scripts/validate_compose_artifacts.sh"), fixture.artifactRoot) + command.Env = append(os.Environ(), "LDD="+lddPath, "SHA256SUM="+fixture.sha256sumPath) + output, err := command.CombinedOutput() + return string(output), err +} + +func (fixture artifactFixture) validateRelative(t *testing.T, lddPath string) (string, error) { + t.Helper() + command := exec.Command("bash", filepath.Join(fixture.root, "scripts/validate_compose_artifacts.sh"), filepath.Base(fixture.artifactRoot)) + command.Dir = fixture.temp + command.Env = append(os.Environ(), "LDD="+lddPath, "SHA256SUM="+fixture.sha256sumPath) + output, err := command.CombinedOutput() + return string(output), err +} + +func (fixture artifactFixture) rewriteManifest(t *testing.T) { + t.Helper() + command := exec.Command("bash", "-c", `find . \( -type f -o -type l \) ! -name MANIFEST.sha256 -print0 | LC_ALL=C sort -z | xargs -0 "$SHA256SUM" > MANIFEST.sha256`) + command.Dir = fixture.artifactRoot + command.Env = append(os.Environ(), "SHA256SUM="+fixture.sha256sumPath) + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) +} + +func assertExplicitArtifactServices(t *testing.T, script string) { + t.Helper() + serviceList := strings.Join(composeArtifactServices, " ") + require.Contains(t, script, "services=("+serviceList+")", + "the nine Compose services must be enumerated explicitly and in a stable order") +} + +func readContractFile(t *testing.T, root, name string) string { + t.Helper() + contents, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(name))) + require.NoError(t, err, "%s must exist", name) + return string(contents) +} diff --git a/tests/pkg/contracts/ci_gates_test.go b/tests/pkg/contracts/ci_gates_test.go new file mode 100644 index 0000000..d303fbd --- /dev/null +++ b/tests/pkg/contracts/ci_gates_test.go @@ -0,0 +1,671 @@ +package contracts + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "chatnow-tests/pkg/client" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" + "gopkg.in/yaml.v3" +) + +func TestDirectProtobufHTTPClient(t *testing.T) { + received := make(chan string, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + request := &wrapperspb.StringValue{} + if err := proto.Unmarshal(body, request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + received <- fmt.Sprintf("%s|%s|%s", r.URL.Path, r.Header.Get("Content-Type"), request.GetValue()) + response, err := proto.Marshal(wrapperspb.String("direct-response")) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/x-protobuf") + _, _ = w.Write(response) + })) + t.Cleanup(server.Close) + + httpClient := client.NewHTTPClient(&client.Config{}) + response := &wrapperspb.StringValue{} + require.NoError(t, httpClient.DoProtobufURL( + server.URL+"/chatnow.push.PushService/PushToUser", + wrapperspb.String("direct-request"), response)) + require.Equal(t, "direct-response", response.GetValue()) + require.Equal(t, + "/chatnow.push.PushService/PushToUser|application/x-protobuf|direct-request", + <-received) +} + +func TestGoCacheRegressionsReplaceTemporaryCPP(t *testing.T) { + root := repositoryRoot(t) + for _, relativePath := range []string{ + "common/test/test_user_info_generation_fence.cc", + "common/test/test_unacked_pending_ledger.cc", + } { + _, err := os.Stat(filepath.Join(root, relativePath)) + require.ErrorIs(t, err, os.ErrNotExist, "%s must be removed", relativePath) + } + + cmake, err := os.ReadFile(filepath.Join(root, "common/test/CMakeLists.txt")) + require.NoError(t, err) + require.NotContains(t, string(cmake), "test_unacked_pending_ledger", + "the temporary C++ Unacked target must be removed") + + cacheTestPath := filepath.Join(root, "tests/func/cache_test.go") + parsed, err := parser.ParseFile(token.NewFileSet(), cacheTestPath, nil, 0) + require.NoError(t, err) + var found bool + for _, declaration := range parsed.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == "TestFN_CA_UnackedSameUserSeqLatestPayloadAndAck" { + found = true + break + } + } + require.True(t, found, + "the Go functional suite must own the same-user_seq latest-payload/ACK regression") +} + +func TestReadAckUsesConversationSequenceWatermark(t *testing.T) { + root := repositoryRoot(t) + serviceProto, err := os.ReadFile(filepath.Join(root, "proto/message/message_service.proto")) + require.NoError(t, err) + require.Contains(t, string(serviceProto), "uint64 seq_id = 3;") + require.NotContains(t, string(serviceProto), "uint64 message_id = 3;") + require.Contains(t, string(serviceProto), "送达确认水位") + require.NotContains(t, string(serviceProto), "已读水位线") + + messageTests, err := os.ReadFile(filepath.Join(root, "tests/func/message_test.go")) + require.NoError(t, err) + require.NotContains(t, string(messageTests), "read watermark") + + server, err := os.ReadFile(filepath.Join(root, "message/source/message_server.h")) + require.NoError(t, err) + updateReadAck := string(server) + start := strings.Index(updateReadAck, "void UpdateReadAck(") + require.GreaterOrEqual(t, start, 0) + end := strings.Index(updateReadAck[start:], "\n // ====== MQ consumer") + require.Greater(t, end, 0) + updateReadAck = updateReadAck[start : start+end] + require.Contains(t, updateReadAck, "req->seq_id()") + require.Contains(t, updateReadAck, "update_last_ack_seq(") + require.NotContains(t, updateReadAck, "req->message_id()") + require.NotContains(t, updateReadAck, "select_by_id(") + + pushProto, err := os.ReadFile(filepath.Join(root, "proto/push/notify.proto")) + require.NoError(t, err) + require.Contains(t, string(pushProto), "uint64 seq_id = 6;") + + pushServer, err := os.ReadFile(filepath.Join(root, "push/source/push_server.h")) + require.NoError(t, err) + clientNotify := string(pushServer) + start = strings.Index(clientNotify, "void onClientNotify(") + require.GreaterOrEqual(t, start, 0) + end = strings.Index(clientNotify[start:], "\n void shutdown_cleanup()") + require.Greater(t, end, 0) + clientNotify = clientNotify[start : start+end] + require.Contains(t, clientNotify, "closure->req.set_seq_id(ack.seq_id())") + require.NotContains(t, clientNotify, "closure->req.set_message_id(") + require.Contains(t, clientNotify, "ack.seq_id() > 0 && !ack.conversation_id().empty()") + require.NotContains(t, clientNotify, "conversation read watermark") +} + +type workflowContract struct { + On struct { + PullRequest map[string]any `yaml:"pull_request"` + Schedule []map[string]any `yaml:"schedule"` + } `yaml:"on"` + Jobs map[string]workflowJob `yaml:"jobs"` +} + +type workflowJob struct { + If string `yaml:"if"` + Env map[string]string `yaml:"env"` + Needs any `yaml:"needs"` + Steps []workflowStep `yaml:"steps"` +} + +type workflowStep struct { + Uses string `yaml:"uses"` + Run string `yaml:"run"` + If string `yaml:"if"` + ContinueOnError any `yaml:"continue-on-error"` + Env map[string]string `yaml:"env"` + With map[string]any `yaml:"with"` +} + +type composeContract struct { + Services map[string]struct { + Entrypoint string `yaml:"entrypoint"` + } `yaml:"services"` +} + +func TestCIGates(t *testing.T) { + root := repositoryRoot(t) + workflowBytes, err := os.ReadFile(filepath.Join(root, ".github/workflows/ci.yml")) + require.NoError(t, err) + composeBytes, err := os.ReadFile(filepath.Join(root, "docker-compose.yml")) + require.NoError(t, err) + + var workflow workflowContract + require.NoError(t, yaml.Unmarshal(workflowBytes, &workflow), "workflow must be valid YAML") + require.NotNil(t, workflow.On.PullRequest, "workflow must handle pull requests") + require.NotEmpty(t, workflow.On.Schedule, "workflow must define a schedule") + assertDecoratedCommandsDoNotSatisfyGate(t) + assertContractsRunInBuild(t, workflow.Jobs["build"]) + + producer, ok := workflow.Jobs["service-artifacts"] + require.True(t, ok, "CI must build the Compose service artifacts once") + require.Equal(t, "github.event_name != 'push' || github.ref != 'refs/heads/main'", producer.If) + assertServiceArtifactProducer(t, producer) + require.Equal(t, "github.event_name != 'push' || github.ref != 'refs/heads/main'", workflow.Jobs["bvt"].If) + + for _, consumer := range []struct { + job string + target string + needs []string + }{ + {"bvt", "cd tests && make test-bvt", []string{"service-artifacts"}}, + {"func", "cd tests && make test-func", []string{"bvt", "service-artifacts"}}, + {"reliability", "cd tests && make test-reliability", []string{"service-artifacts"}}, + {"perf-cache", "cd tests && make test-perf-cache-gate", []string{"service-artifacts"}}, + } { + job, exists := workflow.Jobs[consumer.job] + require.True(t, exists, "%s must be a dedicated clean-runner consumer", consumer.job) + require.ElementsMatch(t, consumer.needs, workflowNeeds(job.Needs)) + assertFullStackGateJob(t, job, consumer.target) + assertInvalidGateJobsRejected(t, job, consumer.target) + } + + reliability, ok := workflow.Jobs["reliability"] + require.True(t, ok, "RL-05 must have a dedicated reliability job") + require.Equal(t, "service-artifacts", reliability.Needs) + require.Equal(t, "github.event_name == 'pull_request' || github.event_name == 'schedule'", reliability.If) + + perfCache, ok := workflow.Jobs["perf-cache"] + require.True(t, ok, "PF-09 must have a dedicated perf-cache job") + require.Equal(t, "service-artifacts", perfCache.Needs) + require.Equal(t, "github.event_name == 'schedule'", perfCache.If) + assertTargetAbsent(t, perfCache, "test-perf-cache") + + require.Equal(t, "2147483647", gateEnv(t, perfCache, "TRANSMITE_RATE_LIMIT_USER_MAX")) + require.Equal(t, "2147483647", gateEnv(t, perfCache, "TRANSMITE_RATE_LIMIT_SESSION_MAX")) + + var compose composeContract + require.NoError(t, yaml.Unmarshal(composeBytes, &compose), "Compose file must be valid YAML") + transmite, ok := compose.Services["transmite_server"] + require.True(t, ok) + require.Contains(t, transmite.Entrypoint, "-rate_limit_user_max=${TRANSMITE_RATE_LIMIT_USER_MAX:-600}") + require.Contains(t, transmite.Entrypoint, "-rate_limit_session_max=${TRANSMITE_RATE_LIMIT_SESSION_MAX:-3000}") +} + +const ( + artifactName = "compose-service-artifacts" + artifactPath = "compose-artifacts" + builderImage = "chatnow-ci-builder:ci" + consumerValidateCommand = `docker run --rm -v "$PWD:/workspace" -w /workspace ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 ./scripts/validate_compose_artifacts.sh compose-artifacts` + nativeBuildCommand = `docker run --rm -v "$PWD:/workspace" -w /workspace chatnow-ci-builder:ci bash -lc ' +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build --parallel "$(nproc)" --target conversation_server gateway_server identity_server media_server message_server presence_server push_server relationship_server transmite_server +'` + packageCommand = `docker run --rm -v "$PWD:/workspace" -w /workspace chatnow-ci-builder:ci ./scripts/package_compose_artifacts.sh build compose-artifacts` + validateCommand = `docker run --rm -v "$PWD:/workspace" -w /workspace chatnow-ci-builder:ci ./scripts/validate_compose_artifacts.sh compose-artifacts` + restoreCommand = `for service in conversation gateway identity media message presence push relationship transmite; do + rm -rf "$service/build" "$service/depends" + cp -a "compose-artifacts/$service/build" "$service/build" + cp -a "compose-artifacts/$service/depends" "$service/depends" +done` + chmodArtifactsCommand = `for service in conversation gateway identity media message presence push relationship transmite; do + chmod +x "compose-artifacts/$service/build/${service}_server" + chmod +x "$service/build/${service}_server" +done` +) + +func assertServiceArtifactProducer(t *testing.T, job workflowJob) { + t.Helper() + require.NoError(t, validateServiceArtifactProducer(job)) + + valid := cloneWorkflowJob(job) + build := exactUsesStepIndex(valid, "docker/build-push-action@v6") + native := exactRunStepIndex(valid, nativeBuildCommand) + pack := exactRunStepIndex(valid, packageCommand) + validate := exactRunStepIndex(valid, validateCommand) + upload := exactUsesStepIndex(valid, "actions/upload-artifact@v4") + for name, mutate := range map[string]func(*workflowJob){ + "builder allowed to fail": func(job *workflowJob) { job.Steps[build].ContinueOnError = true }, + "native build allowed to fail": func(job *workflowJob) { job.Steps[native].ContinueOnError = true }, + "package allowed to fail": func(job *workflowJob) { job.Steps[pack].ContinueOnError = true }, + "validation allowed to fail": func(job *workflowJob) { job.Steps[validate].ContinueOnError = true }, + "upload allowed to fail": func(job *workflowJob) { job.Steps[upload].ContinueOnError = true }, + "validation after upload": func(job *workflowJob) { + job.Steps[validate], job.Steps[upload] = job.Steps[upload], job.Steps[validate] + }, + "native build bypassed": func(job *workflowJob) { job.Steps[native].If = "${{ false }}" }, + "native build duplicated": func(job *workflowJob) { job.Steps = append(job.Steps, job.Steps[native]) }, + } { + t.Run("producer rejects "+name, func(t *testing.T) { + invalid := cloneWorkflowJob(valid) + mutate(&invalid) + require.Error(t, validateServiceArtifactProducer(invalid)) + }) + } +} + +func validateServiceArtifactProducer(job workflowJob) error { + if countExactUsesSteps(job, "docker/build-push-action@v6") != 1 || countExactRunSteps(job, nativeBuildCommand) != 1 { + return fmt.Errorf("services must be built exactly once") + } + ordered := []struct { + label string + index int + }{ + {"checkout", exactUsesStepIndex(job, "actions/checkout@v4")}, + {"Buildx setup", exactUsesStepIndex(job, "docker/setup-buildx-action@v3")}, + {"builder image build", exactUsesStepIndex(job, "docker/build-push-action@v6")}, + {"native service build", exactRunStepIndex(job, nativeBuildCommand)}, + {"artifact package", exactRunStepIndex(job, packageCommand)}, + {"artifact validation", exactRunStepIndex(job, validateCommand)}, + {"artifact upload", exactUsesStepIndex(job, "actions/upload-artifact@v4")}, + } + previous := -1 + for _, required := range ordered { + if required.index < 0 || required.index <= previous { + return fmt.Errorf("missing or out-of-order %s step", required.label) + } + step := job.Steps[required.index] + if strings.TrimSpace(step.If) != "" || continueOnErrorEnabled(step.ContinueOnError) { + return fmt.Errorf("%s step must fail closed", required.label) + } + previous = required.index + } + build := job.Steps[ordered[2].index] + if build.With["context"] != "." || build.With["file"] != "docker/ci/Dockerfile" || build.With["load"] != true || build.With["tags"] != builderImage || build.With["cache-from"] != "type=gha" || build.With["cache-to"] != "type=gha,mode=max" { + return fmt.Errorf("builder image must use the Dockerfile, local load, and BuildKit GHA cache") + } + upload := job.Steps[ordered[len(ordered)-1].index] + if upload.With["name"] != artifactName || upload.With["path"] != artifactPath || upload.With["if-no-files-found"] != "error" { + return fmt.Errorf("artifact upload contract is incomplete") + } + return nil +} + +func assertFullStackGateJob(t *testing.T, job workflowJob, target string) { + t.Helper() + require.NoError(t, validateFullStackGateJob(job, target)) +} + +func assertContractsRunInBuild(t *testing.T, build workflowJob) { + t.Helper() + setupGo := exactUsesStepIndex(build, "actions/setup-go@v5") + proto := exactRunStepIndex(build, "cd tests && make proto") + deps := exactRunStepIndex(build, "cd tests && go mod download") + contracts := exactRunStepIndex(build, "cd tests && go test ./pkg/contracts -count=1") + require.NotEqual(t, -1, setupGo, "build must set up Go") + require.Greater(t, proto, setupGo, "protobuf generation must follow Go setup") + require.Greater(t, deps, proto, "dependency download must follow protobuf generation") + require.Greater(t, contracts, deps, "CI contract tests must run after setup, protobuf generation, and dependency download") +} + +func exactRunStepIndex(job workflowJob, wanted string) int { + for index, step := range job.Steps { + if strings.TrimSpace(step.Run) == strings.TrimSpace(wanted) { + return index + } + } + return -1 +} + +func exactUsesStepIndex(job workflowJob, wanted string) int { + for index, step := range job.Steps { + if step.Uses == wanted { + return index + } + } + return -1 +} + +func countExactUsesSteps(job workflowJob, wanted string) int { + count := 0 + for _, step := range job.Steps { + if step.Uses == wanted { + count++ + } + } + return count +} + +func assertDecoratedCommandsDoNotSatisfyGate(t *testing.T) { + t.Helper() + const gate = "cd tests && make test-perf-cache-gate" + require.Equal(t, 0, exactRunStepIndex(workflowJob{Steps: []workflowStep{{Run: gate}}}, gate)) + for _, lookalike := range []string{ + "# " + gate, + "echo '" + gate + "'", + gate + "-disabled", + "false && " + gate, + "exit 0\n" + gate, + "if false; then " + gate + "; fi", + gate + " # disabled", + gate + "\nexit 0", + "cat <<'EOF'\n" + gate + "\nEOF", + "gate() { " + gate + "; }\ngate", + `cd tests && make "test-perf-cache"`, + } { + job := workflowJob{Steps: []workflowStep{{Run: lookalike}}} + require.Equal(t, -1, exactRunStepIndex(job, gate), "%q must not satisfy the executable gate contract", lookalike) + } +} + +func assertInvalidGateJobsRejected(t *testing.T, valid workflowJob, target string) { + t.Helper() + gate := exactRunStepIndex(valid, target) + setupGo := exactUsesStepIndex(valid, "actions/setup-go@v5") + download := exactUsesStepIndex(valid, "actions/download-artifact@v4") + restore := exactRunStepIndex(valid, restoreCommand) + chmod := exactRunStepIndex(valid, chmodArtifactsCommand) + validate := exactRunStepIndex(valid, consumerValidateCommand) + start := exactRunStepIndex(valid, "docker compose up -d --build") + wait := exactRunStepIndex(valid, "./scripts/wait_for_services.sh") + proto := exactRunStepIndex(valid, "cd tests && make proto") + deps := exactRunStepIndex(valid, "cd tests && go mod download") + teardown := exactRunStepIndex(valid, "docker compose down -v") + require.NotEqual(t, -1, gate) + require.NotEqual(t, -1, setupGo) + require.NotEqual(t, -1, download) + require.NotEqual(t, -1, restore) + require.NotEqual(t, -1, chmod) + require.NotEqual(t, -1, validate) + require.NotEqual(t, -1, start) + require.NotEqual(t, -1, wait) + require.NotEqual(t, -1, proto) + require.NotEqual(t, -1, deps) + require.NotEqual(t, -1, teardown) + + for name, mutate := range map[string]func(*workflowJob){ + "producer dependency missing": func(job *workflowJob) { + job.Needs = nil + }, + "Go setup allowed to fail": func(job *workflowJob) { + job.Steps[setupGo].ContinueOnError = true + }, + "download allowed to fail": func(job *workflowJob) { + job.Steps[download].ContinueOnError = true + }, + "restore allowed to fail": func(job *workflowJob) { + job.Steps[restore].ContinueOnError = true + }, + "missing executable mode repair": func(job *workflowJob) { + job.Steps = append(job.Steps[:chmod], job.Steps[chmod+1:]...) + }, + "executable mode repair after validation": func(job *workflowJob) { + job.Steps[chmod], job.Steps[validate] = job.Steps[validate], job.Steps[chmod] + }, + "artifact validation allowed to fail": func(job *workflowJob) { + job.Steps[validate].ContinueOnError = true + }, + "artifact validation on consumer host": func(job *workflowJob) { + job.Steps[validate].Run = "./scripts/validate_compose_artifacts.sh compose-artifacts" + }, + "artifact validation with mutable Ubuntu tag": func(job *workflowJob) { + job.Steps[validate].Run = `docker run --rm -v "$PWD:/workspace" -w /workspace ubuntu:24.04 ./scripts/validate_compose_artifacts.sh compose-artifacts` + }, + "gate disabled by if": func(job *workflowJob) { + job.Steps[gate].If = "${{ false }}" + }, + "gate allowed to fail": func(job *workflowJob) { + job.Steps[gate].ContinueOnError = true + }, + "startup allowed to fail": func(job *workflowJob) { + job.Steps[start].ContinueOnError = true + }, + "wait allowed to fail": func(job *workflowJob) { + job.Steps[wait].ContinueOnError = true + }, + "proto allowed to fail": func(job *workflowJob) { + job.Steps[proto].ContinueOnError = true + }, + "dependency download allowed to fail": func(job *workflowJob) { + job.Steps[deps].ContinueOnError = true + }, + "exit zero after gate": func(job *workflowJob) { + job.Steps[gate].Run = target + "\nexit 0" + }, + "if false gate": func(job *workflowJob) { + job.Steps[gate].Run = "if false; then " + target + "; fi" + }, + "quoted skip target": func(job *workflowJob) { + insertWorkflowStep(job, teardown, workflowStep{Run: `cd tests && make "test-perf-cache"`}) + }, + "single-quoted skip target": func(job *workflowJob) { + insertWorkflowStep(job, teardown, workflowStep{Run: `cd tests && make 'test-perf-cache'`}) + }, + "gate before dependencies": func(job *workflowJob) { + job.Steps[gate], job.Steps[deps] = job.Steps[deps], job.Steps[gate] + }, + "Compose before artifact validation": func(job *workflowJob) { + job.Steps[start], job.Steps[validate] = job.Steps[validate], job.Steps[start] + }, + "teardown before gate": func(job *workflowJob) { + job.Steps[gate], job.Steps[teardown] = job.Steps[teardown], job.Steps[gate] + }, + "teardown not last": func(job *workflowJob) { + job.Steps = append(job.Steps, workflowStep{Run: "true"}) + }, + } { + t.Run(name, func(t *testing.T) { + invalid := cloneWorkflowJob(valid) + mutate(&invalid) + require.Error(t, validateFullStackGateJob(invalid, target)) + }) + } +} + +func insertWorkflowStep(job *workflowJob, index int, step workflowStep) { + job.Steps = append(job.Steps, workflowStep{}) + copy(job.Steps[index+1:], job.Steps[index:]) + job.Steps[index] = step +} + +func cloneWorkflowJob(job workflowJob) workflowJob { + clone := job + clone.Steps = append([]workflowStep(nil), job.Steps...) + return clone +} + +func assertTargetAbsent(t *testing.T, job workflowJob, target string) { + t.Helper() + for _, step := range job.Steps { + require.False(t, isForbiddenMakeTarget(step.Run, target), + "PF-09 CI must not execute the skip-capable discovery target") + } +} + +func isForbiddenMakeTarget(run, target string) bool { + run = strings.TrimSpace(run) + for _, command := range []string{ + "cd tests && make " + target, + `cd tests && make "` + target + `"`, + "cd tests && make '" + target + "'", + } { + if run == command { + return true + } + } + return false +} + +func validateFullStackGateJob(job workflowJob, target string) error { + const install = "sudo apt-get update\nsudo apt-get install -y protobuf-compiler netcat-openbsd" + requiredNeeds := []string{"service-artifacts"} + if target == "cd tests && make test-func" { + requiredNeeds = append(requiredNeeds, "bvt") + } + if !sameStringSet(workflowNeeds(job.Needs), requiredNeeds) { + return fmt.Errorf("gate job has incorrect producer dependencies") + } + for _, step := range job.Steps { + if isForbiddenMakeTarget(step.Run, "test-perf-cache") { + return fmt.Errorf("skip-capable performance target is forbidden") + } + } + ordered := []struct { + label string + index int + }{ + {"checkout", exactUsesStepIndex(job, "actions/checkout@v4")}, + {"Go setup", exactUsesStepIndex(job, "actions/setup-go@v5")}, + {"system dependency install", exactRunStepIndex(job, install)}, + {"protoc generator install", exactRunStepIndex(job, "go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11")}, + {"artifact download", exactUsesStepIndex(job, "actions/download-artifact@v4")}, + {"artifact restore", exactRunStepIndex(job, restoreCommand)}, + {"executable mode repair", exactRunStepIndex(job, chmodArtifactsCommand)}, + {"artifact validation", exactRunStepIndex(job, consumerValidateCommand)}, + {"full-stack startup", exactRunStepIndex(job, "docker compose up -d --build")}, + {"service wait", exactRunStepIndex(job, "./scripts/wait_for_services.sh")}, + {"protobuf generation", exactRunStepIndex(job, "cd tests && make proto")}, + {"dependency download", exactRunStepIndex(job, "cd tests && go mod download")}, + {"gate", exactRunStepIndex(job, target)}, + {"teardown", exactRunStepIndex(job, "docker compose down -v")}, + } + previous := -1 + for _, step := range ordered { + if step.index < 0 { + return fmt.Errorf("missing exact %s step", step.label) + } + if step.index <= previous { + return fmt.Errorf("%s step is out of order", step.label) + } + previous = step.index + } + teardown := ordered[len(ordered)-1].index + gate := ordered[len(ordered)-2].index + if strings.TrimSpace(job.Steps[gate].If) != "" { + return fmt.Errorf("gate step must not have a step-level if condition") + } + for _, required := range ordered[:len(ordered)-1] { + if continueOnErrorEnabled(job.Steps[required.index].ContinueOnError) { + return fmt.Errorf("%s step must not continue on error", required.label) + } + if strings.TrimSpace(job.Steps[required.index].If) != "" { + return fmt.Errorf("%s step must not have a step-level if condition", required.label) + } + } + download := ordered[4].index + if job.Steps[download].With["name"] != artifactName || job.Steps[download].With["path"] != artifactPath { + return fmt.Errorf("gate job must download the shared Compose artifact") + } + if teardown != len(job.Steps)-1 { + return fmt.Errorf("teardown must be the final step") + } + if job.Steps[teardown].If != "always()" { + return fmt.Errorf("teardown must use if: always()") + } + if countExactRunSteps(job, target) != 1 { + return fmt.Errorf("gate command must appear exactly once") + } + if countExactRunSteps(job, "docker compose down -v") != 1 { + return fmt.Errorf("teardown command must appear exactly once") + } + return nil +} + +func workflowNeeds(raw any) []string { + switch value := raw.(type) { + case string: + return []string{value} + case []any: + needs := make([]string, 0, len(value)) + for _, item := range value { + need, ok := item.(string) + if !ok { + return nil + } + needs = append(needs, need) + } + return needs + default: + return nil + } +} + +func sameStringSet(actual, expected []string) bool { + if len(actual) != len(expected) { + return false + } + seen := make(map[string]bool, len(actual)) + for _, value := range actual { + if seen[value] { + return false + } + seen[value] = true + } + for _, value := range expected { + if !seen[value] { + return false + } + } + return true +} + +func continueOnErrorEnabled(value any) bool { + switch value := value.(type) { + case nil: + return false + case bool: + return value + default: + return true + } +} + +func countExactRunSteps(job workflowJob, wanted string) int { + count := 0 + for _, step := range job.Steps { + if strings.TrimSpace(step.Run) == wanted { + count++ + } + } + return count +} + +func gateEnv(t *testing.T, job workflowJob, name string) string { + t.Helper() + raw := job.Env[name] + if raw == "" { + for _, step := range job.Steps { + if strings.TrimSpace(step.Run) == "docker compose up -d --build" && step.Env[name] != "" { + raw = step.Env[name] + break + } + } + } + require.NotEmpty(t, raw, "%s must be supplied to the PF-09 stack", name) + return raw +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + _, filename, _, ok := runtime.Caller(0) + require.True(t, ok) + return filepath.Clean(filepath.Join(filepath.Dir(filename), "..", "..", "..")) +} diff --git a/tests/pkg/verify/db.go b/tests/pkg/verify/db.go index fcfec5d..b7cdbbb 100644 --- a/tests/pkg/verify/db.go +++ b/tests/pkg/verify/db.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "testing" + "time" _ "github.com/go-sql-driver/mysql" ) @@ -50,6 +51,24 @@ func (v *DBVerifier) MessageExists(t testing.TB, messageID int64) { } } +// WaitMessageExists waits for the asynchronous MQ consumer to persist a message. +func (v *DBVerifier) WaitMessageExists(t testing.TB, messageID int64, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + var count int + err := v.db.QueryRow("SELECT COUNT(*) FROM message WHERE message_id = ?", messageID).Scan(&count) + if err == nil && count == 1 { + return + } + if err != nil { + t.Fatalf("query message %d while waiting: %v", messageID, err) + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("message %d was not persisted within %s", messageID, timeout) +} + // MessageCount 验证某会话 message 表记录数。 // 注意：message 表用 session_id 列名存储 conversation_id。 func (v *DBVerifier) MessageCount(t testing.TB, conversationID string, expected int) { @@ -123,6 +142,21 @@ func (v *DBVerifier) LastReadSeq(t testing.TB, userID, conversationID string, ex } } +// LastAckSeq verifies the delivery acknowledgement cursor. +func (v *DBVerifier) LastAckSeq(t testing.TB, userID, conversationID string, expected uint64) { + var seq uint64 + err := v.db.QueryRow( + "SELECT last_ack_seq FROM conversation_member WHERE user_id = ? AND conversation_id = ?", + userID, conversationID, + ).Scan(&seq) + if err != nil { + t.Fatalf("query last_ack_seq: %v", err) + } + if seq != expected { + t.Fatalf("last_ack_seq user=%s conv=%s 期望 %d，实际 %d", userID, conversationID, expected, seq) + } +} + // UnreadCount 计算并验证未读数 = max(message.seq_id) - last_read_seq。 func (v *DBVerifier) UnreadCount(t testing.TB, userID, conversationID string, expected int) { var lastReadSeq uint64 diff --git a/tests/pkg/verify/redis.go b/tests/pkg/verify/redis.go new file mode 100644 index 0000000..e880529 --- /dev/null +++ b/tests/pkg/verify/redis.go @@ -0,0 +1,58 @@ +package verify + +import ( + "fmt" + "io" + "net/http" + "os/exec" + "strconv" + "strings" + "testing" + "time" +) + +func RedisCLI(t testing.TB, args ...string) string { + t.Helper() + out, err := RedisCLIResult(args...) + if err != nil { + t.Fatal(err) + } + return out +} + +// RedisCLIResult runs redis-cli without invoking testing APIs, so callers may +// safely use it from polling callbacks and report failures on the test goroutine. +func RedisCLIResult(args ...string) (string, error) { + base := []string{"exec", "redis-node1", "redis-cli", "-c"} + out, err := exec.Command("docker", append(base, args...)...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("redis-cli %v: %w: %s", args, err, strings.TrimSpace(string(out))) + } + return strings.TrimSpace(string(out)), nil +} + +func RedisTTL(t testing.TB, key string) time.Duration { + seconds, err := strconv.ParseInt(RedisCLI(t, "TTL", key), 10, 64) + if err != nil || seconds < 0 { + t.Fatalf("invalid TTL for %s: %d (%v)", key, seconds, err) + } + return time.Duration(seconds) * time.Second +} + +func BVar(t testing.TB, baseURL, name string) int64 { + t.Helper() + rsp, err := http.Get(fmt.Sprintf("%s/vars/%s", baseURL, name)) + if err != nil { + t.Fatalf("read bvar %s: %v", name, err) + } + defer rsp.Body.Close() + body, err := io.ReadAll(rsp.Body) + if err != nil { + t.Fatalf("read bvar body %s: %v", name, err) + } + value, err := strconv.ParseInt(strings.TrimSpace(string(body)), 10, 64) + if err != nil { + t.Fatalf("parse bvar %s=%q: %v", name, body, err) + } + return value +} diff --git a/tests/reliability/redis_failover_test.go b/tests/reliability/redis_failover_test.go new file mode 100644 index 0000000..b33859c --- /dev/null +++ b/tests/reliability/redis_failover_test.go @@ -0,0 +1,278 @@ +//go:build reliability + +package reliability_test + +import ( + "bytes" + "fmt" + "net" + "os/exec" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "chatnow-tests/pkg/chaos" + "chatnow-tests/pkg/client" + "chatnow-tests/pkg/fixture" + "chatnow-tests/pkg/verify" + identity "chatnow-tests/proto/chatnow/identity" + msg "chatnow-tests/proto/chatnow/message" + transmite "chatnow-tests/proto/chatnow/transmite" +) + +// RL-05 | P0 | Redis 熔断后快速失败并自动恢复 +func TestRL_RedisCircuitFastFailAndRecovery(t *testing.T) { + user, _, _ := fixture.RegisterAndLogin(t, HTTP) + peer, _, _ := fixture.RegisterAndLogin(t, HTTP) + convID := fixture.CreateGroupWithMembers(t, user, []*client.HTTPClient{peer}, "rl-redis-outage") + prewarmRsp := &transmite.SendMessageRsp{} + require.NoError(t, user.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), + ConversationId: convID, + Content: &msg.MessageContent{ + Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: "prewarm"}}, + }, + ClientMsgId: client.NewRequestID(), + }, prewarmRsp)) + require.True(t, prewarmRsp.GetHeader().GetSuccess()) + endpoints := reliabilityTransmiteEndpoints(t) + openedBefore := reliabilitySumBVar(t, endpoints, "redis_circuit_open_total") + rejectedBefore := reliabilitySumBVar(t, endpoints, "redis_circuit_rejected_total") + recoveredBefore := reliabilitySumBVar(t, endpoints, "redis_circuit_recovered_total") + + t.Cleanup(func() { + chaos.StartRedisCluster(t, HTTP.Config()) + chaos.WaitRedisCluster(t, HTTP.Config(), 60*time.Second) + }) + chaos.StopRedisCluster(t, HTTP.Config()) + + rateLimited := 0 + seqUnavailable := 0 + for i := 0; i < 650; i++ { + sendRsp := &transmite.SendMessageRsp{} + err := user.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), + ConversationId: convID, + Content: &msg.MessageContent{ + Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: fmt.Sprintf("outage-%d", i)}}, + }, + ClientMsgId: client.NewRequestID(), + }, sendRsp) + require.NoError(t, err) + if sendRsp.GetHeader().GetErrorMessage() == "rate_limited" { + rateLimited++ + } + if sendRsp.GetHeader().GetErrorMessage() == "序号生成失败" { + seqUnavailable++ + } + } + require.Greater(t, rateLimited, 0, "Redis outage must retain bounded rate limiting") + require.Greater(t, seqUnavailable, 0, "SeqGen truth source must fail unavailable") + require.Greater(t, reliabilitySumBVar(t, endpoints, "redis_circuit_open_total"), openedBefore) + require.Greater(t, reliabilitySumBVar(t, endpoints, "redis_circuit_rejected_total"), rejectedBefore) + + uid := user.UserID + for i := 0; i < 3; i++ { + rsp := &identity.GetProfileRsp{} + _ = user.DoAuth("/service/identity/get_profile", &identity.GetProfileReq{ + RequestId: client.NewRequestID(), UserId: &uid, + }, rsp) + } + + started := time.Now() + fastFail := &transmite.SendMessageRsp{} + err := user.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), ConversationId: convID, + Content: &msg.MessageContent{Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: "fast-fail"}}}, + ClientMsgId: client.NewRequestID(), + }, fastFail) + require.NoError(t, err) + require.False(t, fastFail.GetHeader().GetSuccess()) + require.Less(t, time.Since(started), 50*time.Millisecond) + + chaos.StartRedisCluster(t, HTTP.Config()) + chaos.WaitRedisCluster(t, HTTP.Config(), 60*time.Second) + time.Sleep(1100 * time.Millisecond) + rsp := &identity.GetProfileRsp{} + require.NoError(t, user.DoAuth("/service/identity/get_profile", &identity.GetProfileReq{ + RequestId: client.NewRequestID(), UserId: &uid, + }, rsp)) + require.True(t, rsp.GetHeader().GetSuccess()) + + recoveredSend := &transmite.SendMessageRsp{} + require.NoError(t, user.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), ConversationId: convID, + Content: &msg.MessageContent{Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: "recovered"}}}, + ClientMsgId: client.NewRequestID(), + }, recoveredSend)) + require.True(t, recoveredSend.GetHeader().GetSuccess(), recoveredSend.GetHeader().GetErrorMessage()) + require.Greater(t, reliabilitySumBVar(t, endpoints, "redis_circuit_recovered_total"), recoveredBefore, + "an Open->HalfOpen probe must recover the Transmite Redis circuit") +} + +// RL-05 Push truth source: Rabbit accepts while Push is paused; after Redis is +// stopped, resuming Push must requeue before websocket delivery. Recovery then +// permits the half-open probe, durable Unacked write, and at-least-once delivery. +func TestRL_PushUnackedRequeuesUntilRedisRecovers(t *testing.T) { + // This black-box choreography is intentionally tied to the compose topology: + // one Push container owns the websocket and exposes the sole configured bvar + // endpoint. Multi-instance CI must provide a dedicated single-Push test stack. + pushEndpoint := reliabilitySinglePushEndpoint(t) + pushContainer := HTTP.Config().Infra.PushContainer + require.NotEmpty(t, pushContainer, + "pause-container Push test requires the websocket-owning container name") + require.GreaterOrEqual(t, HTTP.Config().Infra.PushRouteL1TTLSec, 15, + "Push reliability stack requires route L1 TTL >=15s; docker config uses 30s") + + sender, recipient, convID := fixture.MakeFriends(t, HTTP) + ws, err := client.OpenWebSocket(HTTP.Config()) + require.NoError(t, err) + t.Cleanup(func() { _ = ws.Close() }) + require.NoError(t, ws.WriteBinary(client.PushAuthNotify(recipient.AccessToken, "default_device"))) + + deviceKey := fmt.Sprintf("im:dev:{%s}", recipient.UserID) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && verify.RedisCLI(t, "EXISTS", deviceKey) != "1" { + time.Sleep(50 * time.Millisecond) + } + require.Equal(t, "1", verify.RedisCLI(t, "EXISTS", deviceKey)) + drainDeadline := time.Now().Add(2 * time.Second) + for time.Now().Before(drainDeadline) { + _, readErr := ws.ReadFrame(100 * time.Millisecond) + if timeout, ok := readErr.(net.Error); ok && timeout.Timeout() { + break + } + require.NoError(t, readErr) + } + + // A successful end-to-end delivery warms that instance's route L1 before it + // is paused. The test-only TTL keeps the route through the outage choreography. + warmMarker := "rl-unacked-warm-" + client.NewRequestID() + sendReliabilityMessage(t, sender, convID, warmMarker) + require.True(t, readWebSocketMarker(ws, warmMarker, 5*time.Second), "warm Push delivery missing") + persistBefore := verify.BVar(t, pushEndpoint, "push_unacked_persist_failure_total") + requeueBefore := verify.BVar(t, pushEndpoint, "push_message_requeue_total") + + requireDocker(t, "pause", pushContainer) + paused := true + redisStopped := false + t.Cleanup(func() { + if paused { + _, _ = exec.Command("docker", "unpause", pushContainer).CombinedOutput() + } + if redisStopped { + chaos.StartRedisCluster(t, HTTP.Config()) + chaos.WaitRedisCluster(t, HTTP.Config(), 60*time.Second) + } + }) + + marker := "rl-unacked-" + client.NewRequestID() + sendReliabilityMessage(t, sender, convID, marker) + + chaos.StopRedisCluster(t, HTTP.Config()) + redisStopped = true + requireDocker(t, "unpause", pushContainer) + paused = false + requireBVarIncrease(t, pushEndpoint, "push_unacked_persist_failure_total", persistBefore, 5*time.Second) + requireBVarIncrease(t, pushEndpoint, "push_message_requeue_total", requeueBefore, 5*time.Second) + if payload, readErr := ws.ReadFrame(500 * time.Millisecond); readErr == nil { + t.Fatalf("Push delivered before durable Unacked persistence: %x", payload) + } else if timeout, ok := readErr.(net.Error); !ok || !timeout.Timeout() { + t.Fatalf("websocket failed while awaiting Redis outage: %v", readErr) + } + + chaos.StartRedisCluster(t, HTTP.Config()) + chaos.WaitRedisCluster(t, HTTP.Config(), 60*time.Second) + redisStopped = false + + require.True(t, readWebSocketMarker(ws, marker, 20*time.Second), + "requeued Push was not delivered after Redis recovery") + unackedKey := fmt.Sprintf("im:unack:{%s:default_device}", recipient.UserID) + require.Equal(t, "1", verify.RedisCLI(t, "EXISTS", unackedKey), + "delivery must follow durable Unacked persistence") +} + +func sendReliabilityMessage(t testing.TB, sender *client.HTTPClient, convID, marker string) { + t.Helper() + rsp := &transmite.SendMessageRsp{} + require.NoError(t, sender.DoAuth("/service/transmite/send", &transmite.SendMessageReq{ + RequestId: client.NewRequestID(), ConversationId: convID, + Content: &msg.MessageContent{Type: msg.MessageType_TEXT, + Body: &msg.MessageContent_Text{Text: &msg.TextContent{Text: marker}}}, + ClientMsgId: client.NewRequestID(), + }, rsp)) + require.True(t, rsp.GetHeader().GetSuccess(), rsp.GetHeader().GetErrorMessage()) +} + +func readWebSocketMarker(ws *client.WebSocket, marker string, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + payload, err := ws.ReadFrame(time.Until(deadline)) + if err != nil { + return false + } + if bytes.Contains(payload, []byte(marker)) { + return true + } + } + return false +} + +func requireBVarIncrease(t testing.TB, endpoint, metric string, before int64, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if verify.BVar(t, endpoint, metric) > before { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("%s did not increase from %d", metric, before) +} + +func requireDocker(t testing.TB, args ...string) { + t.Helper() + if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { + t.Fatalf("docker %v: %v: %s", args, err, out) + } +} + +func reliabilityTransmiteEndpoints(t testing.TB) []string { + t.Helper() + return reliabilityEndpoints(t, HTTP.Config().Infra.TransmiteVars) +} + +func reliabilitySinglePushEndpoint(t testing.TB) string { + t.Helper() + endpoints := reliabilityEndpoints(t, HTTP.Config().Infra.PushVars) + require.Len(t, endpoints, 1, + "pause-container Push test requires one bvar endpoint for the websocket-owning container; use a dedicated single-instance Push test stack") + return endpoints[0] +} + +func reliabilityEndpoints(t testing.TB, rawEndpoints string) []string { + t.Helper() + var endpoints []string + for _, raw := range strings.Split(rawEndpoints, ",") { + if endpoint := strings.TrimRight(strings.TrimSpace(raw), "/"); endpoint != "" { + endpoints = append(endpoints, endpoint) + } + } + require.NotEmpty(t, endpoints) + return endpoints +} + +func reliabilitySumBVar(t testing.TB, endpoints []string, metric string) int64 { + t.Helper() + var total int64 + for _, endpoint := range endpoints { + total += verify.BVar(t, endpoint, metric) + } + return total +} diff --git a/tests/reliability/setup_test.go b/tests/reliability/setup_test.go new file mode 100644 index 0000000..0b7abcb --- /dev/null +++ b/tests/reliability/setup_test.go @@ -0,0 +1,18 @@ +//go:build reliability + +package reliability_test + +import ( + "os" + "testing" + + "chatnow-tests/pkg/client" +) + +var HTTP *client.HTTPClient + +func TestMain(m *testing.M) { + cfg := client.LoadConfig("../config.yaml") + HTTP = client.NewHTTPClient(cfg) + os.Exit(m.Run()) +} diff --git a/transmite/Dockerfile b/transmite/Dockerfile index 6fc62fc..fca8bd0 100644 --- a/transmite/Dockerfile +++ b/transmite/Dockerfile @@ -1,5 +1,7 @@ # 声明基础镜像来源 -FROM ubuntu:24.04 +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +FROM ${UBUNTU_IMAGE} +ENV LD_LIBRARY_PATH=/im/depends # 声明工作路径 WORKDIR /im RUN mkdir -p /im/logs &&\ @@ -9,7 +11,7 @@ RUN mkdir -p /im/logs &&\ # 将可执行程序文件，拷贝进入镜像 COPY ./build/transmite_server /im/bin # 将可执行程序依赖，拷贝进入镜像 -COPY ./depends/* /lib/x86_64-linux-gnu/ +COPY ./depends/ /im/depends/ RUN apt-get update && apt-get install -y netcat-openbsd && rm -rf /var/lib/apt/lists/* # 设置容器的启动默认操作 --- 运行程序 -CMD /im/bin/transmite_server -flagfile=/im/conf/transmite_server.conf \ No newline at end of file +CMD /im/bin/transmite_server -flagfile=/im/conf/transmite_server.conf diff --git a/transmite/source/transmite_server.h b/transmite/source/transmite_server.h index 0f0edd0..86130e7 100644 --- a/transmite/source/transmite_server.h +++ b/transmite/source/transmite_server.h @@ -74,6 +74,7 @@ class TransmiteServiceImpl : public chatnow::transmite::MsgTransmitService const Members::ptr &members_cache, const RateLimiter::ptr &rate_limiter, const RedisClient::ptr &redis, + const UserInfoCache::ptr &user_info_cache, LocalCache::ptr local_members_cache = nullptr, LocalCache::ptr local_user_cache = nullptr, InflightRegistry::ptr inflight_registry = nullptr, @@ -92,6 +93,7 @@ class TransmiteServiceImpl : public chatnow::transmite::MsgTransmitService _members_cache(members_cache), _rate_limiter(rate_limiter), _redis(redis), + _user_info_cache(user_info_cache), _local_members_cache(std::move(local_members_cache)), _local_user_cache(std::move(local_user_cache)), _inflight_registry(std::move(inflight_registry)), @@ -237,23 +239,17 @@ class TransmiteServiceImpl : public chatnow::transmite::MsgTransmitService } } - // ⑤ 并行 RPC：Identity.GetProfile（sender 信息）+ Conversation.GetMemberIds（收件人列表） - auto identity_channel = _mm_channels->choose(_identity_service_name); - if (!identity_channel) { - LOG_ERROR("请求ID: {} - identity_service 节点缺失", rid); - return err_response(rid, chatnow::error::kSystemUnavailable, "依赖服务暂不可用"); + // ⑤ sender 资料：L1 → uid singleflight → L2 → Identity RPC。 + auto serialized_user_info = resolve_user_info( + uid, rid, static_cast(controller)); + chatnow::common::UserInfo sender_info; + if (!serialized_user_info || serialized_user_info->empty() || + !sender_info.ParseFromString(*serialized_user_info)) { + LOG_ERROR("请求ID: {} - 获取用户信息失败 uid={}", rid, uid); + return err_response(rid, chatnow::error::kSystemUnavailable, "获取用户信息失败"); } // 成员列表：L1 → InflightRegistry → L2 Redis → RedisMutex → RPC (内置 warm) - // 先发起 profile RPC（异步），与成员解析并行 - chatnow::identity::IdentityService_Stub identity_stub(identity_channel.get()); - chatnow::identity::GetProfileReq profile_req; - chatnow::identity::GetProfileRsp profile_rsp; - brpc::Controller profile_cntl; - profile_req.set_request_id(rid); - profile_req.set_user_id(uid); - chatnow::auth::forward_auth_metadata(static_cast(controller), &profile_cntl); - identity_stub.GetProfile(&profile_cntl, &profile_req, &profile_rsp, brpc::DoNothing()); MembersResult members_result; for (int retry = 0; retry < 3; ++retry) { @@ -264,11 +260,6 @@ class TransmiteServiceImpl : public chatnow::transmite::MsgTransmitService } std::vector member_id_list = std::move(members_result.members); - brpc::Join(profile_cntl.call_id()); - if (profile_cntl.Failed() || !profile_rsp.header().success()) { - LOG_ERROR("请求ID: {} - 获取用户信息失败: {}", rid, profile_cntl.ErrorText()); - return err_response(rid, chatnow::error::kSystemUnavailable, "获取用户信息失败"); - } if (member_id_list.empty()) { LOG_ERROR("请求ID: {} - 会话成员为空 ssid={}", rid, chat_ssid); return err_response(rid, chatnow::error::kConversationNotFound, "会话已解散或不存在"); @@ -563,18 +554,120 @@ class TransmiteServiceImpl : public chatnow::transmite::MsgTransmitService } } - std::string resolve_user_info(const std::string &uid) const { - if (!_local_user_cache) return ""; - std::string ukey = key::local_user_info_cache_key(uid); - auto cached = _local_user_cache->get(ukey); - if (cached.has_value()) return *cached; - return ""; + std::optional resolve_user_info(const std::string &uid, + const std::string &rid, + brpc::Controller *caller) { + const auto lkey = key::local_user_info_cache_key(uid); + auto parseable = [](const std::string &bytes) { + chatnow::common::UserInfo info; + return !bytes.empty() && info.ParseFromString(bytes); + }; + auto read_l1 = [&]() -> std::optional { + if (!_local_user_cache) return std::nullopt; + auto local = _local_user_cache->get(lkey); + if (!local) return std::nullopt; + if (parseable(*local)) { + metrics::g_user_info_l1_hit_total << 1; + return local; + } + _local_user_cache->invalidate(lkey); + if (_user_info_cache) _user_info_cache->invalidate(uid); + return std::nullopt; + }; + + if (auto local = read_l1()) return local; + + auto resolve_after_lock = [&]() -> std::optional { + if (auto local = read_l1()) return local; + if (_user_info_cache) { + if (auto redis = _user_info_cache->get(uid)) { + if (redis->empty()) return std::nullopt; // confirmed-not-found sentinel + if (parseable(*redis)) { + metrics::g_user_info_l2_hit_total << 1; + if (_local_user_cache) { + _local_user_cache->set( + lkey, *redis, randomized_ttl(std::chrono::seconds(45))); + } + return redis; + } + _user_info_cache->invalidate(uid); + } + } + + const auto generation = _user_info_cache + ? _user_info_cache->generation(uid) : std::optional{}; + bool confirmed_not_found = false; + auto info = fetch_user_info_from_identity_(uid, rid, caller, &confirmed_not_found); + if (!info) { + if (confirmed_not_found && generation && _user_info_cache) { + _user_info_cache->set_if_generation(uid, "", *generation); + } + return std::nullopt; + } + auto bytes = info->SerializeAsString(); + auto write_result = GenerationWriteResult::Unavailable; + if (generation && _user_info_cache) { + write_result = _user_info_cache->set_if_generation( + uid, bytes, *generation); + } + // A real generation conflict means invalidation won the race, so the + // stale Identity response is returned only to its current caller. If + // Redis was unavailable, retain the short-lived availability fallback. + publish_user_info_l1(write_result, [&](UserInfoL1Publication) { + if (!_local_user_cache) return; + _local_user_cache->set( + lkey, bytes, randomized_ttl(std::chrono::seconds(45))); + }); + return bytes; + }; + + if (!_inflight_registry) return resolve_after_lock(); + auto guard = _inflight_registry->acquire("user:" + uid); + std::unique_lock lk(*guard.mu); + return resolve_after_lock(); } - void warm_user_info(const std::string &uid, const std::string &serialized_info) { - if (_local_user_cache && !serialized_info.empty()) - _local_user_cache->set(key::local_user_info_cache_key(uid), serialized_info, - randomized_ttl(std::chrono::seconds(45))); + std::optional fetch_user_info_from_identity_( + const std::string &uid, const std::string &rid, brpc::Controller *caller, + bool *confirmed_not_found) { + auto identity_channel = _mm_channels->choose(_identity_service_name); + if (!identity_channel) { + LOG_ERROR("identity_service 节点缺失 (user info {})", uid); + return std::nullopt; + } + chatnow::identity::IdentityService_Stub stub(identity_channel.get()); + chatnow::identity::GetProfileReq req; + chatnow::identity::GetProfileRsp rsp; + brpc::Controller cntl; + req.set_request_id(rid); + req.set_user_id(uid); + if (caller) chatnow::auth::forward_auth_metadata(caller, &cntl); + stub.GetProfile(&cntl, &req, &rsp, nullptr); + metrics::g_user_info_rpc_total << 1; + if (cntl.Failed() || !rsp.header().success()) { + if (confirmed_not_found && !cntl.Failed() && + rsp.header().error_code() == chatnow::error::kAuthUserNotFound) { + *confirmed_not_found = true; + } + LOG_ERROR("获取用户信息失败 uid={}: {} {}", uid, cntl.ErrorText(), + rsp.header().error_message()); + return std::nullopt; + } + if (!rsp.has_user_info()) { + LOG_ERROR("获取用户信息响应缺少 user_info uid={}", uid); + return std::nullopt; + } + auto bytes = rsp.user_info().SerializeAsString(); + if (bytes.empty()) { + LOG_ERROR("获取用户信息响应序列化为空 uid={}", uid); + return std::nullopt; + } + chatnow::common::UserInfo info; + if (!info.ParseFromString(bytes)) { + LOG_ERROR("获取用户信息响应无法解析 uid={}", uid); + return std::nullopt; + } + return info; } std::optional> fetch_members_from_conversation_service_( @@ -647,6 +740,7 @@ class TransmiteServiceImpl : public chatnow::transmite::MsgTransmitService Members::ptr _members_cache; RateLimiter::ptr _rate_limiter; RedisClient::ptr _redis; + UserInfoCache::ptr _user_info_cache; LocalCache::ptr _local_members_cache; LocalCache::ptr _local_user_cache; InflightRegistry::ptr _inflight_registry; @@ -823,6 +917,7 @@ class TransmiteServerBuilder } _seq_gen = std::make_shared(_redis_client); _members_cache = std::make_shared(_redis_client); + _user_info_cache = std::make_shared(_redis_client); _rate_limiter = std::make_shared(_redis_client); } @@ -865,6 +960,7 @@ class TransmiteServerBuilder _members_cache, _rate_limiter, _redis_client, + _user_info_cache, _local_members_cache, _local_user_cache, _inflight_registry, @@ -922,6 +1018,7 @@ class TransmiteServerBuilder SeqGen::ptr _seq_gen; Members::ptr _members_cache; RateLimiter::ptr _rate_limiter; + UserInfoCache::ptr _user_info_cache; LocalCache::ptr _local_members_cache; LocalCache::ptr _local_user_cache; InflightRegistry::ptr _inflight_registry;