Add safe bank failure diagnostics for T5 and T6

Address #14 with request correlation, bounded parser reasons, startup version reporting, and rate-limited storage and settlement logs. Preserve accounting and API identity contracts, extend portable tests, and update release versions and rollout guidance.

t6/plugins/README.md

@@ -109,3 +109,23 @@ decisions and strict response validation for the new bank script. Build and
 install the DLL, then restart the full process. No upstream pins or external
 dependencies changed. See [bank setup and recovery](../../README.md#player-bank-t5-and-t6).
 Portable checks: `python3 tools/test_plugin_bank.py`.
+
+## Bank diagnostics (TWS 1.2.0)
+
+`bankVersion()` returns the loaded DLL's TWS component version, independently of
+upstream pins and the native loader's unchanged `plugin_version()` value.
+`bankResponse()` now also returns `reason`: `ok` on success, otherwise one of
+`empty_body`, `body_limit`, `invalid_json`, `schema`, `field_limit`, `state`,
+`amount_range`, `ok_state`, `missing_id`, or `zero_transfer`. Invalid responses
+clear all parsed fields. Existing acknowledgment acceptance rules are unchanged.
+The returned `code` on valid responses remains API data; bank scripts allowlist
+it before logging. Callers must never log arbitrary response fields.
+
+Native journal logs name load/write/remove, limit identical failures to one
+per 30 seconds, and log recovery with suppressed counts. Exception text, journal
+contents and credentials are excluded. The new scripts require this DLL even
+when disabled by dvar because `bankVersion()` must link. No new third-party
+dependencies or upstream pin changes are required. Rebuild/install the DLL and
+restart the full server process; see the [diagnostic rollout](../../README.md#bank-diagnostics--tws-14).
+Portable checks: `python3 tools/test_plugin_bank.py` executes the real parser,
+storage and native registration adapter with sanitizers and engine shims.

t6/plugins/tws-t6-gsc-utils/src/component/bank.cpp

@@ -3,6 +3,8 @@
 #include "gsc.hpp"
 #include "game/scripting/array.hpp"
 #include <utils/bank.hpp>
+#include <array>
+#include <chrono>
 
 namespace bank
 {
@@ -13,11 +15,33 @@ namespace bank
             return {"tws-bank/t6", server, endpoint};
         }
 
-        void storage_error()
+        // Only fixed operation names are printed, never exception text or paths
+        // from private configuration. Per-operation state has bounded memory.
+        void storage_result(size_t operation, bool failed)
         {
-            // Do not echo parse exceptions: their text can contain player data.
-            printf("^1[tws_bank] Outbox operation failed; check tws-bank/t6 files, disk space and permissions. Existing reports retained.\n");
+            struct status
+            {
+                bool failed = false;
+                std::chrono::steady_clock::time_point logged{};
+                unsigned suppressed = 0;
+            };
+            static std::array<status, 3> states;
+            constexpr const char* names[] = {"load", "write", "remove"};
+            auto& state = states[operation];
+            const auto now = std::chrono::steady_clock::now();
+            if (failed && state.failed && now - state.logged < std::chrono::seconds(30))
+            {
+                ++state.suppressed;
+                return;
+            }
+            if (failed || state.failed)
+                printf("[tws_bank] native_storage operation=%s outcome=%s suppressed=%u\n",
+                    names[operation], failed ? "storage_failure" : "recovered", state.suppressed);
+            state.failed = failed;
+            state.logged = now;
+            state.suppressed = 0;
         }
+
     }
 
     class component final : public component_interface
@@ -25,6 +49,7 @@ namespace bank
     public:
         void on_startup([[maybe_unused]] plugin::plugin* plugin) override
         {
+            gsc::function::add("bankversion", []() { return std::string("1.2.0"); });
             gsc::function::add("bankload", [](const std::string& server, const std::string& endpoint) -> scripting::script_value
             {
                 try
@@ -38,25 +63,27 @@ namespace bank
                         entry["body"] = value.body;
                         entries.emplace_back(entry);
                     }
+                    storage_result(0, false);
                     return entries;
                 }
-                catch (...) { storage_error(); return {}; }
+                catch (...) { storage_result(0, true); return {}; }
             });
             gsc::function::add("bankstore", [](const std::string& server, const std::string& endpoint, const std::string& body)
             {
-                try { spool(server, endpoint).store(body); return true; }
-                catch (...) { storage_error(); return false; }
+                try { spool(server, endpoint).store(body); storage_result(1, false); return true; }
+                catch (...) { storage_result(1, true); return false; }
             });
             gsc::function::add("bankremove", [](const std::string& server, const std::string& endpoint, const std::string& id)
             {
-                try { spool(server, endpoint).erase(id); return true; }
-                catch (...) { storage_error(); return false; }
+                try { spool(server, endpoint).erase(id); storage_result(2, false); return true; }
+                catch (...) { storage_result(2, true); return false; }
             });
             gsc::function::add("bankresponse", [](const std::string& response)
             {
                 const auto ack = utils::bank::parse_acknowledgement(response);
                 scripting::array result;
                 result["valid"] = ack.valid;
+                result["reason"] = ack.reason;
                 result["state"] = ack.state;
                 result["code"] = ack.code;
                 result["request_id"] = ack.id;

t6/plugins/tws-t6-gsc-utils/src/utils/bank.cpp

@@ -29,30 +29,42 @@ namespace utils::bank
 
     acknowledgement parse_acknowledgement(const std::string& response)
     {
-        acknowledgement result;
-        if (response.empty() || response.size() >= max_body_size) return result;
+        auto invalid = [](const char* reason)
+        {
+            acknowledgement value;
+            value.reason = reason;
+            return value;
+        };
+        if (response.empty()) return invalid("empty_body");
+        if (response.size() >= max_body_size) return invalid("body_limit");
         const auto data = nlohmann::json::parse(response, nullptr, false);
-        if (!data.is_object()) return result;
+        if (data.is_discarded()) return invalid("invalid_json");
+        if (!data.is_object()) return invalid("schema");
         try
         {
-            if (!data.at("ok").is_boolean()) return result;
+            acknowledgement result;
+            if (!data.at("ok").is_boolean()) return invalid("schema");
             result.state = data.at("state").get<std::string>();
             result.code = data.at("code").get<std::string>();
             result.id = data.at("request_id").get<std::string>();
-            if (result.id.size() > 64 || result.code.size() > 64 ||
-                (result.state != "balance" && result.state != "prepared" && result.state != "committed" &&
-                 result.state != "cancelled" && result.state != "rejected")) return {};
+            if (result.id.size() > 64 || result.code.size() > 64)
+                return invalid("field_limit");
+            if (result.state != "balance" && result.state != "prepared" && result.state != "committed" &&
+                result.state != "cancelled" && result.state != "rejected") return invalid("state");
             for (const auto* key : {"amount", "balance"})
-                if (!data.at(key).is_number_integer() || data.at(key) < 0 || data.at(key) > 1000000) return {};
+                if (!data.at(key).is_number_integer() || data.at(key) < 0 || data.at(key) > 1000000)
+                    return invalid("amount_range");
             result.amount = data.at("amount").get<int>();
             result.balance = data.at("balance").get<int>();
-            if (data.at("ok").get<bool>() != (result.state != "rejected") ||
-                (result.state != "balance" && result.id.empty()) ||
-                ((result.state == "prepared" || result.state == "committed") && result.amount == 0)) return {};
+            if (data.at("ok").get<bool>() != (result.state != "rejected")) return invalid("ok_state");
+            if (result.state != "balance" && result.id.empty()) return invalid("missing_id");
+            if ((result.state == "prepared" || result.state == "committed") && result.amount == 0)
+                return invalid("zero_transfer");
             result.valid = true;
+            result.reason = "ok";
+            return result;
         }
-        catch (...) { return {}; }
-        return result;
+        catch (...) { return invalid("schema"); }
     }
 
     outbox::outbox(std::filesystem::path root, std::string server, std::string endpoint)

t6/plugins/tws-t6-gsc-utils/src/utils/bank.hpp

@@ -10,6 +10,8 @@ namespace utils::bank
     {
         bool valid = false;
         std::string state, code, id;
+        // Fixed diagnostic vocabulary; never includes response text.
+        std::string reason = "schema";
         int amount = 0, balance = 0;
     };
     acknowledgement parse_acknowledgement(const std::string& response);

t6/scripts/README.md

@@ -14,7 +14,7 @@ private server token.
 | `tws_t6_telemetry.gsc` | Async queued game reports (game_start/heartbeat/round_change/game_end) | tws-t6-gsc-utils |
 | `tws_t6_join.gsc` | `.join` chat command lets a spectator newcomer enter mid-match once | Common |
 | `tws_t6_leaderboard.gsc` | Private map high-round record and every tied holder on connection | Common, tws-t6-gsc-utils |
-| `tws_t6_bank.gsc` | Persistent shared GUID bank | Common, telemetry, TWS plugin 1.1.0 |
+| `tws_t6_bank.gsc` | Persistent shared GUID bank | Common, telemetry, TWS plugin 1.2.0 |
 
 ## Install
 
@@ -44,9 +44,13 @@ its plugin dependency during linking. The same rule applies to leaderboard.
 
 ## Player bank
 
+Bank `2.0.2` requires plugin `1.2.0+` for startup version reporting and bounded
+parser diagnostics. Install the DLL and restart the full server process before
+loading these scripts. Follow the [diagnostic rollout and live checks](../../README.md#bank-diagnostics--tws-14).
+
 The optional `tws_t6_bank.gsc` adds `.b`, `.d <amount|all>` and
 `.w <amount|all>` and registers them in help. It requires common and telemetry
-`1.0.0` or newer, the TWS t6 plugin `1.1.0`, Electric Cherry's bank API and
+`1.0.0` or newer, the TWS t6 plugin `1.2.0`, Electric Cherry's bank API and
 a private per-server credential. Follow [bank setup and recovery](../../README.md#player-bank-t5-and-t6),
 including a full process restart for the plugin. Bank `2.0.0+` uses only
 `https://ec.thewrightserver.net/api/bank` and ignores `tws_bank_url`.

t6/scripts/tws_t6_bank.gsc

@@ -14,16 +14,33 @@ init()
     level.bank_url = "https://ec.thewrightserver.net/api/bank";
     level.bank_server = GetDvar("tws_server_id");
     level.bank_token = GetDvar("tws_bank_token");
+    level.bank_logs = [];
+    level.bank_diagnostic_serial = 0;
+    credentials = "valid";
+    if (level.bank_token.size < 40 || getSubStr(level.bank_token, 0, 8) != "ec_bank_")
+        credentials = "invalid";
+    println("[tws_bank] startup script=2.0.2 plugin=" + bankVersion() +
+        " endpoint=" + level.bank_url + " server=" + bank_log_token(level.bank_server) +
+        " credentials=" + credentials);
     // A private per-server cfg supplies this; never embed bank credentials.
     if (level.bank_server == "" || level.bank_token.size < 40 ||
         getSubStr(level.bank_token, 0, 8) != "ec_bank_")
     {
-        println("^1[tws_bank] Disabled: configure tws_server_id and a private tws_bank_token");
+        println("^1[tws_bank] Disabled: configure tws_server_id and a private tws_bank_token; pending=not_loaded");
         return;
     }
     level.bank_queue = bankLoad(level.bank_server, level.bank_url);
     if (!isDefined(level.bank_queue))
+    {
+        println("[tws_bank] startup journal_load=failure pending=unknown bank=disabled");
         return;
+    }
+    println("[tws_bank] startup journal_load=success pending=" + level.bank_queue.size);
+    for (i = 0; i < level.bank_queue.size; i++)
+    {
+        level.bank_queue[i]["since"] = getTime();
+        level.bank_queue[i]["attempts"] = 0;
+    }
     level.bank_read = 0;
     level.bank_write = level.bank_queue.size;
     level.bank_busy = false;
@@ -222,8 +239,9 @@ bank_transfer(entry, args)
     data["capacity"] = capacity;
     data["decision"] = "cancel";
     body = jsonSerialize(data);
+    pending_since = getTime();
     // Persist cancellation BEFORE prepare can reach EC, including lost replies.
-    if (!bankStore(level.bank_server, level.bank_url, body))
+    if (!bank_store(body, id, "cancel"))
     {
         bank_say(entry, "^3Bank storage unavailable. Your points are unchanged.");
         return;
@@ -241,7 +259,7 @@ bank_transfer(entry, args)
             commit_body = jsonSerialize(data);
             // No yield between recording the decision and updating the balance.
             // Do not change score_total: bank funds are not earned match points.
-            if (bankStore(level.bank_server, level.bank_url, commit_body))
+            if (bank_store(commit_body, id, "commit"))
             {
                 if (operation == "deposit")
                     player.score -= transfer;
@@ -268,6 +286,8 @@ bank_transfer(entry, args)
     pending["id"] = id;
     pending["body"] = body;
     pending["decision"] = data["decision"];
+    pending["since"] = pending_since;
+    pending["attempts"] = 0;
     level.bank_queue[level.bank_write] = pending;
     level.bank_write++;
 }
@@ -287,8 +307,118 @@ bank_error(code)
     return "^3Bank rejected the transfer. Your points are unchanged.";
 }
 
+// Only locally generated identities/configured server names pass through here.
+// Never use this to print response fields: those use fixed vocabularies below.
+bank_log_token(value)
+{
+    if (!isDefined(value) || typeof(value) != "string" || value.size == 0 || value.size > 128)
+        return "unavailable";
+    allowed = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.:-";
+    for (i = 0; i < value.size; i++)
+    {
+        found = false;
+        for (j = 0; j < allowed.size; j++)
+            if (value[i] == allowed[j])
+                found = true;
+        if (!found)
+            return "invalid";
+    }
+    return value;
+}
+
+bank_response_code(code)
+{
+    // Unknown values may contain secrets or console controls, even if short.
+    codes = strTok("ok insufficient_balance empty bank_limit points_limit not_in_match pending invalid_request unauthorized forbidden not_found conflict disabled", " ");
+    for (i = 0; i < codes.size; i++)
+        if (isDefined(code) && code == codes[i])
+            return codes[i];
+    return "unknown";
+}
+
+bank_response_reason(reason)
+{
+    reasons = strTok("empty_body body_limit invalid_json schema field_limit state amount_range ok_state missing_id zero_transfer", " ");
+    for (i = 0; i < reasons.size; i++)
+        if (isDefined(reason) && reason == reasons[i])
+            return reasons[i];
+    return "unknown";
+}
+
+bank_diagnostic_number(diagnostics, key)
+{
+    if (!isDefined(diagnostics) || typeof(diagnostics) != "array" || !isDefined(diagnostics[key]))
+        return "unavailable";
+    value = diagnostics[key];
+    if ((typeof(value) != "int" && typeof(value) != "float") || value < 0 || value > 2147483647)
+        return "unavailable";
+    return "" + int(value);
+}
+
+bank_http_details(diagnostics)
+{
+    detail = "";
+    keys = strTok("http_status curl_code dns_ms connect_ms tls_ms total_ms", " ");
+    for (i = 0; i < keys.size; i++)
+        detail += " " + keys[i] + "=" + bank_diagnostic_number(diagnostics, keys[i]);
+    return detail;
+}
+
+// Fixed keys bound memory. Correlation IDs/timings never form the signature.
+// Report first failure, reason/status/code changes, 30s reminders and recovery.
+bank_log(key, signature, detail, failed)
+{
+    now = getTime();
+    previous = level.bank_logs[key];
+    suppressed = 0;
+    recovery = "";
+    if (isDefined(previous))
+    {
+        suppressed = previous["suppressed"];
+        if (failed && previous["signature"] == signature && now >= previous["time"] && now - previous["time"] < 30000)
+        {
+            previous["suppressed"]++;
+            level.bank_logs[key] = previous;
+            return;
+        }
+        if (previous["failed"] && !failed)
+            recovery = " recovery=true";
+    }
+    println("[tws_bank] " + detail + recovery + " suppressed=" + suppressed);
+    current = [];
+    current["signature"] = signature;
+    current["time"] = now;
+    current["failed"] = failed;
+    current["suppressed"] = 0;
+    level.bank_logs[key] = current;
+}
+
+bank_store(body, id, decision)
+{
+    saved = bankStore(level.bank_server, level.bank_url, body);
+    outcome = "success";
+    if (!saved)
+        outcome = "storage_failure";
+    bank_log("write_" + decision, outcome, "journal_write decision=" + decision +
+        " request_id=" + bank_log_token(id) + " outcome=" + outcome, !saved);
+    return saved;
+}
+
+bank_http_outcome(action, id, correlation, reason, code, diagnostics, failed)
+{
+    level.bank_http_signature = reason + ":" + code + ":" +
+        bank_diagnostic_number(diagnostics, "http_status") + ":" + bank_diagnostic_number(diagnostics, "curl_code");
+    level.bank_http_detail = "correlation=" + correlation + " action=" + action +
+        " request_id=" + bank_log_token(id) + " outcome=" + reason + " code=" + code + bank_http_details(diagnostics);
+    // Settlement adds progress and cleanup outcome before logging this attempt.
+    if (action != "commit" && action != "cancel")
+        bank_log(action, level.bank_http_signature, level.bank_http_detail, failed);
+}
+
 bank_http(action, body, id)
 {
+    level.bank_diagnostic_serial++;
+    correlation = "bank-" + getTime() + "-" + level.bank_diagnostic_serial;
     headers = [];
     headers["Content-Type"] = "application/json";
     headers["Authorization"] = "Bearer " + level.bank_token;
@@ -300,15 +430,46 @@ bank_http(action, body, id)
     options["timeout_ms"] = 15000;
     request = http::request(level.bank_url + "/" + action, options);
     if (!isDefined(request))
+    {
+        bank_http_outcome(action, id, correlation, "request_creation", "none", undefined, true);
         return undefined;
+    }
     request waittill("done", response, success, error, diagnostics);
-    if (!isDefined(success) || !success || !isDefined(response) ||
-        !isDefined(diagnostics) || !isDefined(diagnostics["http_status"]) ||
-        diagnostics["http_status"] != 200)
+    reason = "ok";
+    if (!isDefined(success) || !success)
+        reason = "transport_failure";
+    else if (bank_diagnostic_number(diagnostics, "http_status") == "unavailable")
+        reason = "missing_diagnostics";
+    else if (diagnostics["http_status"] != 200)
+        reason = "http_rejection";
+    else if (!isDefined(response))
+        reason = "missing_response";
+    if (reason != "ok")
+    {
+        bank_http_outcome(action, id, correlation, reason, "none", diagnostics, true);
         return undefined;
+    }
     reply = bankResponse(response);
-    if (!reply["valid"] || reply["request_id"] != id)
+    if (!isDefined(reply) || !isDefined(reply["valid"]) || !reply["valid"])
+    {
+        reason = "unknown";
+        if (isDefined(reply))
+            reason = bank_response_reason(reply["reason"]);
+        bank_http_outcome(action, id, correlation, "invalid_ack_" + reason, "none", diagnostics, true);
         return undefined;
+    }
+    if (reply["request_id"] != id)
+    {
+        bank_http_outcome(action, id, correlation, "request_id_mismatch", "none", diagnostics, true);
+        return undefined;
+    }
+    if (reply["state"] == "rejected")
+        reason = "application_rejection";
+    else if ((action == "balance" && reply["state"] != "balance") ||
+        ((action == "deposit" || action == "withdraw") && reply["state"] != "prepared") ||
+        ((action == "commit" || action == "cancel") && !bank_resolved(action, reply["state"])))
+        reason = "unexpected_state";
+    bank_http_outcome(action, id, correlation, reason, bank_response_code(reply["code"]), diagnostics, reason != "ok");
     return reply;
 }
 
@@ -330,12 +491,20 @@ bank_sender()
             continue;
         }
         entry = level.bank_queue[level.bank_read];
+        entry["attempts"]++;
+        level.bank_queue[level.bank_read] = entry;
         reply = bank_http(entry["decision"], entry["body"], entry["id"]);
+        duration = int((getTime() - entry["since"]) / 1000);
+        progress = " decision=" + entry["decision"] + " attempt=" + entry["attempts"] +
+            " pending_seconds=" + duration;
+        outcome = "pending";
         if (isDefined(reply) && bank_resolved(entry["decision"], reply["state"]))
         {
             // Cleanup failures never reapply points; settlement is idempotent.
             if (bankRemove(level.bank_server, level.bank_url, entry["id"]))
             {
+                bank_log("settlement", "resolved", level.bank_http_detail + progress +
+                    " journal_remove=success settlement=resolved next_retry_seconds=0", false);
                 level.bank_queue[level.bank_read] = undefined;
                 level.bank_read++;
                 if (level.bank_read == level.bank_write)
@@ -347,8 +516,10 @@ bank_sender()
                 delay = 1;
                 continue;
             }
+            outcome = "journal_remove_failure";
         }
-        println("^1[tws_bank] Transfer settlement pending; retaining durable decision and retrying");
+        bank_log("settlement", level.bank_http_signature + ":" + outcome, level.bank_http_detail + progress +
+            " settlement=" + outcome + " next_retry_seconds=" + delay, true);
         wait delay;
         delay *= 2;
         if (delay > 30)