Organize scripts and add TWS plugins with HTTP timeouts
Vendor pinned T5 and T6 plugin sources under the new plugins layout. Add native HTTP deadlines, status and timing diagnostics, telemetry integration, and build/test tooling.
This commit is contained in:
@@ -0,0 +1,485 @@
|
||||
/* tws_t6_telemetry.gsc - Zombies game reports via tws-t6-gsc-utils. */
|
||||
|
||||
#include maps\mp\_utility;
|
||||
#include common_scripts\utility;
|
||||
|
||||
// Plutonium probes both entry points; runtime setup belongs in init.
|
||||
main()
|
||||
{
|
||||
println("[tws_t6_telemetry] main reached (HTTP API target: tws-t6-gsc-utils with deadlines/diagnostics)");
|
||||
}
|
||||
|
||||
init()
|
||||
{
|
||||
println("[tws_t6_telemetry] init reached");
|
||||
|
||||
scripts\zm\tws_t6_common::ensure_config();
|
||||
|
||||
if (GetDvar("tws_telemetry_enabled") == "0")
|
||||
{
|
||||
println("[tws_t6_telemetry] Disabled by tws_telemetry_enabled=0");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDefined(level.telemetry_initialized))
|
||||
return;
|
||||
|
||||
level.telemetry_url = GetDvar("tws_telemetry_url");
|
||||
if (level.telemetry_url == "")
|
||||
level.telemetry_url = "https://ec.thewrightserver.net/api/telemetry";
|
||||
|
||||
if (getSubStr(toLower(level.telemetry_url), 0, 8) != "https://")
|
||||
{
|
||||
println("^1[tws_t6_telemetry] Telemetry disabled: tws_telemetry_url must use HTTPS");
|
||||
return;
|
||||
}
|
||||
|
||||
// Shared reporter secret; update both scripts when rotating it.
|
||||
// This must match Electric Cherry's EC_TELEMETRY_TOKEN environment variable.
|
||||
level.telemetry_token = "e03a03f7d22418f0c062b0b2a7317fd91b1194c4400e84b469866de501180dc2";
|
||||
if (level.telemetry_token.size < 32)
|
||||
{
|
||||
println("^1[tws_t6_telemetry] Telemetry disabled: embedded telemetry token must be at least 32 characters");
|
||||
return;
|
||||
}
|
||||
|
||||
level.telemetry_server = GetDvar("tws_server_id");
|
||||
if (level.telemetry_server == "" || level.telemetry_server.size > 255)
|
||||
{
|
||||
println("^1[tws_t6_telemetry] Set tws_server_id to the exact registered server name (1-255 characters)");
|
||||
return;
|
||||
}
|
||||
|
||||
level.telemetry_initialized = true;
|
||||
level.telemetry_started = false;
|
||||
level.telemetry_ended = false;
|
||||
level.telemetry_last_round = -1;
|
||||
level.telemetry_queue = [];
|
||||
level.telemetry_queue_read = 0;
|
||||
level.telemetry_queue_write = 0;
|
||||
level.telemetry_sequence = 0;
|
||||
level.telemetry_completed = 0;
|
||||
level.telemetry_failed = 0;
|
||||
level.telemetry_http_rejected = 0;
|
||||
level.telemetry_missing_diagnostics_warned = false;
|
||||
level.telemetry_failure_streak = 0;
|
||||
level.telemetry_last_completed = -1;
|
||||
level.telemetry_last_capture = getTime();
|
||||
level.telemetry_round_tick = getTime();
|
||||
level.telemetry_capture_event = "none";
|
||||
level.telemetry_sender_state = "starting";
|
||||
level.telemetry_inflight = undefined;
|
||||
|
||||
telemetry_log(1, "Initialized; waiting for match start. log_level=" + telemetry_log_level());
|
||||
|
||||
// Route through the shared dispatcher; a duplicate connected listener
|
||||
// would fight the T6 common spawn dispatcher.
|
||||
scripts\zm\tws_t6_common::register_spawn_handler(::telemetry_game_start);
|
||||
|
||||
level thread telemetry_game_end_watcher();
|
||||
level thread telemetry_sender();
|
||||
level thread telemetry_round_watcher();
|
||||
level thread telemetry_watchdog();
|
||||
}
|
||||
|
||||
// First playable spawn starts the match, including maps that pre-seed round 1.
|
||||
// Later spawns and joins must not reopen the match or repeat game_start.
|
||||
telemetry_game_start()
|
||||
{
|
||||
if (level.telemetry_started || level.telemetry_ended)
|
||||
return;
|
||||
|
||||
level.telemetry_started = true;
|
||||
level.telemetry_last_capture = getTime();
|
||||
telemetry_log(1, "Match start detected");
|
||||
telemetry_enqueue("game_start");
|
||||
level thread telemetry_heartbeat_watcher();
|
||||
}
|
||||
|
||||
// Start the interval with the match, independently of round changes or joins.
|
||||
telemetry_heartbeat_watcher()
|
||||
{
|
||||
level endon("telemetry_finished");
|
||||
|
||||
for (;;)
|
||||
{
|
||||
wait (30);
|
||||
telemetry_report_round();
|
||||
telemetry_enqueue("heartbeat");
|
||||
}
|
||||
}
|
||||
|
||||
// Independent of HUD installation and player count. Poll for a positive round
|
||||
// with any player present as a fallback in case the dispatcher spawn hook missed
|
||||
// the first spawn (late-attach or stock ordering quirk).
|
||||
telemetry_round_watcher()
|
||||
{
|
||||
level endon("telemetry_finished");
|
||||
|
||||
for (;;)
|
||||
{
|
||||
level.telemetry_round_tick = getTime();
|
||||
if (!level.telemetry_started && isDefined(level.round_number) &&
|
||||
level.round_number >= 1 && getPlayers().size > 0)
|
||||
{
|
||||
telemetry_log(1, "Starting from round_watcher fallback");
|
||||
telemetry_game_start();
|
||||
}
|
||||
|
||||
telemetry_report_round();
|
||||
wait (0.5);
|
||||
}
|
||||
}
|
||||
|
||||
telemetry_report_round()
|
||||
{
|
||||
if (!level.telemetry_started || level.telemetry_ended ||
|
||||
!isDefined(level.round_number) || level.round_number < 0)
|
||||
return;
|
||||
|
||||
round = telemetry_nonnegative_int(level.round_number);
|
||||
if (round == level.telemetry_last_round)
|
||||
return;
|
||||
|
||||
level.telemetry_last_round = round;
|
||||
telemetry_enqueue("round_change");
|
||||
}
|
||||
|
||||
telemetry_game_end_watcher()
|
||||
{
|
||||
// One waiter consumes the first end signal, even if a map fires all three.
|
||||
level waittill_any("end_game", "game_ended", "intermission");
|
||||
telemetry_log(1, "Game-end signal received; stopping producers and draining queue");
|
||||
|
||||
// Preserve order even if the match ends before the next polling tick.
|
||||
telemetry_game_start();
|
||||
telemetry_report_round();
|
||||
level.telemetry_ended = true;
|
||||
level notify("telemetry_finished");
|
||||
telemetry_enqueue("game_end");
|
||||
}
|
||||
|
||||
telemetry_nonnegative_int(value)
|
||||
{
|
||||
if (!isDefined(value) || value < 0)
|
||||
return 0;
|
||||
|
||||
return int(value);
|
||||
}
|
||||
|
||||
// T6 stock initializes and updates match counters in player.pers[] through
|
||||
// _globallogic_score::initpersstat / incpersstat; getpersstat reads this array.
|
||||
telemetry_stat(player, name)
|
||||
{
|
||||
if (!isDefined(player.pers) || !isDefined(player.pers[name]))
|
||||
return 0;
|
||||
|
||||
return telemetry_nonnegative_int(player.pers[name]);
|
||||
}
|
||||
|
||||
telemetry_players_json(include_stats)
|
||||
{
|
||||
players = getPlayers();
|
||||
body = "[";
|
||||
|
||||
for (i = 0; i < players.size; i++)
|
||||
{
|
||||
player = players[i];
|
||||
if (!isDefined(player))
|
||||
{
|
||||
telemetry_log(0, "WARNING: skipping undefined player at roster index " + i);
|
||||
continue;
|
||||
}
|
||||
|
||||
data = [];
|
||||
if (isDefined(player.name))
|
||||
data["name"] = "" + player.name;
|
||||
else if (isDefined(player.playername))
|
||||
data["name"] = "" + player.playername;
|
||||
else
|
||||
{
|
||||
telemetry_log(0, "WARNING: skipping player without a name at roster index " + i);
|
||||
continue;
|
||||
}
|
||||
|
||||
// T6 exposes the persistent per-player identity as the .guid property
|
||||
// (Plutonium engine field; not a method call and not from the plugin).
|
||||
// With a linked Plutonium account it is the account ID; guest/local
|
||||
// sessions fall back to the client slot number, which is expected.
|
||||
// Force to string so the receiver never sees a numeric guid.
|
||||
guid = player.guid;
|
||||
if (!isDefined(guid) || guid == "" || data["name"] == "")
|
||||
{
|
||||
telemetry_log(0, "WARNING: skipping player with incomplete identity at roster index " + i);
|
||||
continue;
|
||||
}
|
||||
data["guid"] = "" + guid;
|
||||
|
||||
if (include_stats)
|
||||
{
|
||||
data["kills"] = telemetry_stat(player, "kills");
|
||||
data["headshots"] = telemetry_stat(player, "headshots");
|
||||
data["downs"] = telemetry_stat(player, "downs");
|
||||
data["revives"] = telemetry_stat(player, "revives");
|
||||
// Current spendable points, not the lifetime points-earned stat.
|
||||
data["score"] = telemetry_nonnegative_int(player.score);
|
||||
}
|
||||
|
||||
if (body != "[")
|
||||
body += ",";
|
||||
body += jsonSerialize(data);
|
||||
}
|
||||
|
||||
// t6-gsc-utils serializes an empty GSC array as null. Build only the
|
||||
// array delimiters ourselves; the serializer still escapes all strings.
|
||||
return body + "]";
|
||||
}
|
||||
|
||||
telemetry_enqueue(event_name)
|
||||
{
|
||||
level.telemetry_capture_event = event_name;
|
||||
telemetry_log(2, "Capturing event=" + event_name);
|
||||
// Snapshot and serialize before yielding, including during game over.
|
||||
body = "{\"event\":" + jsonSerialize(event_name);
|
||||
body += ",\"server\":" + jsonSerialize(level.telemetry_server);
|
||||
body += ",\"map\":" + jsonSerialize(GetDvar("mapname"));
|
||||
if (event_name != "game_start")
|
||||
body += ",\"round\":" + telemetry_nonnegative_int(level.round_number);
|
||||
body += ",\"players\":" + telemetry_players_json(event_name != "game_start") + "}";
|
||||
|
||||
entry = [];
|
||||
level.telemetry_sequence++;
|
||||
entry["id"] = level.telemetry_sequence;
|
||||
entry["event"] = event_name;
|
||||
entry["round"] = telemetry_nonnegative_int(level.round_number);
|
||||
entry["captured"] = getTime();
|
||||
entry["body"] = body;
|
||||
level.telemetry_queue[level.telemetry_queue_write] = entry;
|
||||
level.telemetry_queue_write++;
|
||||
level.telemetry_last_capture = getTime();
|
||||
level.telemetry_capture_event = "none";
|
||||
telemetry_log(2, "Queued " + telemetry_event_label(entry) + " pending=" + telemetry_pending());
|
||||
level notify("telemetry_queued");
|
||||
}
|
||||
|
||||
// A single worker keeps one request in flight. It survives game-over notifies
|
||||
// to drain the final snapshot; no retries, discovery, or beta-specific calls.
|
||||
telemetry_sender()
|
||||
{
|
||||
for (;;)
|
||||
{
|
||||
if (level.telemetry_queue_read == level.telemetry_queue_write)
|
||||
{
|
||||
level.telemetry_queue = [];
|
||||
level.telemetry_queue_read = 0;
|
||||
level.telemetry_queue_write = 0;
|
||||
if (level.telemetry_ended)
|
||||
{
|
||||
level.telemetry_sender_state = "drained";
|
||||
telemetry_log(1, "Queue drained after game end; completed=" + level.telemetry_completed + " failed=" + level.telemetry_failed);
|
||||
level notify("telemetry_drained");
|
||||
return;
|
||||
}
|
||||
|
||||
level.telemetry_sender_state = "idle";
|
||||
level waittill("telemetry_queued");
|
||||
continue;
|
||||
}
|
||||
|
||||
entry = level.telemetry_queue[level.telemetry_queue_read];
|
||||
level.telemetry_queue[level.telemetry_queue_read] = undefined;
|
||||
level.telemetry_queue_read++;
|
||||
telemetry_send(entry);
|
||||
}
|
||||
}
|
||||
|
||||
// v3.1.1 global alias: httpPost(url, options) with options["method"|"headers"|"body"].
|
||||
// The plugin exposes no cancellation API and no HTTP status code; treat the
|
||||
// success flag as transport completion only and never overlap requests.
|
||||
telemetry_send(entry)
|
||||
{
|
||||
level.telemetry_inflight = entry;
|
||||
level.telemetry_request_started = getTime();
|
||||
level.telemetry_sender_state = "creating_request";
|
||||
label = telemetry_event_label(entry);
|
||||
telemetry_log(1, "Sending " + label + " queue_ms=" + (getTime() - entry["captured"]) + " pending=" + telemetry_pending());
|
||||
|
||||
headers = [];
|
||||
headers["Content-Type"] = "application/json";
|
||||
headers["Authorization"] = "Bearer " + level.telemetry_token;
|
||||
|
||||
options = [];
|
||||
options["method"] = "POST";
|
||||
options["headers"] = headers;
|
||||
options["body"] = entry["body"];
|
||||
options["connect_timeout_ms"] = 5000;
|
||||
options["timeout_ms"] = 15000;
|
||||
|
||||
request = httpPost(level.telemetry_url, options);
|
||||
if (!isDefined(request))
|
||||
{
|
||||
telemetry_transport_failed(label + " request handle is undefined; skipping event");
|
||||
level.telemetry_inflight = undefined;
|
||||
level.telemetry_sender_state = "idle";
|
||||
return;
|
||||
}
|
||||
|
||||
level.telemetry_sender_state = "waiting_http";
|
||||
request waittill("done", response, success, error, diagnostics);
|
||||
elapsed = getTime() - level.telemetry_request_started;
|
||||
level.telemetry_last_completed = getTime();
|
||||
timing = telemetry_http_diagnostics(diagnostics);
|
||||
if (!isDefined(diagnostics) && !level.telemetry_missing_diagnostics_warned)
|
||||
{
|
||||
level.telemetry_missing_diagnostics_warned = true;
|
||||
telemetry_log(0, "WARNING: plugin returned no HTTP diagnostics; native timeouts are unverified. Install the TWS plugin build and restart the server process");
|
||||
}
|
||||
|
||||
// The plugin's success flag reports transport completion, not HTTP status.
|
||||
// No automatic retry: the receiver may already have accepted the event.
|
||||
if (!isDefined(success) || !success)
|
||||
{
|
||||
if (!isDefined(error))
|
||||
error = "unknown transport error";
|
||||
telemetry_transport_failed(label + " elapsed_ms=" + elapsed + timing + " error=" + telemetry_preview(error));
|
||||
}
|
||||
else
|
||||
{
|
||||
level.telemetry_completed++;
|
||||
if (level.telemetry_failure_streak > 0)
|
||||
telemetry_log(0, "Transport recovered after " + level.telemetry_failure_streak + " failures; " + label);
|
||||
level.telemetry_failure_streak = 0;
|
||||
if (isDefined(diagnostics) && isDefined(diagnostics["http_status"]))
|
||||
{
|
||||
if (diagnostics["http_status"] < 200 || diagnostics["http_status"] >= 300)
|
||||
{
|
||||
level.telemetry_http_rejected++;
|
||||
telemetry_log(0, "ERROR: HTTP rejected " + label + " elapsed_ms=" + elapsed + timing + " body=" + telemetry_preview(response));
|
||||
}
|
||||
else
|
||||
telemetry_log(1, "HTTP completed " + label + " elapsed_ms=" + elapsed + timing);
|
||||
}
|
||||
else
|
||||
telemetry_log(1, "Transport completed " + label + " elapsed_ms=" + elapsed + timing + " (API acceptance unverified)");
|
||||
if (!isDefined(response) || response == "")
|
||||
telemetry_log(0, "WARNING: empty API response for " + label);
|
||||
else
|
||||
telemetry_log(2, "Response " + label + " body=" + telemetry_preview(response));
|
||||
}
|
||||
|
||||
level.telemetry_inflight = undefined;
|
||||
level.telemetry_sender_state = "idle";
|
||||
}
|
||||
|
||||
// The native plugin appends diagnostics while preserving the old done arguments.
|
||||
// Curl timings are cumulative milestones, measured in real milliseconds.
|
||||
telemetry_http_diagnostics(diagnostics)
|
||||
{
|
||||
if (!isDefined(diagnostics))
|
||||
return " diagnostics=unavailable";
|
||||
|
||||
keys = strTok("http_status,curl_code,dns_ms,connect_ms,tls_ms,first_byte_ms,total_ms,worker_queue_ms,notify_ms,connect_timeout_ms,timeout_ms", ",");
|
||||
detail = "";
|
||||
for (i = 0; i < keys.size; i++)
|
||||
{
|
||||
key = keys[i];
|
||||
if (isDefined(diagnostics[key]))
|
||||
detail += " " + key + "=" + int(diagnostics[key]);
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
telemetry_transport_failed(detail)
|
||||
{
|
||||
level.telemetry_failed++;
|
||||
level.telemetry_failure_streak++;
|
||||
telemetry_log(0, "ERROR: transport failed " + detail + " consecutive_failures=" + level.telemetry_failure_streak);
|
||||
}
|
||||
|
||||
telemetry_pending()
|
||||
{
|
||||
return level.telemetry_queue_write - level.telemetry_queue_read;
|
||||
}
|
||||
|
||||
telemetry_event_label(entry)
|
||||
{
|
||||
return "id=" + entry["id"] + " event=" + entry["event"] + " round=" + entry["round"];
|
||||
}
|
||||
|
||||
// Read live so diagnostics can be increased without restarting the map.
|
||||
telemetry_log_level()
|
||||
{
|
||||
raw = GetDvar("tws_telemetry_log_level");
|
||||
if (raw == "0")
|
||||
return 0;
|
||||
if (raw == "2")
|
||||
return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
telemetry_log(verbosity, message)
|
||||
{
|
||||
if (verbosity > telemetry_log_level())
|
||||
return;
|
||||
|
||||
println("[tws_t6_telemetry] server=" + level.telemetry_server + " map=" + GetDvar("mapname") + " time_ms=" + getTime() + " " + message);
|
||||
}
|
||||
|
||||
// Escape newlines/control characters and bound response/error output. Do not
|
||||
// jsonParse arbitrary replies: a proxy may return HTML or malformed JSON.
|
||||
telemetry_preview(value)
|
||||
{
|
||||
if (!isDefined(value))
|
||||
return "<undefined>";
|
||||
value = "" + value;
|
||||
if (value.size > 256)
|
||||
value = getSubStr(value, 0, 256) + "... [truncated]";
|
||||
return jsonSerialize(value);
|
||||
}
|
||||
|
||||
// Independent of producers and the HTTP wait; survives game over until drain.
|
||||
// Native requests have deadlines. Warn without opening overlapping POSTs
|
||||
// that could arrive out of order or exhaust its native worker pool.
|
||||
telemetry_watchdog()
|
||||
{
|
||||
level endon("telemetry_drained");
|
||||
for (;;)
|
||||
{
|
||||
wait (30);
|
||||
now = getTime();
|
||||
pending = telemetry_pending();
|
||||
completion_age = -1;
|
||||
if (level.telemetry_last_completed >= 0)
|
||||
completion_age = now - level.telemetry_last_completed;
|
||||
|
||||
telemetry_log(1, "Health started=" + level.telemetry_started + " ended=" + level.telemetry_ended +
|
||||
" sender=" + level.telemetry_sender_state + " capturing=" + level.telemetry_capture_event +
|
||||
" pending=" + pending + " queued_total=" + level.telemetry_sequence +
|
||||
" completed=" + level.telemetry_completed + " failed=" + level.telemetry_failed + " http_rejected=" + level.telemetry_http_rejected +
|
||||
" last_done_ms_ago=" + completion_age);
|
||||
|
||||
if (isDefined(level.telemetry_inflight))
|
||||
{
|
||||
age = now - level.telemetry_request_started;
|
||||
if (age >= 30000)
|
||||
telemetry_log(0, "WARNING: HTTP stalled " + telemetry_event_label(level.telemetry_inflight) +
|
||||
" elapsed_ms=" + age + " pending=" + pending + " stage=" + level.telemetry_sender_state +
|
||||
"; native deadline exceeded; check plugin version, worker queue and completion delivery");
|
||||
}
|
||||
else if (pending > 0)
|
||||
{
|
||||
oldest = level.telemetry_queue[level.telemetry_queue_read];
|
||||
if (now - oldest["captured"] >= 30000)
|
||||
telemetry_log(0, "WARNING: sender is not consuming queued reports; " + telemetry_event_label(oldest));
|
||||
}
|
||||
|
||||
if (!level.telemetry_ended)
|
||||
{
|
||||
if (now - level.telemetry_round_tick > 5000)
|
||||
telemetry_log(0, "WARNING: round watcher has stopped advancing");
|
||||
if (level.telemetry_started && now - level.telemetry_last_capture >= 65000)
|
||||
telemetry_log(0, "WARNING: no snapshot captured for " + (now - level.telemetry_last_capture) +
|
||||
"ms; capture_event=" + level.telemetry_capture_event);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user