1374 lines
50 KiB
PHP
1374 lines
50 KiB
PHP
<?php
|
|
|
|
/*
|
|
Phoronix Test Suite
|
|
URLs: http://www.phoronix.com, http://www.phoronix-test-suite.com/
|
|
Copyright (C) 2009 - 2022, Phoronix Media
|
|
Copyright (C) 2009 - 2022, Michael Larabel
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
class phoromatic extends pts_module_interface
|
|
{
|
|
const module_name = 'Phoromatic Client';
|
|
const module_version = '1.2.0';
|
|
const module_description = 'The Phoromatic client is used for connecting to a Phoromatic server (Phoromatic.com or a locally run server) to facilitate the automatic running of tests, generally across multiple test nodes in a routine manner. For more details visit http://www.phoromatic.com/. This module is intended to be used with Phoronix Test Suite 5.2+ clients and servers.';
|
|
const module_author = 'Phoronix Media';
|
|
|
|
private static $account_id = null;
|
|
private static $server_address = null;
|
|
private static $server_http_port = null;
|
|
private static $use_https = false;
|
|
private static $server_ws_port = null;
|
|
private static $is_running_as_phoromatic_node = false;
|
|
private static $log_file = null;
|
|
private static $limit_network_communication = false;
|
|
private static $system_id = null;
|
|
private static $server_core_version = 0;
|
|
private static $progressive_result_uploads = false;
|
|
|
|
private static $p_save_identifier = null;
|
|
private static $p_schedule_id = null;
|
|
private static $p_trigger_id = null;
|
|
private static $benchmark_ticket_id = null;
|
|
private static $in_stress_mode = false;
|
|
private static $has_run_server_setup_func = false;
|
|
|
|
private static $test_run_manager = null;
|
|
|
|
public static function module_info()
|
|
{
|
|
return 'The Phoromatic module contains the client support for interacting with Phoromatic and Phoromatic Tracker services.';
|
|
}
|
|
public static function user_commands()
|
|
{
|
|
return array('connect' => 'run_connection', 'explore' => 'explore_network', 'upload_result' => 'upload_unscheduled_result', 'set_root_admin_password' => 'set_root_admin_password', 'list_results' => 'recent_phoromatic_server_results', 'clone' => 'clone_phoromatic_server_result', 'export_results_for_account_schedules' => 'generate_export_result_schedule_dump');
|
|
}
|
|
public static function upload_unscheduled_result($args)
|
|
{
|
|
$server_setup = self::setup_server_addressing($args);
|
|
|
|
if(!$server_setup)
|
|
return false;
|
|
|
|
$uploads = 0;
|
|
foreach($args as $arg)
|
|
{
|
|
if(pts_types::is_result_file($arg))
|
|
{
|
|
$uploads++;
|
|
echo PHP_EOL . 'Uploading: ' . $arg . PHP_EOL;
|
|
$result_file = pts_types::identifier_to_object($arg);
|
|
$server_response = self::upload_test_result($result_file);
|
|
$server_response = json_decode($server_response, true);
|
|
if(isset($server_response['phoromatic']['response']))
|
|
echo ' Result Uploaded' . PHP_EOL;
|
|
else
|
|
echo ' Upload Failed' . PHP_EOL;
|
|
}
|
|
}
|
|
|
|
if($uploads == 0)
|
|
echo PHP_EOL . 'No Result Files Found To Upload.' . PHP_EOL;
|
|
}
|
|
public static function set_root_admin_password()
|
|
{
|
|
phoromatic_server::prepare_database();
|
|
$root_admin_pw = phoromatic_server::read_setting('root_admin_pw');
|
|
|
|
// DISABLED THE INITIAL CHECKING CODE SINCE REALLY WAS SILLY SINCE IF THEY ALREADY HAVE TERMINAL ACCESS, THEY CAN ACCESS FILE ANYWAY....
|
|
if(false && $root_admin_pw != null)
|
|
{
|
|
do
|
|
{
|
|
$check_root_pw = pts_user_io::prompt_user_input('Please enter the existing root-admin password');
|
|
}
|
|
while(hash('sha256', 'PTS' . $check_root_pw) != $root_admin_pw);
|
|
}
|
|
|
|
echo PHP_EOL . 'The new root-admin password must be at least six characters long.' . PHP_EOL;
|
|
do
|
|
{
|
|
$new_root_pw = pts_user_io::prompt_user_input('Please enter the new root-admin password');
|
|
}
|
|
while(strlen($new_root_pw) < 6);
|
|
|
|
$new_root_pw = hash('sha256', 'PTS' . $new_root_pw);
|
|
$root_admin_pw = phoromatic_server::save_setting('root_admin_pw', $new_root_pw);
|
|
}
|
|
public static function generate_export_result_schedule_dump($r)
|
|
{
|
|
phoromatic_server::prepare_database();
|
|
if(isset($r[0]) && !empty($r[0]))
|
|
{
|
|
phoromatic_server::generate_result_export_dump($r[0]);
|
|
}
|
|
}
|
|
public static function phoromatic_server_supports_https($address, $port)
|
|
{
|
|
// Use 3 second timeout for HTTPS request to ensure not wasting too much time...
|
|
$response = pts_network::http_get_contents('https://' . $address . ':' . $port . '/server.php?phoromatic_info', false, false, false, false, 3);
|
|
|
|
if(!empty($response) && ($response = json_decode($response, true)) != null && isset($response['pts']))
|
|
{
|
|
// Successfully obtaining data via HTTPS
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
public static function explore_network()
|
|
{
|
|
pts_client::$display->generic_heading('Phoromatic Servers');
|
|
$archived_servers = pts_client::available_phoromatic_servers();
|
|
|
|
$server_count = 0;
|
|
foreach($archived_servers as $archived_server)
|
|
{
|
|
$supports_https = $archived_server['protocol'] == 'https'; // Previous code: self::phoromatic_server_supports_https($archived_server['ip'], $archived_server['http_port']);
|
|
$response = pts_network::http_get_contents(($supports_https ? 'https://' : 'http://') . $archived_server['ip'] . ':' . $archived_server['http_port'] . '/server.php?phoromatic_info');
|
|
|
|
if(!empty($response))
|
|
{
|
|
$response = json_decode($response, true);
|
|
if($response && isset($response['pts']))
|
|
{
|
|
$server_count++;
|
|
echo PHP_EOL . 'IP: ' . $archived_server['ip'] . PHP_EOL;
|
|
echo 'HTTP PORT: ' . $archived_server['http_port'] . PHP_EOL;
|
|
echo 'PROTOCOL: ' . ($supports_https ? 'HTTPS': 'HTTP') . PHP_EOL;
|
|
echo 'WEBSOCKET PORT: ' . $response['ws_port'] . PHP_EOL;
|
|
echo 'SERVER: ' . $response['http_server'] . PHP_EOL;
|
|
echo 'PHORONIX TEST SUITE: ' . $response['pts'] . ' [' . $response['pts_core'] . ']' . PHP_EOL;
|
|
|
|
// TODO XXX fix/finish below code...
|
|
if(false && ($ws = new phoromatic_client_comm_ws($archived_server['ip'], $response['ws_port'])))
|
|
{
|
|
// Query the WebSocket Server for some Phoromatic Server details
|
|
$s = $ws->send(array('phoromatic' => array('event' => 'pts-version')));
|
|
$s = $ws->send(array('phoromatic' => array('event' => 'download-cache')));
|
|
$r = $ws->receive_until('download-cache');
|
|
var_dump($r);
|
|
}
|
|
else
|
|
{
|
|
|
|
// Provide some other server info via HTTP
|
|
|
|
$repo = pts_network::http_get_contents(($supports_https ? 'https://' : 'http://') . $archived_server['ip'] . ':' . $archived_server['http_port'] . '/download-cache.php?repo');
|
|
echo 'DOWNLOAD CACHE: ';
|
|
if(!empty($repo))
|
|
{
|
|
$repo = json_decode($repo, true);
|
|
if($repo && isset($repo['phoronix-test-suite']['download-cache']))
|
|
{
|
|
$total_file_size = 0;
|
|
foreach($repo['phoronix-test-suite']['download-cache'] as $file_name => $inf)
|
|
{
|
|
$total_file_size += $repo['phoronix-test-suite']['download-cache'][$file_name]['file_size'];
|
|
}
|
|
echo count($repo['phoronix-test-suite']['download-cache']) . ' FILES / ' . round($total_file_size / 1000000) . ' MB CACHE SIZE';
|
|
}
|
|
}
|
|
else
|
|
{
|
|
echo 'N/A';
|
|
}
|
|
}
|
|
|
|
echo PHP_EOL;
|
|
|
|
$repo = pts_network::http_get_contents(($supports_https ? 'https://' : 'http://') . $archived_server['ip'] . ':' . $archived_server['http_port'] . '/openbenchmarking-cache.php?repos');
|
|
echo 'SUPPORTED OPENBENCHMARKING.ORG REPOSITORIES:' . PHP_EOL;
|
|
if(!empty($repo))
|
|
{
|
|
$repo = json_decode($repo, true);
|
|
if($repo && is_array($repo['repos']))
|
|
{
|
|
foreach($repo['repos'] as $data)
|
|
{
|
|
echo ' ' . $data['title'] . ' - Last Generated: ' . date('d M Y H:i', $data['generated']) . PHP_EOL;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
echo ' N/A' . PHP_EOL;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if($server_count == 0)
|
|
{
|
|
echo PHP_EOL . 'No Phoromatic Servers detected.' . PHP_EOL . PHP_EOL;
|
|
}
|
|
}
|
|
protected static function tick_thread($force_manual_poll = false)
|
|
{
|
|
static $last_phoromatic_log = 0;
|
|
|
|
while(true)
|
|
{
|
|
$j = array();
|
|
|
|
$log_size = pts_client::$pts_logger->get_log_file_size();
|
|
if($log_size != $last_phoromatic_log)
|
|
{
|
|
$phoromatic_log = file_get_contents(pts_client::$pts_logger->get_log_file_location());
|
|
$last_phoromatic_log = $log_size;
|
|
$j['phoromatic']['client-log'] = $phoromatic_log;
|
|
}
|
|
|
|
foreach(phodevi::query_sensors() as $sensor)
|
|
{
|
|
$j['phoromatic']['stats']['sensors'][phodevi::sensor_name($sensor)] = array('value' => phodevi::read_sensor($sensor), 'unit' => phodevi::read_sensor_unit($sensor));
|
|
}
|
|
|
|
$j['phoromatic']['stats']['uptime'] = ceil(phodevi::system_uptime() / 60);
|
|
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'tick',
|
|
'j' => json_encode($j),
|
|
));
|
|
|
|
if($force_manual_poll)
|
|
{
|
|
return;
|
|
}
|
|
|
|
$server_response = json_decode($server_response, true);
|
|
if($server_response && isset($server_response['phoromatic']['tick_thread']))
|
|
{
|
|
switch($server_response['phoromatic']['tick_thread'])
|
|
{
|
|
case 'reboot':
|
|
phodevi::reboot();
|
|
break;
|
|
case 'halt-testing':
|
|
touch(PTS_USER_PATH . 'halt-testing');
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Randomize the thread work a little bit to ensure not hitting the systems at the same time
|
|
sleep(rand(60, 90));
|
|
}
|
|
}
|
|
protected static function upload_to_remote_server($to_post, $server_address = null, $server_http_port = null, $account_id = null, $try_https = false)
|
|
{
|
|
static $last_communication_minute = null;
|
|
static $communication_attempts = 0;
|
|
|
|
if($last_communication_minute == date('i') && $communication_attempts > 8)
|
|
{
|
|
// Something is wrong, Phoromatic shouldn't be communicating with server more than four times a minute
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(date('i') != $last_communication_minute)
|
|
{
|
|
$last_communication_minute = date('i');
|
|
$communication_attempts = 0;
|
|
}
|
|
|
|
$communication_attempts++;
|
|
}
|
|
|
|
if($server_address == null && self::$server_address != null)
|
|
{
|
|
$server_address = self::$server_address;
|
|
}
|
|
if($server_http_port == null && self::$server_http_port != null)
|
|
{
|
|
$server_http_port = self::$server_http_port;
|
|
}
|
|
if($account_id == null && self::$account_id != null)
|
|
{
|
|
$account_id = self::$account_id;
|
|
}
|
|
|
|
if($try_https)
|
|
{
|
|
$try_https = self::phoromatic_server_supports_https($server_address, $server_http_port);
|
|
}
|
|
|
|
$to_post['aid'] = $account_id;
|
|
$to_post['pts'] = PTS_VERSION;
|
|
$to_post['pts_core'] = PTS_CORE_VERSION;
|
|
$to_post['gsid'] = defined('PTS_GSID') ? PTS_GSID : null;
|
|
$to_post['lip'] = phodevi::read_property('network', 'ip');
|
|
$to_post['h'] = phodevi::system_hardware(true);
|
|
$to_post['nm'] = phodevi::read_property('network', 'mac-address');
|
|
$to_post['nw'] = implode(', ', pts_network::get_network_wol());
|
|
$to_post['s'] = phodevi::system_software(true);
|
|
$to_post['n'] = phodevi::read_property('system', 'hostname');
|
|
$to_post['pp'] = json_encode(phodevi::read_all_properties());
|
|
$to_post['msi'] = PTS_MACHINE_SELF_ID;
|
|
return pts_network::http_upload_via_post((self::$use_https || $try_https ? 'https://' : 'http://') . $server_address . ':' . $server_http_port . '/phoromatic.php', $to_post, false);
|
|
}
|
|
protected static function update_system_status($current_task, $estimated_time_remaining = 0, $percent_complete = 0, $for_schedule = null, $estimate_to_next_comm = 0)
|
|
{
|
|
static $last_msg = null;
|
|
|
|
// Avoid an endless flow of "idling" messages, etc
|
|
if($current_task != $last_msg)
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log($current_task);
|
|
$last_msg = $current_task;
|
|
|
|
if(self::$limit_network_communication)
|
|
{
|
|
static $last_comm_time = 0;
|
|
if(time() > ($last_comm_time + 800 + rand(0, 180)))
|
|
{
|
|
// It's been at least half hour since last update, so report in state...
|
|
$last_comm_time = time();
|
|
}
|
|
else
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
|
|
return phoromatic::upload_to_remote_server(array(
|
|
'r' => 'update_system_status',
|
|
'a' => $current_task,
|
|
'time' => $estimated_time_remaining,
|
|
'pc' => $percent_complete,
|
|
'sched' => (!empty($for_schedule) ? $for_schedule : self::$p_schedule_id),
|
|
'bid' => (!empty(self::$benchmark_ticket_id) ? self::$benchmark_ticket_id : 0),
|
|
'o' => $estimate_to_next_comm
|
|
));
|
|
}
|
|
protected static function set_server_info($address, $port, $account_id)
|
|
{
|
|
self::$server_address = $address;
|
|
self::$server_http_port = $port;
|
|
self::$account_id = $account_id;
|
|
self::$use_https = self::phoromatic_server_supports_https(self::$server_address, self::$server_http_port);
|
|
}
|
|
protected static function setup_server_addressing($server_string = null)
|
|
{
|
|
self::$has_run_server_setup_func = true;
|
|
|
|
if(isset($server_string[0]) && strpos($server_string[0], '/', strpos($server_string[0], ':')) > 6)
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to connect to Phoromatic Server: ' . $server_string[0]);
|
|
self::set_server_info(substr($server_string[0], 0, strpos($server_string[0], ':')), substr($server_string[0], strlen(substr($server_string[0], 0, strpos($server_string[0], ':'))) + 1, -1 - strlen(substr($server_string[0], strrpos($server_string[0], '/') + 1))), substr($server_string[0], strrpos($server_string[0], '/') + 1));
|
|
pts_client::$display->generic_heading('Server IP: ' . self::$server_address . PHP_EOL . 'Server HTTP Port: ' . self::$server_http_port . PHP_EOL . 'Account ID: ' . self::$account_id);
|
|
pts_client::register_phoromatic_server(self::$server_address, self::$server_http_port);
|
|
}
|
|
else if(($last_server = trim(pts_module::read_file('last-phoromatic-server'))) && !empty($last_server))
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to connect to last server connection: ' . $last_server);
|
|
$last_account_id = substr($last_server, strrpos($last_server, '/') + 1);
|
|
$last_server_address = substr($last_server, 0, strpos($last_server, ':'));
|
|
$last_server_http_port = substr($last_server, strlen($last_server_address) + 1, -1 - strlen($last_account_id));
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Last Server IP: ' . $last_server_address . ' Last Server HTTP Port: ' . $last_server_http_port . ' Last Account ID: ' . $last_account_id);
|
|
|
|
for($i = 0; $i < 10; $i++)
|
|
{
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'ping',
|
|
), $last_server_address, $last_server_http_port, $last_account_id, true);
|
|
|
|
$server_response = json_decode($server_response, true);
|
|
if($server_response && isset($server_response['phoromatic']['ping']))
|
|
{
|
|
self::set_server_info($last_server_address, $last_server_http_port, $last_account_id);
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Phoromatic Server connection restored');
|
|
pts_client::register_phoromatic_server(self::$server_address, self::$server_http_port);
|
|
break;
|
|
}
|
|
else
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Phoromatic Server connection failed');
|
|
sleep((12 * ($i + 1)));
|
|
}
|
|
}
|
|
}
|
|
|
|
if(self::$server_address == null)
|
|
{
|
|
$archived_servers = pts_client::available_phoromatic_servers();
|
|
if(!empty($archived_servers))
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to auto-discover Phoromatic Servers');
|
|
self::attempt_phoromatic_server_auto_discover($archived_servers);
|
|
}
|
|
}
|
|
|
|
if(self::$server_address == null || self::$server_http_port == null || self::$account_id == null)
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Phoromatic Server connection setup failed');
|
|
echo PHP_EOL . 'You must pass the Phoromatic Server information as an argument to phoromatic.connect, or otherwise configure your network setup.' . PHP_EOL . ' e.g. phoronix-test-suite phoromatic.connect 192.168.1.2:5555/I0SSJY' . PHP_EOL . PHP_EOL;
|
|
|
|
if(PTS_IS_DAEMONIZED_SERVER_PROCESS && !empty($archived_servers))
|
|
{
|
|
echo 'The Phoromatic client appears to be running as a system service/daemon so will attempt to continue auto-polling to find the Phoromatic Server.' . PHP_EOL . PHP_EOL;
|
|
|
|
$success = false;
|
|
do
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Will auto-poll connected servers every 90 seconds looking for a claim by a Phoromatic Server');
|
|
sleep(90);
|
|
$success = self::attempt_phoromatic_server_auto_discover($archived_servers);
|
|
}
|
|
while($success == false);
|
|
}
|
|
else
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
protected static function attempt_phoromatic_server_auto_discover(&$phoromatic_servers)
|
|
{
|
|
foreach($phoromatic_servers as &$archived_server)
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to auto-discover Phoromatic Server on: ' . $archived_server['ip'] . ': ' . $archived_server['http_port']);
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'ping',
|
|
), $archived_server['ip'], $archived_server['http_port'], null, true);
|
|
|
|
$server_response = json_decode($server_response, true);
|
|
if($server_response && isset($server_response['phoromatic']['account_id']))
|
|
{
|
|
self::set_server_info($archived_server['ip'], $archived_server['http_port'], $server_response['phoromatic']['account_id']);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
protected static function setup_system_environment()
|
|
{
|
|
if(is_writable('/boot/grub/grubenv') && pts_client::executable_in_path('grub-editenv'))
|
|
{
|
|
// In case system fails or reboots in process and don't want to hang on GRUB recordfail
|
|
shell_exec('grub-editenv /boot/grub/grubenv unset recordfail 2>&1');
|
|
}
|
|
}
|
|
public static function run_connection($args)
|
|
{
|
|
if(pts_client::create_lock(PTS_USER_PATH . 'phoromatic_lock') == false)
|
|
{
|
|
trigger_error('Phoromatic is already running.', E_USER_ERROR);
|
|
return false;
|
|
}
|
|
define('PHOROMATIC_PROCESS', true);
|
|
|
|
if(pts_client::$pts_logger == false)
|
|
{
|
|
pts_client::$pts_logger = new pts_logger(null, null, true, true);
|
|
}
|
|
pts_client::$pts_logger->log(pts_core::program_title() . ' [' . PTS_CORE_VERSION . '] starting Phoromatic client');
|
|
|
|
if(phodevi::system_uptime() < 60)
|
|
{
|
|
echo 'PHOROMATIC: Sleeping for 60 seconds as system freshly started.' . PHP_EOL;
|
|
pts_client::$pts_logger->log('Sleeping for 60 seconds as system freshly started');
|
|
sleep(60);
|
|
}
|
|
|
|
$server_setup = self::setup_server_addressing($args);
|
|
//$http_comm = new phoromatic_client_comm_http();
|
|
|
|
/*
|
|
if(!$server_setup)
|
|
{
|
|
if(getenv('PTS_NO_REBOOT_ON_NETWORK_FAILURE') == false && PTS_IS_DAEMONIZED_SERVER_PROCESS)
|
|
{
|
|
phodevi::reboot();
|
|
}
|
|
|
|
return false;
|
|
}
|
|
*/
|
|
|
|
$times_failed = 0;
|
|
$has_success = false;
|
|
$do_exit = false;
|
|
$just_started = true;
|
|
|
|
self::setup_system_environment();
|
|
pts_client::$pts_logger->log('SYSTEM HARDWARE: ' . phodevi::system_hardware(true));
|
|
pts_client::$pts_logger->log('SYSTEM SOFTWARE: ' . phodevi::system_software(true));
|
|
|
|
while($do_exit == false)
|
|
{
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'start',
|
|
));
|
|
|
|
if($server_response == false)
|
|
{
|
|
$times_failed++;
|
|
|
|
pts_client::$pts_logger->log('Server response failed');
|
|
|
|
if($times_failed >= 2)
|
|
{
|
|
trigger_error('Communication with server failed.', E_USER_ERROR);
|
|
|
|
if(PTS_IS_DAEMONIZED_SERVER_PROCESS == false && $times_failed > 5)
|
|
{
|
|
return false;
|
|
}
|
|
else if(PTS_IS_DAEMONIZED_SERVER_PROCESS && $times_failed > 10)
|
|
{
|
|
if(getenv('PTS_NO_REBOOT_ON_NETWORK_FAILURE') == false)
|
|
{
|
|
phodevi::reboot();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if(substr($server_response, 0, 1) == '[')
|
|
{
|
|
// Likely a notice/warning from server
|
|
echo PHP_EOL . substr($server_response, 0, strpos($server_response, PHP_EOL)) . PHP_EOL;
|
|
}
|
|
else if(substr($server_response, 0, 1) == '{')
|
|
{
|
|
$times_failed = 0;
|
|
$json = json_decode($server_response, true);
|
|
|
|
if($has_success == false)
|
|
{
|
|
$has_success = true;
|
|
pts_module::save_file('last-phoromatic-server', self::$server_address . ':' . self::$server_http_port . '/' . self::$account_id);
|
|
}
|
|
|
|
if($json != null)
|
|
{
|
|
if(isset($json['phoromatic']['error']) && !empty($json['phoromatic']['error']))
|
|
{
|
|
trigger_error($json['phoromatic']['error'], E_USER_ERROR);
|
|
}
|
|
if(isset($json['phoromatic']['response']) && !empty($json['phoromatic']['response']))
|
|
{
|
|
echo PHP_EOL . $json['phoromatic']['response'] . PHP_EOL;
|
|
}
|
|
if(isset($json['phoromatic']['system_id']) && !empty($json['phoromatic']['system_id']))
|
|
{
|
|
if(self::$system_id != $json['phoromatic']['system_id'])
|
|
{
|
|
// The Phoromatic server/account's system ID for this given system
|
|
self::$system_id = $json['phoromatic']['system_id'];
|
|
|
|
// Save the system ID to text file if it's useful for other purposes...
|
|
pts_module::set_option('system_id', self::$system_id);
|
|
}
|
|
}
|
|
if(isset($json['phoromatic']['server_core_version']) && !empty($json['phoromatic']['server_core_version']))
|
|
{
|
|
if(self::$server_core_version != $json['phoromatic']['server_core_version'])
|
|
{
|
|
// The PTS core version of the Phoromatic Server
|
|
self::$server_core_version = $json['phoromatic']['server_core_version'];
|
|
}
|
|
}
|
|
}
|
|
|
|
self::$limit_network_communication = isset($json['phoromatic']['settings']['LimitNetworkCommunication']) && pts_strings::string_bool($json['phoromatic']['settings']['LimitNetworkCommunication']);
|
|
if(self::$limit_network_communication)
|
|
{
|
|
// Sleep to ensure network communication is somewhat random in case all systems started at same time
|
|
sleep(rand(0, 20));
|
|
}
|
|
|
|
if($just_started)
|
|
{
|
|
if(PTS_IS_DAEMONIZED_SERVER_PROCESS && !self::$limit_network_communication && function_exists('pcntl_fork'))
|
|
{
|
|
$pid = pcntl_fork();
|
|
if($pid == 0)
|
|
{
|
|
// Start the tick thread
|
|
self::tick_thread();
|
|
}
|
|
}
|
|
|
|
self::upload_test_install_manifest();
|
|
$just_started = false;
|
|
}
|
|
|
|
pts_tests::clear_extra_env_vars();
|
|
if(isset($json['phoromatic']['pre_set_sys_env_vars']) && !empty($json['phoromatic']['pre_set_sys_env_vars']))
|
|
{
|
|
// pre_set_sys_env_vars was added during PTS 5.8 development
|
|
// Sets environment variables on client as specified via the Phoromatic Server's systems page
|
|
foreach(explode(';', $json['phoromatic']['pre_set_sys_env_vars']) as $i => $v_string)
|
|
{
|
|
$var = explode('=', $v_string);
|
|
if(count($var) == 2)
|
|
{
|
|
putenv($var[0] . '=' . $var[1]);
|
|
pts_tests::add_extra_env_var($var[0], $var[1]);
|
|
}
|
|
}
|
|
}
|
|
|
|
$task = isset($json['phoromatic']['task']) ? $json['phoromatic']['task'] : null;
|
|
|
|
if($task != 'idle')
|
|
{
|
|
pts_client::$pts_logger->log("Received " . $task . " command");
|
|
}
|
|
|
|
switch($task)
|
|
{
|
|
case 'install':
|
|
phoromatic::update_system_status('Installing Tests');
|
|
$phoromatic_suite = new pts_test_suite($json['phoromatic']['test_suite']);
|
|
pts_test_installer::standard_install($phoromatic_suite, false, true);
|
|
break;
|
|
case 'benchmark':
|
|
|
|
// Make sure all latest tests are available
|
|
pts_openbenchmarking::refresh_repository_lists(null, true);
|
|
|
|
$benchmark_timer = time();
|
|
self::$is_running_as_phoromatic_node = true;
|
|
$phoromatic_suite = new pts_test_suite($json['phoromatic']['test_suite']);
|
|
$phoromatic_results_identifier = $json['phoromatic']['trigger_id'];
|
|
self::$p_save_identifier = $json['phoromatic']['save_identifier'];
|
|
self::$p_schedule_id = isset($json['phoromatic']['schedule_id']) ? $json['phoromatic']['schedule_id'] : false;
|
|
self::$p_trigger_id = $json['phoromatic']['trigger_id'];
|
|
self::$benchmark_ticket_id = isset($json['phoromatic']['benchmark_ticket_id']) ? $json['phoromatic']['benchmark_ticket_id'] : null;
|
|
phoromatic::update_system_status('Running Benchmarks For: ' . self::$p_save_identifier);
|
|
// PTS 10.8 result viewer should deal fine with binary logs so can just upload by default there...
|
|
pts_client::$skip_log_file_type_checks = self::$server_core_version >= 10800 || (isset($json['phoromatic']['settings']['AllowAnyDataForLogFiles']) && pts_strings::string_bool($json['phoromatic']['settings']['AllowAnyDataForLogFiles']));
|
|
self::$progressive_result_uploads = isset($json['phoromatic']['settings']['ProgressiveResultUploads']) && pts_strings::string_bool($json['phoromatic']['settings']['ProgressiveResultUploads']);
|
|
pts_module::set_option('skip_log_file_type_checks', (pts_client::$skip_log_file_type_checks ? 1 : 0));
|
|
|
|
if(pts_strings::string_bool($json['phoromatic']['settings']['RunInstallCommand']))
|
|
{
|
|
if(isset($json['phoromatic']['pre_install_set_context']))
|
|
{
|
|
phoromatic::set_user_context($json['phoromatic']['pre_install_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'PRE_INSTALL');
|
|
}
|
|
|
|
pts_test_installer::standard_install($phoromatic_suite, pts_strings::string_bool($json['phoromatic']['settings']['ForceInstallTests']), true);
|
|
if(isset($json['phoromatic']['post_install_set_context']))
|
|
{
|
|
phoromatic::set_user_context($json['phoromatic']['post_install_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'POST_INSTALL');
|
|
}
|
|
}
|
|
$env_vars = isset($json['phoromatic']['environment_variables']) ? pts_strings::parse_value_string_vars($json['phoromatic']['environment_variables']) : array();
|
|
$is_stress_run = isset($env_vars['PTS_CONCURRENT_TEST_RUNS']) && $env_vars['PTS_CONCURRENT_TEST_RUNS'] > 1;
|
|
|
|
// Do the actual running
|
|
phodevi::clear_cache();
|
|
$original_env_var_overrides = pts_env::get_overrides();
|
|
$original_pts_modules = pts_module_manager::attached_modules();
|
|
|
|
if(isset($json['phoromatic']['settings']['GlobalEnvironmentVariables']) && !empty($json['phoromatic']['settings']['GlobalEnvironmentVariables']))
|
|
{
|
|
// The global environment variables set on the Phoromatic Settings page, rather than individual schedule/benchmark ticket specific env vars
|
|
pts_env::set_array(pts_strings::parse_value_string_vars($json['phoromatic']['settings']['GlobalEnvironmentVariables']));
|
|
}
|
|
|
|
if(!empty($env_vars))
|
|
{
|
|
pts_env::set_array($env_vars);
|
|
}
|
|
|
|
if($is_stress_run)
|
|
{
|
|
self::$test_run_manager = new pts_stress_run_manager(array(
|
|
'UploadResults' => false,
|
|
'SaveResults' => false,
|
|
'PromptForTestDescription' => false,
|
|
'RunAllTestCombinations' => false,
|
|
'PromptSaveName' => false,
|
|
'PromptForTestIdentifier' => false,
|
|
'OpenBrowser' => false
|
|
), true);
|
|
|
|
if(self::$test_run_manager->initial_checks($phoromatic_suite, 'SHORT'))
|
|
{
|
|
if(self::$test_run_manager->load_tests_to_run($phoromatic_suite))
|
|
{
|
|
self::$test_run_manager->action_on_stress_log_set(array('phoromatic', 'upload_stress_log_sane'));
|
|
self::$in_stress_mode = self::$p_save_identifier;
|
|
self::$test_run_manager->multi_test_stress_run_execute($env_vars['PTS_CONCURRENT_TEST_RUNS'], $env_vars['TOTAL_LOOP_TIME']);
|
|
self::$in_stress_mode = false;
|
|
self::upload_stress_log(self::$test_run_manager->get_stress_log());
|
|
}
|
|
}
|
|
self::$benchmark_ticket_id = null;
|
|
break;
|
|
}
|
|
else
|
|
{
|
|
self::$test_run_manager = new pts_test_run_manager(array(
|
|
'UploadResults' => (isset($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking'])),
|
|
'SaveResults' => true,
|
|
'RunAllTestCombinations' => false,
|
|
'OpenBrowser' => false
|
|
), true);
|
|
}
|
|
|
|
if(self::$test_run_manager->initial_checks($phoromatic_suite, 'SHORT'))
|
|
{
|
|
// Load the tests to run
|
|
if(self::$test_run_manager->load_tests_to_run($phoromatic_suite))
|
|
{
|
|
phoromatic::update_system_status('Tests In Run Queue: ' . implode(', ', self::$test_run_manager->get_tests_to_run_identifiers()));
|
|
if(isset($json['phoromatic']['pre_run_set_context']))
|
|
{
|
|
phoromatic::set_user_context($json['phoromatic']['pre_run_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'PRE_RUN');
|
|
}
|
|
|
|
if(isset($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']))
|
|
{
|
|
self::$test_run_manager->auto_upload_to_openbenchmarking();
|
|
pts_openbenchmarking_client::override_client_setting('UploadSystemLogsByDefault', pts_strings::string_bool($json['phoromatic']['settings']['UploadSystemLogs']));
|
|
}
|
|
|
|
// Save results?
|
|
|
|
// Run the actual tests
|
|
self::$test_run_manager->auto_save_results(self::$p_save_identifier, $phoromatic_results_identifier, (isset($json['phoromatic']['test_description']) ? $json['phoromatic']['test_description'] : 'A Phoromatic run.'));
|
|
self::$test_run_manager->pre_execution_process();
|
|
self::$test_run_manager->call_test_runs();
|
|
|
|
phoromatic::update_system_status('Benchmarks Completed For: ' . self::$p_save_identifier);
|
|
self::$test_run_manager->post_execution_process();
|
|
$elapsed_benchmark_time = time() - $benchmark_timer;
|
|
|
|
// Handle uploading data to server
|
|
$result_file = new pts_result_file(self::$test_run_manager->get_file_name());
|
|
$upload_system_logs = pts_strings::string_bool($json['phoromatic']['settings']['UploadSystemLogs']);
|
|
pts_module::set_option('upload_system_logs', ($upload_system_logs ? 1 : 0));
|
|
$upload_install_logs = isset($json['phoromatic']['settings']['UploadInstallLogs']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadInstallLogs']);
|
|
pts_module::set_option('upload_install_logs', ($upload_install_logs ? 1 : 0));
|
|
$upload_run_logs = isset($json['phoromatic']['settings']['UploadRunLogs']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadRunLogs']);
|
|
pts_module::set_option('upload_run_logs', ($upload_run_logs ? 1 : 0));
|
|
$server_response = self::upload_test_result($result_file, $upload_system_logs, self::$p_schedule_id, self::$p_save_identifier, self::$p_trigger_id, $elapsed_benchmark_time, self::$benchmark_ticket_id);
|
|
//pts_client::$pts_logger->log('DEBUG RESPONSE MESSAGE: ' . $server_response);
|
|
if(!pts_strings::string_bool($json['phoromatic']['settings']['ArchiveResultsLocally']))
|
|
{
|
|
pts_results::remove_saved_result_file(self::$test_run_manager->get_file_name());
|
|
}
|
|
}
|
|
|
|
if(isset($json['phoromatic']['post_run_set_context']))
|
|
{
|
|
phoromatic::set_user_context($json['phoromatic']['post_run_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'POST_RUN');
|
|
}
|
|
}
|
|
self::$p_schedule_id = null;
|
|
self::$is_running_as_phoromatic_node = false;
|
|
self::$benchmark_ticket_id = null;
|
|
|
|
// Restore any environment variables that may have been set within process / overridden
|
|
if(!empty($original_env_var_overrides))
|
|
{
|
|
pts_env::set_array($original_env_var_overrides, true);
|
|
$original_env_var_overrides = null;
|
|
}
|
|
// Unload any modules that were loaded just during this benchmarking run (i.e. by a passed environment variable from Phoromatic)
|
|
pts_module_manager::detach_extra_modules($original_pts_modules);
|
|
break;
|
|
case 'reboot':
|
|
echo PHP_EOL . 'Phoromatic received a remote command to reboot.' . PHP_EOL;
|
|
phoromatic::update_system_status('Attempting System Reboot');
|
|
self::tick_thread(true);
|
|
phodevi::reboot();
|
|
break;
|
|
case 'shutdown-if-supports-wake':
|
|
$supports_wol = false;
|
|
foreach(pts_network::get_network_wol() as $net_device)
|
|
{
|
|
if(strpos($net_device, 'g') !== false)
|
|
{
|
|
$supports_wol = true;
|
|
break;
|
|
}
|
|
}
|
|
if(!$supports_wol)
|
|
break;
|
|
case 'shutdown':
|
|
if(isset($json['phoromatic']['client_update_script']) && !empty($json['phoromatic']['client_update_script']))
|
|
{
|
|
self::run_client_update_script($json['phoromatic']['client_update_script']);
|
|
sleep(10);
|
|
}
|
|
|
|
echo PHP_EOL . 'Phoromatic received a remote command to shutdown.' . PHP_EOL;
|
|
phoromatic::update_system_status('Attempting System Shutdown');
|
|
self::tick_thread(true);
|
|
phodevi::shutdown();
|
|
break;
|
|
case 'maintenance':
|
|
echo PHP_EOL . 'Idling, system maintenance mode set by Phoromatic Server.' . PHP_EOL;
|
|
phoromatic::update_system_status('Maintenance Mode' . self::check_for_separate_pts_thread_process());
|
|
sleep(60);
|
|
break;
|
|
case 'idle':
|
|
if(isset($json['phoromatic']['client_update_script']) && !empty($json['phoromatic']['client_update_script']))
|
|
{
|
|
self::run_client_update_script($json['phoromatic']['client_update_script']);
|
|
}
|
|
//echo PHP_EOL . 'Idling, waiting for task.' . PHP_EOL;
|
|
phoromatic::update_system_status('Idling, Waiting For Task' . self::check_for_separate_pts_thread_process());
|
|
break;
|
|
case 'exit':
|
|
echo PHP_EOL . 'Phoromatic received a remote command to exit.' . PHP_EOL;
|
|
phoromatic::update_system_status('Exiting Phoromatic');
|
|
self::tick_thread(true);
|
|
$do_exit = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if(!$do_exit)
|
|
{
|
|
if($server_response == false)
|
|
sleep(rand(10, 30));
|
|
else if(self::$limit_network_communication)
|
|
sleep(rand(60, 240));
|
|
else
|
|
sleep(60);
|
|
}
|
|
}
|
|
|
|
pts_client::release_lock(PTS_USER_PATH . 'phoromatic_lock');
|
|
}
|
|
private static function check_for_separate_pts_thread_process()
|
|
{
|
|
$report = null;
|
|
$log_file = pts_logger::default_log_file_path() . 'phoronix-test-suite-benchmark.log';
|
|
if(is_file($log_file) && filemtime($log_file) > (time() - 1200))
|
|
{
|
|
$log_file = pts_file_io::file_get_contents($log_file);
|
|
$log_file = substr($log_file, strrpos($log_file, PHP_EOL) + 1);
|
|
if(($x = strpos($log_file, ']')) !== false)
|
|
{
|
|
$log_file = substr($log_file, ($x + 1));
|
|
}
|
|
$report .= '; Separate Process: ' . trim($log_file);
|
|
}
|
|
|
|
return $report;
|
|
}
|
|
public static function __post_test_run()
|
|
{
|
|
// Progressively upload result file at end of each test execution
|
|
if(self::$progressive_result_uploads)
|
|
{
|
|
self::upload_progressive_test_result();
|
|
}
|
|
}
|
|
private static function upload_progressive_test_result()
|
|
{
|
|
if(!self::$progressive_result_uploads || !(self::$test_run_manager->result_file instanceof pts_result_file))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
$composite_xml = self::$test_run_manager->result_file->get_xml();
|
|
return phoromatic::upload_to_remote_server(array(
|
|
'r' => 'result_upload',
|
|
'sched' => self::$p_schedule_id,
|
|
'bid' => self::$benchmark_ticket_id,
|
|
'o' => self::$p_save_identifier,
|
|
'ts' => self::$p_trigger_id,
|
|
'composite_xml' => base64_encode($composite_xml),
|
|
'composite_xml_hash' => sha1($composite_xml),
|
|
'progressive_upload' => 1
|
|
));
|
|
}
|
|
private static function upload_test_result(&$result_file, $upload_system_logs = true, $schedule_id = 0, $save_identifier = null, $trigger = null, $elapsed_time = 0, $benchmark_ticket_id = null)
|
|
{
|
|
$system_logs = null;
|
|
$system_logs_hash = null;
|
|
if(self::$server_core_version < 10602)
|
|
{
|
|
// On newer PTS servers, upload as separate upload afterwards to better handle large log files that otherwise may go beyond max request size, etc
|
|
|
|
// TODO: Potentially integrate this code below shared with pts_openbenchmarking_client into a unified function for validating system log files
|
|
$system_log_dir = $result_file->get_system_log_dir();
|
|
if(is_dir($system_log_dir) && ($upload_system_logs || pts_module::read_option('upload_system_logs', 0) != 0))
|
|
{
|
|
$is_valid_log = true;
|
|
if(pts_client::$skip_log_file_type_checks == false)
|
|
{
|
|
// For security/malicious purposes, ensure system log directory only contains text files
|
|
$is_valid_log = pts_file_io::directory_only_contains_text_files($system_log_dir);
|
|
}
|
|
|
|
if($is_valid_log)
|
|
{
|
|
$system_logs_zip = pts_client::create_temporary_file('.zip');
|
|
pts_compression::zip_archive_create($system_logs_zip, $system_log_dir);
|
|
|
|
if(filesize($system_logs_zip) == 0)
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('System log ZIP file failed to generate. Missing PHP ZIP support?');
|
|
}
|
|
else if(pts_client::$skip_log_file_type_checks == false && filesize($system_logs_zip) > 2097152)
|
|
{
|
|
// If it's over 2MB, probably too big
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('System log ZIP file too big to upload');
|
|
}
|
|
else
|
|
{
|
|
$system_logs = base64_encode(file_get_contents($system_logs_zip));
|
|
$system_logs_hash = sha1($system_logs);
|
|
}
|
|
unlink($system_logs_zip);
|
|
}
|
|
}
|
|
}
|
|
|
|
$composite_xml = $result_file->get_xml();
|
|
$composite_xml_hash = sha1($composite_xml);
|
|
$composite_xml_type = 'composite_xml';
|
|
|
|
// Compress the result file XML if it's big
|
|
if(isset($composite_xml[50000]) && function_exists('gzdeflate'))
|
|
{
|
|
$composite_xml_gz = gzdeflate($composite_xml);
|
|
if($composite_xml_gz != false)
|
|
{
|
|
$composite_xml = $composite_xml_gz;
|
|
$composite_xml_type = 'composite_xml_gz';
|
|
}
|
|
}
|
|
|
|
// Upload to Phoromatic
|
|
$times_tried = 0;
|
|
do
|
|
{
|
|
if($times_tried > 0)
|
|
{
|
|
sleep(rand(5, 20));
|
|
}
|
|
|
|
$res = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'result_upload',
|
|
//'ob' => $ob_data['id'],
|
|
'sched' => $schedule_id,
|
|
'bid' => $benchmark_ticket_id,
|
|
'o' => $save_identifier,
|
|
'ts' => $trigger,
|
|
'et' => $elapsed_time,
|
|
$composite_xml_type => base64_encode($composite_xml),
|
|
'composite_xml_hash' => $composite_xml_hash,
|
|
'system_logs_zip' => $system_logs,
|
|
'system_logs_hash' => $system_logs_hash,
|
|
'progressive_upload' => (self::$progressive_result_uploads ? 2 : 0)
|
|
));
|
|
|
|
$times_tried++;
|
|
}
|
|
while($res == false && $times_tried < 4);
|
|
|
|
$server_response = json_decode($res, true);
|
|
|
|
if(isset($server_response['phoromatic']['upload_id']) && !empty($server_response['phoromatic']['upload_id']))
|
|
{
|
|
// On newer PTS servers, upload as separate upload afterwards to better handle large log files that otherwise may go beyond max request size, etc
|
|
|
|
$log_types = array();
|
|
|
|
if($upload_system_logs)
|
|
{
|
|
$log_types['system-logs'] = $result_file->get_system_log_dir();
|
|
}
|
|
|
|
if(pts_module::read_option('upload_install_logs', 0) != 0)
|
|
{
|
|
$log_types['installation-logs'] = $result_file->get_test_installation_log_dir();
|
|
}
|
|
if(pts_module::read_option('upload_run_logs', 0) != 0)
|
|
{
|
|
$log_types['test-logs'] = $result_file->get_test_log_dir();
|
|
}
|
|
|
|
foreach($log_types as $index => $log_dir)
|
|
{
|
|
if(is_dir($log_dir))
|
|
{
|
|
$is_valid_log = true;
|
|
if(pts_client::$skip_log_file_type_checks == false)
|
|
{
|
|
// For security/malicious purposes, ensure system log directory only contains text files
|
|
$is_valid_log = pts_file_io::directory_only_contains_text_files($log_dir);
|
|
}
|
|
|
|
if($is_valid_log)
|
|
{
|
|
$system_logs_zip = pts_client::create_temporary_file('.zip');
|
|
pts_compression::zip_archive_create($system_logs_zip, $log_dir);
|
|
|
|
if(filesize($system_logs_zip) == 0)
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log($index . ' log ZIP file failed to generate. Missing PHP ZIP support?');
|
|
}
|
|
else
|
|
{
|
|
$system_logs = base64_encode(file_get_contents($system_logs_zip));
|
|
$system_logs_hash = sha1($system_logs);
|
|
}
|
|
unlink($system_logs_zip);
|
|
|
|
phoromatic::upload_to_remote_server(array(
|
|
'r' => 'result_log_upload',
|
|
'i' => $server_response['phoromatic']['upload_id'],
|
|
'system_logs_type' => $index,
|
|
'system_logs_zip' => $system_logs,
|
|
'system_logs_hash' => $system_logs_hash
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return $res;
|
|
}
|
|
private static function upload_stress_log($stress_log)
|
|
{
|
|
// Upload Logs to Phoromatic
|
|
if($stress_log == null || self::$benchmark_ticket_id == null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
$times_tried = 0;
|
|
do
|
|
{
|
|
if($times_tried > 0)
|
|
{
|
|
sleep(rand(5, 20));
|
|
}
|
|
|
|
$res = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'stress_log_upload',
|
|
'bid' => self::$benchmark_ticket_id,
|
|
'l' => pts_user_io::strip_ansi_escape_sequences($stress_log)
|
|
));
|
|
|
|
$times_tried++;
|
|
}
|
|
while($res == false && $times_tried < 4);
|
|
|
|
return $res;
|
|
}
|
|
public static function upload_stress_log_sane($stress_log)
|
|
{
|
|
static $last_log_upload = 0;
|
|
|
|
if(time() > ($last_log_upload + 60))
|
|
{
|
|
self::upload_stress_log($stress_log);
|
|
$last_log_upload = time();
|
|
}
|
|
}
|
|
public static function recent_phoromatic_server_results()
|
|
{
|
|
self::setup_server_addressing();
|
|
$server_response = phoromatic::upload_to_remote_server(array('r' => 'list_results'), null, null, null, true);
|
|
$server_response = json_decode($server_response, true);
|
|
|
|
if(isset($server_response['phoromatic']['results']) && !empty($server_response['phoromatic']['results']))
|
|
{
|
|
foreach($server_response['phoromatic']['results'] as $pprid => $result)
|
|
{
|
|
echo pts_client::cli_just_bold($result['Title']) . ' ' . $pprid . ' ' . date('j M H:i', strtotime($result['UploadTime'])) . PHP_EOL;
|
|
echo ' ' . $result['SystemName'] . ' ' . $result['GroupName'];
|
|
echo PHP_EOL . PHP_EOL;
|
|
}
|
|
}
|
|
else
|
|
echo PHP_EOL . 'No Phoromatic Server results discovered.';
|
|
|
|
echo PHP_EOL;
|
|
}
|
|
public static function clone_phoromatic_server_result($args)
|
|
{
|
|
self::setup_server_addressing();
|
|
|
|
$id = $args[0];
|
|
$server_response = phoromatic::upload_to_remote_server(array('r' => 'clone_result', 'i' => $id), null, null, null, true);
|
|
$server_response = json_decode($server_response, true);
|
|
|
|
if(isset($server_response['phoromatic']['result']['composite_xml']) && !empty($server_response['phoromatic']['result']['composite_xml']))
|
|
{
|
|
$composite_xml = base64_decode($server_response['phoromatic']['result']['composite_xml']);
|
|
$result_file = new pts_result_file($composite_xml);
|
|
// TODO XXX: Add system log downloading support
|
|
pts_client::save_test_result($id . '/composite.xml', $result_file->get_xml(), true);
|
|
echo PHP_EOL . 'Result File Saved As: ' . $id . PHP_EOL . PHP_EOL;
|
|
}
|
|
else
|
|
echo PHP_EOL . 'No Phoromatic result found.' . PHP_EOL;
|
|
}
|
|
private static function run_client_update_script($update_script)
|
|
{
|
|
static $last_update_script_check_time = 0;
|
|
|
|
// Don't keep checking it so check no more than every 30 minutes
|
|
if($last_update_script_check_time < (time() - (60 * 30)) && !empty($update_script))
|
|
{
|
|
$last_update_script_check_time = time();
|
|
$update_file = pts_client::create_temporary_file();
|
|
$update_script = str_replace("\r", PHP_EOL, $update_script);
|
|
file_put_contents($update_file, $update_script);
|
|
phoromatic::update_system_status('Running Phoromatic Update Script');
|
|
$env_vars = array();
|
|
|
|
$append_cmd = '';
|
|
if(phodevi::is_windows() && is_executable('C:\cygwin64\bin\bash.exe') && strpos($update_script, '#!/bin') !== false)
|
|
{
|
|
$cmd = 'C:\cygwin64\bin\bash.exe';
|
|
}
|
|
else if(phodevi::is_windows() && is_executable('C:\Windows\System32\cmd.exe'))
|
|
{
|
|
$cmd = 'C:\Windows\System32\cmd.exe /c';
|
|
}
|
|
else if(pts_client::executable_in_path('bash'))
|
|
{
|
|
$cmd = 'bash';
|
|
$append_cmd = ' 2>&1';
|
|
}
|
|
else
|
|
{
|
|
$cmd = 'sh';
|
|
$append_cmd = ' 2>&1';
|
|
}
|
|
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Running Update Script: ' . $cmd . ' ' . $update_file . $append_cmd);
|
|
$update_output = pts_client::shell_exec($cmd . ' ' . $update_file . $append_cmd, $env_vars);
|
|
if(trim($update_output) != '')
|
|
{
|
|
pts_client::$pts_logger && pts_client::$pts_logger->log('Update Script Output: ' . trim($update_output));
|
|
}
|
|
unlink($update_file);
|
|
phoromatic::update_system_status('Phoromatic Update Script Completed');
|
|
self::tick_thread(true);
|
|
}
|
|
}
|
|
private static function set_user_context($context_script, $trigger, $schedule_id, $process)
|
|
{
|
|
if(!empty($context_script))
|
|
{
|
|
$context_file = pts_client::create_temporary_file();
|
|
file_put_contents($context_file, $context_script);
|
|
chmod($context_file, 0755);
|
|
|
|
pts_file_io::mkdir(pts_module::save_dir());
|
|
$storage_path = pts_module::save_dir() . 'memory.pt2so';
|
|
$storage_object = pts_storage_object::recover_from_file($storage_path);
|
|
$notes_log_file = pts_module::save_dir() . sha1($trigger . $schedule_id . $process);
|
|
|
|
// We check to see if the context was already set but the system rebooted or something in that script
|
|
if($storage_object == false)
|
|
{
|
|
$storage_object = new pts_storage_object(true, true);
|
|
}
|
|
else if($storage_object->read_object('last_set_context_trigger') == $trigger && $storage_object->read_object('last_set_context_schedule') == $schedule_id && $storage_object->read_object('last_set_context_process') == $process)
|
|
{
|
|
// If the script already ran once for this trigger, don't run it again
|
|
self::check_user_context_log($trigger, $schedule_id, $process, $notes_log_file, null);
|
|
return false;
|
|
}
|
|
|
|
$storage_object->add_object('last_set_context_trigger', $trigger);
|
|
$storage_object->add_object('last_set_context_schedule', $schedule_id);
|
|
$storage_object->add_object('last_set_context_process', $process);
|
|
$storage_object->save_to_file($storage_path);
|
|
|
|
phoromatic::update_system_status('Setting context for: ' . $schedule_id . ' - ' . $trigger . ' - ' . $process);
|
|
|
|
// Run the set context script
|
|
$env_vars['PHOROMATIC_TRIGGER'] = $trigger;
|
|
$env_vars['PHOROMATIC_SCHEDULE_ID'] = $schedule_id;
|
|
$env_vars['PHOROMATIC_SCHEDULE_PROCESS'] = $process;
|
|
$env_vars['PHOROMATIC_LOG_FILE'] = $notes_log_file;
|
|
$log_output = pts_client::shell_exec($context_file . ' ' . $trigger . ' 2>&1', $env_vars);
|
|
self::check_user_context_log($trigger, $schedule_id, $process, $notes_log_file, $log_output);
|
|
|
|
// Just simply return true for now, perhaps check exit code status and do something
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
private static function check_user_context_log($trigger, $schedule_id, $process, $log_file, $log_output)
|
|
{
|
|
if(is_file($log_file) && ($lf_conts = pts_file_io::file_get_contents($log_file)) != null)
|
|
{
|
|
$context_log = $lf_conts;
|
|
unlink($log_file);
|
|
}
|
|
else
|
|
{
|
|
$context_log = trim($log_output);
|
|
}
|
|
|
|
if($context_log != null)
|
|
{
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'user_context_log',
|
|
'sched' => $schedule_id,
|
|
'ts' => $trigger,
|
|
'i' => $process,
|
|
'o' => $context_log,
|
|
));
|
|
}
|
|
}
|
|
protected static function upload_test_install_manifest()
|
|
{
|
|
$json_to_upload = array();
|
|
foreach(pts_tests::tests_installations_with_metadata() as $test_profile)
|
|
{
|
|
if($test_profile->test_installation)
|
|
{
|
|
$json_to_upload[$test_profile->get_identifier()] = $test_profile->test_installation->get_array();
|
|
}
|
|
}
|
|
|
|
if(!empty($json_to_upload))
|
|
{
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'test_install_manifest',
|
|
'manifest' => json_encode($json_to_upload)
|
|
));
|
|
}
|
|
}
|
|
public static function __post_install_process()
|
|
{
|
|
self::upload_test_install_manifest();
|
|
}
|
|
public static function __post_run_process()
|
|
{
|
|
self::upload_test_install_manifest();
|
|
}
|
|
public static function __pre_test_install(&$test_install_request)
|
|
{
|
|
/* if(self::$has_run_server_setup_func == false)
|
|
{
|
|
self::setup_server_addressing();
|
|
}
|
|
*/
|
|
// XXX finish wiring in the above code to various parts for making auto-reporting from clients
|
|
|
|
if(!self::$is_running_as_phoromatic_node)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static $last_update_time = 0;
|
|
|
|
if(time() > ($last_update_time + 30))
|
|
{
|
|
phoromatic::update_system_status('Installing: ' . $test_install_request->test_profile->get_identifier());
|
|
$last_update_time = time();
|
|
}
|
|
}
|
|
public static function __pre_test_run($pts_test_result)
|
|
{
|
|
if(!self::$is_running_as_phoromatic_node)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if(self::$in_stress_mode)
|
|
{
|
|
static $time_in_stress_run = 0;
|
|
|
|
$msg = 'Stress-Run Testing';
|
|
if(($time_in_stress_run + (60 * 60)) > time())
|
|
{
|
|
// Don't report this same string so often...
|
|
return;
|
|
}
|
|
$time_in_stress_run = time();
|
|
}
|
|
else
|
|
{
|
|
$msg = 'Running: ' . $pts_test_result->test_profile->get_identifier() . ($pts_test_result->get_arguments_description() != null ? ' [' . $pts_test_result->get_arguments_description() . ']' : null);
|
|
}
|
|
|
|
phoromatic::update_system_status($msg,
|
|
ceil(self::$test_run_manager->get_estimated_run_time() / 60),
|
|
self::$test_run_manager->get_percent_complete(),
|
|
null,
|
|
ceil($pts_test_result->get_estimated_run_time() / 60));
|
|
}
|
|
public static function __event_results_saved($test_run_manager)
|
|
{
|
|
/*if(pts_env::read('AUTO_UPLOAD_RESULTS_TO_PHOROMATIC') && pts_module::is_module_setup())
|
|
{
|
|
phoromatic::upload_unscheduled_test_results($test_run_manager->get_file_name());
|
|
}*/
|
|
}
|
|
public static function __event_pre_run_error($error_obj)
|
|
{
|
|
list($test_profile, $error_msg) = $error_obj;
|
|
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'error_report',
|
|
'sched' => self::$p_schedule_id,
|
|
'ts' => self::$p_trigger_id,
|
|
'err' => $error_msg,
|
|
'ti' => $test_profile->get_identifier(),
|
|
'o' => ''
|
|
));
|
|
}
|
|
public static function __event_run_error($error_obj)
|
|
{
|
|
list($test_run_manager, $test_run_request, $error_msg) = $error_obj;
|
|
|
|
if(stripos('Download Failed', $error_msg) !== false || stripos('check-sum of the downloaded file failed', $error_msg) !== false || stripos('attempting', $error_msg) !== false)
|
|
return false;
|
|
|
|
$server_response = phoromatic::upload_to_remote_server(array(
|
|
'r' => 'error_report',
|
|
'sched' => self::$p_schedule_id,
|
|
'ts' => self::$p_trigger_id,
|
|
'err' => $error_msg,
|
|
'ti' => $test_run_request->test_profile->get_identifier(),
|
|
'o' => $test_run_request->get_arguments_description()
|
|
));
|
|
//pts_client::$pts_logger && pts_client::$pts_logger->log('TEMP DEBUG MESSAGE: ' . $server_response);
|
|
}
|
|
public static function __event_reboot($passed_obj)
|
|
{
|
|
$reboot_msg = 'Attemtping system reboot';
|
|
if($passed_obj != null)
|
|
{
|
|
if($passed_obj instanceof pts_test_result)
|
|
{
|
|
$reboot_msg = 'Attempting system reboot, requested by ' . $passed_obj->test_profile->get_identifier();
|
|
}
|
|
else if($passed_obj instanceof pts_test_profile)
|
|
{
|
|
$reboot_msg = 'Attempting system reboot, requested by ' . $passed_obj->get_identifier();
|
|
}
|
|
}
|
|
phoromatic::update_system_status($reboot_msg);
|
|
}
|
|
}
|
|
|
|
?>
|