pinglerr/pinglerr_web/ping_stats_compressor.php

<?php
/**
 * Since we are gathering ping statistics at a high resolution interval this builds up really
 * quickly for longer running ping jobs causing slowdowns and using lots of storage.
 * 
 * This script is meant to be ran periodically; it merges the high resolution dataset into a lower
 * resolution dataset without losing too much important information.
 */

$config = require 'config.php';

function get_mysqli(): MySQLi
{
    static $handle;
    if (!isset($handle)) {
        $handle = new MySQLi(
            hostname: $config['db_host'],
            username: $config['db_name'], 
            password: $config['db_pass'], 
            database: $config['db_database'],
        );
        // make sure that the timezone is set to UTC!!!
        $handle->query("SET time_zone = '+00:00'");
    }
    return $handle;
}

$scans = (function(): array
{
    $stmt = get_mysqli()->prepare(
        "SELECT * FROM pinglerr_scans"
    );
    $stmt->execute();
    $result = $stmt->get_result();
    $assoc_array = [];
    while ($row = $result->fetch_assoc()) {
        $assoc_array[] = $row;
    }
    return $assoc_array;
})();

foreach ($scans as $scan) {
    $pings = (function() use($scan): array
    {
        $stmt = get_mysqli()->prepare(
            "SELECT ping_avg, pkt_transmitted, pkt_received, date_added FROM pinglerr_pings WHERE scan_id = ? ORDER BY date_added ASC"
        );
        $stmt->bind_param("i", $scan['id']);
        $stmt->execute();
        $result = $stmt->get_result();
        $assoc_array = [];
        while ($row = $result->fetch_assoc()) {
            $assoc_array[] = $row;
        }
        return $assoc_array;
    })();
    if (empty($pings)) {
        continue;
    }

    echo 'Processing scan_id: ' . $scan['id'] . ' rows: ' . count($pings) . "\n";

    $limit_length = 1000;
    if (count($pings) > $limit_length) {
        $merged_pings = [];

        $merge_gap = count($pings) / $limit_length;
        $merge_iteration = 0;
        $merge_remainder = 0;

        $ping_avg_sum_storage = [];
        $ping_pkt_transmitted_storage = [];
        $ping_pkt_received_storage = [];
        foreach ($pings as $ping) {
            if ($merge_iteration >= $merge_gap - $merge_remainder) {
                $merge_remainder+= $merge_iteration - $merge_gap;

                $how_many_nulls = 0;
                for ($i=0; $i < count($ping_avg_sum_storage); $i++) { 
                    if ($ping_avg_sum_storage[$i] === null) {
                        $how_many_nulls++;
                    }
                }
                if ($how_many_nulls > count($ping_avg_sum_storage) / 2) {
                    $ping_avg = null;
                } else {
                    for ($i=0; $i < count($ping_avg_sum_storage); $i++) { 
                        if ($ping_avg_sum_storage[$i] === null) {
                            unset($ping_avg_sum_storage[$i]);
                        }
                    }
                    $ping_avg = round(array_sum($ping_avg_sum_storage) / count($ping_avg_sum_storage), 3);
                }

                $merged_pings[] = [
                    'ping_avg'   => $ping_avg, 
                    'date_added' => $ping['date_added'],
                    'pkt_transmitted' => round(array_sum($ping_pkt_transmitted_storage) / count($ping_pkt_transmitted_storage), 3),
                    'pkt_received' => round(array_sum($ping_pkt_received_storage) / count($ping_pkt_received_storage), 3),
                ];
                $ping_pkt_transmitted_storage = [];
                $ping_pkt_received_storage = [];
                $ping_avg_sum_storage = [];
                $merge_iteration = 0;
            }
            $ping_pkt_transmitted_storage[] = $ping['pkt_transmitted'];
            $ping_pkt_received_storage[] = $ping['pkt_received'];
            $ping_avg_sum_storage[] = $ping['ping_avg'];
            $merge_iteration++;
        }

        $pings = $merged_pings;
    } else {
        echo "Skipped\n";
        continue;
    }

    (function() use($scan) {
        $stmt = get_mysqli()->prepare(
            "DELETE FROM pinglerr_pings WHERE scan_id = ?"
        );
        $status_code = 'DONE';
        $error_message = 'Stopped because end_date was reached';
        $stmt->bind_param("s",
            $scan['id']
        );
        $stmt->execute();
    })();

    foreach ($pings as $ping) {
        $stmt = get_mysqli()->prepare(
            "INSERT INTO pinglerr_pings (scan_id, ping_avg, pkt_transmitted, pkt_received, date_added) VALUES (?,?,?,?,?)"
        );
        $stmt->bind_param("idiis",
            $scan['id'],
            $ping['ping_avg'],
            $ping['pkt_transmitted'],
            $ping['pkt_received'],
            $ping['date_added']
        );
        $stmt->execute();
    }

    echo 'Done: ' . count($pings) . "\n";
}

exit();