From 141deaa35b6a59e1a94c731de02b70d77ad2c8e5 Mon Sep 17 00:00:00 2001 From: panariga Date: Tue, 17 Dec 2024 15:50:30 +0200 Subject: [PATCH] modified: xbotcontrol.php --- .gitignore | 5 +- composer.json | 3 +- composer.lock | 81 ++++- ...fa076808377b28b01c478_2.file_index.tpl.php | 326 ++++++++++++++++++ smarty/template/index.tpl | 273 +++++++++++++++ src/Classes/LoadStat.php | 25 ++ src/Classes/Report.php | 132 +++++++ src/Classes/Schedule.php | 62 ++++ src/Controllers/APIController.php | 28 ++ src/Controllers/IndexController.php | 25 ++ src/InitTables.php | 21 +- src/Instances/Whitelist.php | 33 ++ src/Request.php | 56 +-- src/Storage.php | 6 +- xbotcontrol.php | 26 +- 15 files changed, 1042 insertions(+), 60 deletions(-) create mode 100644 smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php create mode 100644 smarty/template/index.tpl create mode 100644 src/Classes/LoadStat.php create mode 100644 src/Classes/Report.php create mode 100644 src/Classes/Schedule.php create mode 100644 src/Controllers/APIController.php create mode 100644 src/Controllers/IndexController.php create mode 100644 src/Instances/Whitelist.php diff --git a/.gitignore b/.gitignore index 4c8ad88..b9c4685 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /vendor .env -requests.sqlite3 \ No newline at end of file +composer.lock +requests.sqlite3 +requests.sqlite3-shm +requests.sqlite3-wal \ No newline at end of file diff --git a/composer.json b/composer.json index fb5951b..4f64167 100644 --- a/composer.json +++ b/composer.json @@ -5,7 +5,8 @@ "vlucas/phpdotenv": "^5.6", "react/cache": "^1.2", "clue/mq-react": "^1.6", - "smarty/smarty": "^5.4" + "smarty/smarty": "^5.4", + "react/promise-timer": "^1.11" }, "autoload": { "psr-4": { diff --git a/composer.lock b/composer.lock index a0ad34e..40be67e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2ff629509c131622c46d8c67505d54b2", + "content-hash": "8e299b08324c21b9f02c215ffc70c444", "packages": [ { "name": "clue/framework-x", @@ -1160,6 +1160,85 @@ ], "time": "2024-05-24T10:39:05+00:00" }, + { + "name": "react/promise-timer", + "version": "v1.11.0", + "source": { + "type": "git", + "url": "https://github.com/reactphp/promise-timer.git", + "reference": "4f70306ed66b8b44768941ca7f142092600fafc1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/promise-timer/zipball/4f70306ed66b8b44768941ca7f142092600fafc1", + "reference": "4f70306ed66b8b44768941ca7f142092600fafc1", + "shasum": "" + }, + "require": { + "php": ">=5.3", + "react/event-loop": "^1.2", + "react/promise": "^3.2 || ^2.7.0 || ^1.2.1" + }, + "require-dev": { + "phpunit/phpunit": "^9.6 || ^5.7 || ^4.8.36" + }, + "type": "library", + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "React\\Promise\\Timer\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Christian Lück", + "email": "christian@clue.engineering", + "homepage": "https://clue.engineering/" + }, + { + "name": "Cees-Jan Kiewiet", + "email": "reactphp@ceesjankiewiet.nl", + "homepage": "https://wyrihaximus.net/" + }, + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com", + "homepage": "https://sorgalla.com/" + }, + { + "name": "Chris Boden", + "email": "cboden@gmail.com", + "homepage": "https://cboden.dev/" + } + ], + "description": "A trivial implementation of timeouts for Promises, built on top of ReactPHP.", + "homepage": "https://github.com/reactphp/promise-timer", + "keywords": [ + "async", + "event-loop", + "promise", + "reactphp", + "timeout", + "timer" + ], + "support": { + "issues": "https://github.com/reactphp/promise-timer/issues", + "source": "https://github.com/reactphp/promise-timer/tree/v1.11.0" + }, + "funding": [ + { + "url": "https://opencollective.com/reactphp", + "type": "open_collective" + } + ], + "time": "2024-06-04T14:27:45+00:00" + }, { "name": "react/socket", "version": "v1.16.0", diff --git a/smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php b/smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php new file mode 100644 index 0000000..bda19ff --- /dev/null +++ b/smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php @@ -0,0 +1,326 @@ +getCompiled()->isFresh($_smarty_tpl, array ( + 'version' => '5.4.2', + 'unifunc' => 'content_676160989e3635_07407148', + 'has_nocache_code' => false, + 'file_dependency' => + array ( + 'affb24851ed623b62affa076808377b28b01c478' => + array ( + 0 => 'index.tpl', + 1 => 1734434957, + 2 => 'file', + ), + ), + 'includes' => + array ( + ), +))) { +function content_676160989e3635_07407148 (\Smarty\Template $_smarty_tpl) { +$_smarty_current_dir = '/home/l/public_html/xbotcontrol/smarty/template'; +?> + + + + + + + + XBotControl + + + + src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js" + integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"> + +> + + + src="https://code.jquery.com/jquery-3.7.1.min.js"> +> + + + src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.5/dist/bootstrap-table.min.js"> +> + + + src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.5/dist/locale/bootstrap-table-zh-CN.min.js"> +> + + + + src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.5/dist/extensions/filter-control/bootstrap-table-filter-control.js"> + +> + + + + + + + +
+ +
+ +
+ + + + +
+
+
+
+
+
+ +
+ +
+
+ +
+
+
Limit
+ + + +
+
+
+
+
From
+ + +
+
+
+
+
To
+ + + +
+ +
+
+ + +
+ + +
+
+
+ +> + document.getElementById('date-from').addEventListener('change', refreshTable); + + document.getElementById('limit').addEventListener('change', refreshTable); + + function refreshTable() { + $('#table').bootstrapTable('refresh'); + } + window.onload = function() { + const dateFrom = document.getElementById('date-from'); + const dateTo = document.getElementById('date-to'); + + const today = new Date(); + const yesterday = new Date(today); + yesterday.setDate(today.getDate() - 1); + + const tomorrow = new Date(today); + tomorrow.setDate(today.getDate() + 1); + + dateFrom.value = yesterday.toISOString().slice(0, 16); + dateTo.value = tomorrow.toISOString().slice(0, 16); + }; + document.getElementById('date-to').addEventListener('change', refreshTable); + + + function initializeTable(latest_requests) { + var url = location.pathname + '/api/report/' + latest_requests; + var $table = $('#table'); + + if ($table.length) { + $table.bootstrapTable('destroy'); + } + + $.get(url, function(response) { + $table.bootstrapTable({ + url: url, + sortable: true, + toolbar: '#toolbar', + showRefresh: true, + iconsPrefix: 'fa', + showColumns: true, + classes: ['table', 'table-borderless', 'table-hover', 'table-striped'], + filterControl: true, + searchable: true, + pagination: false, + sidePagination: "server", + serverSort: false, + columns: response.columns, + queryParams: queryParams, + loadingFontSize: '12px' + }); + }); + } + + function queryParams(params) { + const limit = document.getElementById('limit').value; + const from = document.getElementById('date-from').value; + const to = document.getElementById('date-to').value; + + params.limit = limit; + params.from = from; + params.to = to; + + return params; + } + + + + + function buttons() { + + return { + btnAdd: { + text: 'Add new list', + icon: 'fa-plus', + event: function() { + // Prompt the user for a new list name + const newListName = prompt('Enter new list name:'); + + // Only proceed if the user provides a valid list name + if (newListName) { + // Define the URL where the form needs to be posted + const url = ' +/lists/create'; // Replace with actual URL + + // Create a new hidden form element + const form = document.createElement('form'); + form.method = 'POST'; + form.action = url; + + // Create hidden input to store the list name + const input = document.createElement('input'); + input.type = 'hidden'; + input.name = 'listName'; // The name expected by the server + input.value = newListName; + + // Append the input to the form + form.appendChild(input); + + // Append the form to the body to make it part of the DOM + document.body.appendChild(form); + + // Submit the form automatically + form.submit(); + } else { + // Handle case where user cancels or enters an empty name + alert('List creation was cancelled or name was empty.'); + } + }, + attributes: { + title: 'Add a new list to the table' + } + } + } + + return { + + } + + + } + + function listFormatter(value, row, index) { + var editBtn = ' '; + + var showBtn = ''; + + return [showBtn, editBtn, value, ].join('') + + return [showBtn, value, ].join('') + + + } + +> + + + + + + + + + + XBotControl + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ + + + +
+
+
+
+
+
+ +
+ +
+
+ +
+
+
Limit
+ + + +
+
+
+
+
From
+ + +
+
+
+
+
To
+ + + +
+ +
+
+ + +
+ + +
+
+
+ + + \ No newline at end of file diff --git a/src/Classes/LoadStat.php b/src/Classes/LoadStat.php new file mode 100644 index 0000000..e45da3e --- /dev/null +++ b/src/Classes/LoadStat.php @@ -0,0 +1,25 @@ +db->query($query, $params); + } +} diff --git a/src/Classes/Report.php b/src/Classes/Report.php new file mode 100644 index 0000000..8bb677a --- /dev/null +++ b/src/Classes/Report.php @@ -0,0 +1,132 @@ + 'id', + 'field' => 'id', + 'visible' => false, + 'sortable' => true, + 'filterControl' => 'input', + 'widthUnit' => 'input', + 'width' => 'input', + ], + [ + 'sortable' => true, + 'title' => 'ip', + 'field' => 'ip', + 'sortable' => true, + 'filterControl' => 'input', + + ], + [ + 'sortable' => true, + 'title' => 'domain', + 'field' => 'domain', + 'sortable' => true, + 'visible' => false, + 'filterControl' => 'input', + + ], + [ + 'sortable' => true, + 'title' => 'path', + 'field' => 'path', + 'sortable' => true, + 'filterControl' => 'input', + + ], + [ + 'sortable' => true, + 'title' => 'useragent', + 'field' => 'useragent', + 'sortable' => true, + 'filterControl' => 'input', + + ], + [ + 'sortable' => true, + 'title' => 'load', + 'field' => 'load', + 'sortable' => true, + 'filterControl' => 'input', + + ], + [ + 'sortable' => true, + 'title' => 'datetime', + 'field' => 'datetime', + 'sortable' => true, + 'filterControl' => 'input', + + ], + ]; + + + $sql = "SELECT + req.rowid AS id, + ip.data AS ip, + domain.data AS domain, + path.data AS path, + useragent.data AS useragent, + headers.data AS headers , + (SELECT load.load1 + FROM load + WHERE load.rowid >= req.timestamp + ORDER BY load.rowid DESC LIMIT 1) AS load, + datetime(req.timestamp, 'auto') AS datetime + + FROM + request req + LEFT JOIN + ip ON req.id_ip = ip.rowid + LEFT JOIN + domain ON req.id_domain = domain.rowid + LEFT JOIN + path ON req.id_path = path.rowid + LEFT JOIN + useragent ON req.id_useragent = useragent.rowid + LEFT JOIN + headers ON req.id_headers = headers.rowid + WHERE 1=1 "; + + + $params = []; + $query = $request->getQueryParams(); + if (isset($query['filter'])) { + $filter = json_decode($request->getQueryParams()['filter'], true); + } else { + $filter = []; + } + + foreach ($filter as $field => $value) { + $sql .= 'AND ' . $field . ' LIKE ? '; + $params[] = '%' . $value . '%'; + } + $sql .= " AND req.timestamp BETWEEN ? AND ? "; + $sql .= ' ORDER BY req.rowid DESC '; + $sql .= ' LIMIT ? ;'; + $params[] = strtotime($request->getQueryParams()['from'] ?? 'yesterday'); + $params[] = strtotime($request->getQueryParams()['to'] ?? 'now'); + $params[] = (int)$request->getQueryParams()['limit'] ?? 100; + + return \XBotControl\Storage::getInstance()->db->query($sql, $params)->then(function ($result) use ($columnsDefinition) { + return [ + "columns" => $columnsDefinition, + "rows" => $result->rows, + ]; + }); + } +} diff --git a/src/Classes/Schedule.php b/src/Classes/Schedule.php new file mode 100644 index 0000000..e33e65c --- /dev/null +++ b/src/Classes/Schedule.php @@ -0,0 +1,62 @@ + 60, 'task' => [\XBotControl\Classes\LoadStat::class, 'saveLoad1']], + + ]; + + /** + * Initializes and runs the task scheduler. + * + * Loops through the SCHEDULE array, setting up periodic timers + * using React's event loop. If multiple tasks share the same interval, + * they are distributed by adding evenly spaced offsets to avoid collisions. + * + * @return void + */ + public static function run(): void + { + $tasksByInterval = []; + + // Group tasks by interval + foreach (self::SCHEDULE as $schedule) { + $interval = $schedule['interval']; + $tasksByInterval[$interval][] = $schedule['task']; + } + + // Schedule tasks for each interval + foreach ($tasksByInterval as $interval => $tasks) { + $taskCount = count($tasks); + foreach ($tasks as $index => $task) { + // Distribute tasks evenly within the interval + $offset = ($index / $taskCount) * $interval; + + Loop::addTimer($offset, function () use ($interval, $task) { + Loop::addPeriodicTimer($interval, fn() => call_user_func($task)); + }); + } + } + } +} diff --git a/src/Controllers/APIController.php b/src/Controllers/APIController.php new file mode 100644 index 0000000..e858b7f --- /dev/null +++ b/src/Controllers/APIController.php @@ -0,0 +1,28 @@ +getAttribute('action')) { + case 'report': + return call_user_func([\XBotControl\Classes\Report::class, $request->getAttribute('resource')], $request)->then(function ($result) { + return \React\Http\Message\Response::json($result); + }); + default: + return \React\Http\Message\Response::json( + ['empty_response'] + ); + } + } +} diff --git a/src/Controllers/IndexController.php b/src/Controllers/IndexController.php new file mode 100644 index 0000000..f8bb02d --- /dev/null +++ b/src/Controllers/IndexController.php @@ -0,0 +1,25 @@ +smarty; + $smarty->assign([ + + ]); + + return \React\Http\Message\Response::html( + $smarty->fetch('index.tpl') + ); + } +} diff --git a/src/InitTables.php b/src/InitTables.php index d11165a..83448da 100644 --- a/src/InitTables.php +++ b/src/InitTables.php @@ -11,6 +11,7 @@ class InitTables public static function create():PromiseInterface { + $db = Storage::getInstance()->db; return $db->exec("CREATE TABLE IF NOT EXISTS ip (data TEXT UNIQUE NOT NULL CHECK (data LIKE '%'), CONSTRAINT valid_ip CHECK (data LIKE '%.%' OR data LIKE '%:%')) STRICT ;") ->then(function () use ($db) { @@ -28,26 +29,12 @@ class InitTables })->then(function () use ($db) { return $db->exec('CREATE TABLE IF NOT EXISTS bot ( name TEXT NOT NULL, keyword TEXT NULL ) STRICT ;'); })->then(function () use ($db) { - return $db->exec('PRAGMA journal_mode=WAL;'); - }); - - return $db->exec("CREATE TABLE IF NOT EXISTS ip (id_ip INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT UNIQUE NOT NULL CHECK (ip LIKE '%'), CONSTRAINT valid_ip CHECK (ip LIKE '%.%' OR ip LIKE '%:%')) STRICT ;") - ->then(function () use ($db) { - return $db->exec('CREATE TABLE IF NOT EXISTS domain (id_domain INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL) STRICT ;'); + return $db->exec('CREATE TABLE IF NOT EXISTS settings ( key TEXT UNIQUE NOT NULL, value TEXT NULL ) STRICT ;'); })->then(function () use ($db) { - return $db->exec('CREATE TABLE IF NOT EXISTS path ( id_path INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT UNIQUE NOT NULL) STRICT ;'); - })->then(function () use ($db) { - return $db->exec('CREATE TABLE IF NOT EXISTS useragent ( id_useragent INTEGER PRIMARY KEY AUTOINCREMENT, useragent TEXT UNIQUE NOT NULL) STRICT ;'); - })->then(function () use ($db) { - return $db->exec('CREATE TABLE IF NOT EXISTS headers ( id_headers INTEGER PRIMARY KEY AUTOINCREMENT, headers TEXT UNIQUE NOT NULL) STRICT ;'); - })->then(function () use ($db) { - return $db->exec("CREATE TABLE IF NOT EXISTS networkwhitelist ( id_networkwhitelist INTEGER PRIMARY KEY AUTOINCREMENT, network TEXT UNIQUE NOT NULL CHECK (network LIKE '%/%'), CONSTRAINT valid_network CHECK ( network LIKE '%.%/%' OR network LIKE '%:%/%' )) STRICT WITHOUT ROWID ;"); - })->then(function () use ($db) { - return $db->exec('CREATE TABLE IF NOT EXISTS request ( id_request INTEGER PRIMARY KEY AUTOINCREMENT, id_ip INTEGER NOT NULL, id_method INTEGER NOT NULL, id_domain INTEGER NOT NULL, id_path INTEGER NOT NULL, id_useragent INTEGER NOT NULL, id_headers INTEGER NOT NULL, timestamp INTEGER NOT NULL, FOREIGN KEY (id_ip) REFERENCES ip(id_ip), FOREIGN KEY (id_domain) REFERENCES domain(id_domain), FOREIGN KEY (id_path) REFERENCES path(id_path), FOREIGN KEY (id_useragent) REFERENCES useragent(id_useragent), FOREIGN KEY (id_headers) REFERENCES headers(id_headers) ) STRICT WITHOUT ROWID ;'); - })->then(function () use ($db) { - return $db->exec('CREATE TABLE IF NOT EXISTS bot ( id_bot INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, keyword TEXT NULL ) STRICT ;'); + return $db->exec('CREATE TABLE IF NOT EXISTS load (load1 REAL NOT NULL) STRICT ;'); })->then(function () use ($db) { return $db->exec('PRAGMA journal_mode=WAL;'); }); + } } diff --git a/src/Instances/Whitelist.php b/src/Instances/Whitelist.php new file mode 100644 index 0000000..bb7931d --- /dev/null +++ b/src/Instances/Whitelist.php @@ -0,0 +1,33 @@ +getHeaderLine('User-Agent') ?: 'Unknown'; - $headers = json_encode($request->getHeaders(), JSON_UNESCAPED_UNICODE); - $uri = $request->getUri(); - $storage = Storage::getInstance(); + $realIp = self::getRealIP($request); + $userAgent = $request->getHeaderLine('User-Agent') ?: 'Unknown'; + $headers = json_encode($request->getHeaders(), JSON_UNESCAPED_UNICODE); + $uri = $request->getUri(); + $storage = Storage::getInstance(); - // Use parallel promises for ID generation to avoid waiting for each in sequence - $idPromises = [ - 'id_ip' => $storage::getId('ip', $realIp), - 'id_domain' => $storage::getId('domain', $uri->getHost()), - 'id_path' => $storage::getId('path', $uri->getPath()), - 'id_useragent' => $storage::getId('useragent', $userAgent), - 'id_headers' => $storage::getId('headers', md5($headers)), - ]; + // Use parallel promises for ID generation to avoid waiting for each in sequence + $idPromises = [ + 'id_ip' => $storage::getId('ip', $realIp), + 'id_domain' => $storage::getId('domain', $uri->getHost()), + 'id_path' => $storage::getId('path', '/' . $request->getAttribute('original_uri', '')), + 'id_useragent' => $storage::getId('useragent', $userAgent), + 'id_headers' => 0, + ]; - return \React\Promise\all($idPromises) - ->then(function ($resolvedValues) use ($request, $storage) { - // Set resolved values efficiently - $resolvedValues['id_method'] = self::METHOD[$request->getMethod()] ?? 0; - $resolvedValues['timestamp'] = time(); + if ($_ENV['SAVE_HEADERS'] === true) { + $idPromises['id_headers'] = $storage::getId('headers', $headers); + } - // Directly save data asynchronously - return $storage::insert('request', $resolvedValues); - }) - ->then(function () { - return \React\Http\Message\Response::plaintext(''); - }); + + + return \React\Promise\all($idPromises) + ->then(function ($resolvedValues) use ($request, $storage) { + // Set resolved values efficiently + $resolvedValues['id_method'] = self::METHOD[$request->getMethod()] ?? 0; + $resolvedValues['timestamp'] = time(); + + // Directly save data asynchronously + return $storage::insert('request', $resolvedValues); + }) + ->then(function () { + return \React\Http\Message\Response::plaintext(''); + }); + } public static function getRealIP(ServerRequestInterface $request): string diff --git a/src/Storage.php b/src/Storage.php index bbbbdb6..371b5ba 100644 --- a/src/Storage.php +++ b/src/Storage.php @@ -10,8 +10,10 @@ use Clue\React\SQLite\Result; class Storage { - protected static $instance; + private static ?Storage $instance = null; + /** @var DatabaseInterface $db */ + public $db; public $cache = []; @@ -23,7 +25,7 @@ class Storage 'path' ]; - public function __construct() + private function __construct() { $this->db = (new \Clue\React\SQLite\Factory())->openLazy($_ENV['APP_DIR'] . '/requests.sqlite3'); diff --git a/xbotcontrol.php b/xbotcontrol.php index 3b3d75d..3021755 100644 --- a/xbotcontrol.php +++ b/xbotcontrol.php @@ -7,35 +7,33 @@ require __DIR__ . '/vendor/autoload.php'; $dotenv = Dotenv\Dotenv::createImmutable(__DIR__); $dotenv->load(); $_ENV['APP_DIR'] = __DIR__; -$_ENV['X_LISTEN'] = '0.0.0.0:7500'; + XBotControl\InitTables::create(); $app = new FrameworkX\App(); -$app->get( - '/mirror', +$app->any( + '/mirror[/{original_uri:.*}]', function (Psr\Http\Message\ServerRequestInterface $request) { return XBotControl\Request::save($request); } ); -$app->get('/mirror1', function () { - return React\Http\Message\Response::plaintext( - "Hello wörld!\n" - ); -}); +$app->any('/', XBotControl\Controllers\IndexController::class); + +$app->any('/api/{action}/{resource}', XBotControl\Controllers\APIController::class); + + +XBotControl\Classes\Schedule::run(); -$app->get('/cp', function (Psr\Http\Message\ServerRequestInterface $request) { - return React\Http\Message\Response::plaintext( - "Hello " . $request->getAttribute('name') . "!\n" - ); -}); $app->run(); -XBotControl\Storage::getInstance()->db->query('PRAGMA main.wal_checkpoint;') +XBotControl\Storage::getInstance()->db->query('VACUUM;') ->then(function () { + XBotControl\Storage::getInstance()->db->query('PRAGMA main.wal_checkpoint;'); + })->then(function () { XBotControl\Storage::getInstance()->db->quit(); });