From b7c79ea47262583929aee0d793e4ae7a58ef2612 Mon Sep 17 00:00:00 2001 From: Lea9250 Date: Wed, 11 Oct 2023 16:07:49 +0200 Subject: [PATCH 1/3] refactor(cron_cve): add batch processing to cron_cve script + add logs --- crontab/cron_cve.php | 38 ++++++++- crontab/cron_cve_computer.php | 2 +- require/cve/Cve.php | 155 ++++++++++++++++++++-------------- 3 files changed, 127 insertions(+), 68 deletions(-) diff --git a/crontab/cron_cve.php b/crontab/cron_cve.php index f10d55121..24b7c2b67 100644 --- a/crontab/cron_cve.php +++ b/crontab/cron_cve.php @@ -10,7 +10,32 @@ $_SESSION['OCS']["writeServer"] = dbconnect(SERVER_WRITE, COMPTE_BASE, PSWD_BASE, DB_NAME, SSL_KEY, SSL_CERT, CA_CERT, SERVER_PORT); $_SESSION['OCS']["readServer"] = dbconnect(SERVER_READ, COMPTE_BASE, PSWD_BASE, DB_NAME, SSL_KEY, SSL_CERT, CA_CERT, SERVER_PORT); +$shortOptions = "c:h:d::"; +$longOptions = array("chunk:","help::","debug::"); + +$options = getopt($shortOptions, $longOptions); + +$chunk = 5000; +$debug = false; + +if(array_key_exists("h", $options) || array_key_exists("help", $options)) { + echo "Usage: php cron_cve.php [--] [args...]\n\n"; + echo " -c, --chunk Process software publishers by pool of , default 5000\n\n"; + echo " -d, --debug Display debug messages\n\n"; + exit(); +} + +if(array_key_exists("c", $options) || array_key_exists("chunk", $options)) { + $chunk = $options["c"] ?? $options["chunk"]; +} + +if (array_key_exists("d", $options) || array_key_exists("debug", $options)) { + $debug = true; +} + $cve = new Cve(); +$cve->setDebug($debug); + $date = null; $clean = false; @@ -20,6 +45,7 @@ if($cve->CVE_EXPIRE_TIME != null && $cve->CVE_EXPIRE_TIME != "" && $cve->CVE_EXPIRE_TIME != "0") { $date = date('Y/m/d H:i:s', time() - (3600 * $cve->CVE_EXPIRE_TIME)); $clean = true; + $cve->verbose("CVE_EXPIRE_TIME is set to ".$cve->CVE_EXPIRE_TIME." hour(s)", "DEBUG"); } $curl = curl_init($cve->CVE_SEARCH_URL); @@ -34,16 +60,20 @@ // Check if any error occured on cve-search server if(curl_errno($curl)) { $info = curl_getinfo($curl); - $cve->verbose($cve->CVE_VERBOSE, 1); + $cve->verbose("Error when connecting to cve-search server: ".$cve->CVE_SEARCH_URL." (".$info['http_code'].")", "INFO"); + $cve->verbose("Curl error: ".curl_error($curl), "DEBUG"); curl_close($curl); exit(); } else { curl_close($curl); - $cve->getSoftwareInformations($date, $clean); - $cve->verbose($cve->CVE_VERBOSE, 2); + $cve->verbose("Connected to cve-search server: ".$cve->CVE_SEARCH_URL, "INFO"); + $cve->verbose("When using a self-signed certificate, you can disable SSL verification in cron_cve.php", "INFO"); + $cve->verbose("Debug mode is ".($debug ? "enabled" : "disabled"), "INFO"); + $cve->getSoftwareInformations($date, $clean, $chunk); + $cve->verbose($this->cveNB." CVE have been added to database.", "INFO"); } } else { - $cve->verbose($cve->CVE_VERBOSE, 3); + $cve->verbose("CVE feature isn't enabled.", "INFO"); exit(); } diff --git a/crontab/cron_cve_computer.php b/crontab/cron_cve_computer.php index be3f39ee2..c1c828b2c 100644 --- a/crontab/cron_cve_computer.php +++ b/crontab/cron_cve_computer.php @@ -57,7 +57,7 @@ } print("[".date("Y-m-d H:i:s"). "] End of process\n"); } else { - $cve->verbose($cve->CVE_VERBOSE, 3); + $cve->verbose("CVE feature isn't enabled.", "INFO"); exit(); } diff --git a/require/cve/Cve.php b/require/cve/Cve.php index a0a8f8489..468cf7a2e 100644 --- a/require/cve/Cve.php +++ b/require/cve/Cve.php @@ -29,8 +29,10 @@ class Cve public $CVE_ACTIVE; private $CVE_BAN; public $CVE_LINK; + public $CVE_VERBOSE; public $CVE_EXPIRE_TIME; public $CVE_DELAY_TIME; + private $CVE_DEBUG; private $publisherName; public $cve_attr = []; public $cve_history = [ @@ -64,6 +66,13 @@ function __construct(){ } + /** + * Set debug mode + */ + public function setDebug($debug){ + $this->CVE_DEBUG = $debug; + } + /** * History cve history */ @@ -100,7 +109,7 @@ private function insertFlag() { /** * Get all publisher */ - private function getPublisher($date, $check_history) { + private function getPublisher($date = null, $check_history = false, $offset = null, $limit = null) { $sql = 'SELECT DISTINCT p.ID, p.PUBLISHER FROM software_publisher p LEFT JOIN software_link sl ON p.ID = sl.PUBLISHER_ID LEFT JOIN software_name n ON n.ID = sl.NAME_ID @@ -115,7 +124,12 @@ private function getPublisher($date, $check_history) { if($date != null && $check_history != 0) { $sql .= ' AND (h.FLAG_DATE <= "'.$date.'" OR p.ID NOT IN (SELECT PUBLISHER_ID FROM cve_search_history))'; } - $sql .= " ORDER BY p.PUBLISHER"; + + if (isset($offset) && isset($limit)) { + $sql .= " ORDER BY p.PUBLISHER LIMIT $offset, $limit"; + } else { + $sql .= " ORDER BY p.PUBLISHER"; + } return mysqli_query($_SESSION['OCS']["readServer"], $sql); } @@ -154,42 +168,64 @@ private function getSoftwareVersion($name_id) { /** * Get distinct all software name and publisher */ - public function getSoftwareInformations($date = null, $clean = false){ - - $this->verbose($this->CVE_VERBOSE, 4); + public function getSoftwareInformations($date = null, $clean = false, $poolSize){ + if ($this->CVE_BAN != "") { + // get names of banned categories + $banned = []; + $sql = "SELECT CATEGORY_NAME FROM software_categories WHERE ID IN (".$this->CVE_BAN.")"; + $result = mysql2_query_secure($sql, $_SESSION['OCS']["readServer"]); + while ($item = mysqli_fetch_array($result)) { + $banned[] = $item["CATEGORY_NAME"]; + } + $this->verbose("Banned categories (VULN_BAN_LIST): ".implode(", ", $banned), "DEBUG"); + } - $check_history = $this->history_is_empty(); - $publishers = $this->getPublisher($date, $check_history); + // total count of publishers and nb of chunks + $this->verbose("Getting total count of software publishers to process..", "INFO"); - $this->verbose($this->CVE_VERBOSE, 5); - - while ($item_publisher = mysqli_fetch_array($publishers)) { - # Reset date - $this->cve_history['FLAG'] = date('Y-m-d H:i:s'); - # Reset CVE NB - $this->cve_history['CVE_NB'] = 0; - $this->cve_history['PUBLISHER_ID'] = $item_publisher['ID']; - $this->clean_cve($item_publisher['ID']); - - $this->publisherName = $item_publisher['PUBLISHER']; - - $result_soft = $this->getSoftwareName($item_publisher['ID']); - - $this->verbose($this->CVE_VERBOSE, 6); + $totalPublishers = mysqli_num_rows($this->getPublisher($date, $this->history_is_empty())); + $chunks = ceil($totalPublishers / $poolSize); + + $this->verbose("Total publishers: $totalPublishers, chunks number: $chunks", "INFO"); + $this->verbose("Chunk size: $poolSize publishers", "INFO"); + $this->verbose("CVE processing is in progress, this could take a while ...", "INFO"); + for ($i = 0; $i < $chunks; $i++) { + $this->verbose("Processing publisher chunk " . ($i + 1) . " out of $chunks", "DEBUG"); + $publishers = $this->getPublisher($date, $this->history_is_empty(), $i * $poolSize, $poolSize); + + if (!$publishers) { + $this->verbose("Error fetching publishers in chunk " . ($i + 1) . ".", "DEBUG"); + continue; + } - while ($item_soft = mysqli_fetch_array($result_soft)) { - $this->cve_attr = null; - if(!preg_match('/[^\x00-\x7F]/', $item_soft['NAME']) && !preg_match('#\\{([^}]+)\\}#', $item_soft['NAME'])){ - $this->cve_history['NAME_ID'] = $item_soft['NAME_ID']; - $this->cve_attr[] = ["NAME" => $item_soft['NAME'], "VENDOR" => $item_publisher['PUBLISHER'], "VERSION" => null, "REAL_NAME" => $item_soft['NAME'], "REAL_VENDOR" => $item_publisher['PUBLISHER']]; - if($this->cve_attr != null) { - $this->get_cve($this->cve_attr); + while ($item_publisher = mysqli_fetch_array($publishers)) { + # Reset date + $this->cve_history['FLAG'] = date('Y-m-d H:i:s'); + # Reset CVE NB + $this->cve_history['CVE_NB'] = 0; + $this->cve_history['PUBLISHER_ID'] = $item_publisher['ID']; + $this->clean_cve($item_publisher['ID']); + + $this->publisherName = $item_publisher['PUBLISHER']; + + $result_soft = $this->getSoftwareName($item_publisher['ID']); + + $this->verbose("Processing publisher: ".$item_publisher['PUBLISHER'], "DEBUG"); + + while ($item_soft = mysqli_fetch_array($result_soft)) { + $this->cve_attr = null; + if(!preg_match('/[^\x00-\x7F]/', $item_soft['NAME']) && !preg_match('#\\{([^}]+)\\}#', $item_soft['NAME'])){ + $this->cve_history['NAME_ID'] = $item_soft['NAME_ID']; + $this->cve_attr[] = ["NAME" => $item_soft['NAME'], "VENDOR" => $item_publisher['PUBLISHER'], "VERSION" => null, "REAL_NAME" => $item_soft['NAME'], "REAL_VENDOR" => $item_publisher['PUBLISHER']]; + if($this->cve_attr != null) { + $this->get_cve($this->cve_attr); + } + } } - } - } - $this->insertFlag(); + $this->insertFlag(); + } } } @@ -238,6 +274,7 @@ public function get_cve($cve_attr){ $curl = curl_init(); foreach($cve_attr as $values){ $values = $this->match($values); + $this->verbose("Processing publisher: ".$values['VENDOR']." for software : ".$values['NAME'], "DEBUG"); $url = trim($this->CVE_SEARCH_URL)."/api/search/".$values['VENDOR']."/".$values['NAME']; curl_setopt($curl, CURLOPT_HTTPHEADER, array('content-type: application/json')); curl_setopt($curl, CURLOPT_URL, $url); @@ -245,10 +282,26 @@ public function get_cve($cve_attr){ // Uncomment if using a self-signed certificate on CVE server //curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, false); //curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false); + + $this->verbose("Sending request to ".$url, "DEBUG"); + $result = curl_exec ($curl); + + // check curl request + if($result == false) { + $this->verbose("Error while fetching CVE data from ".$url, "INFO"); + $this->verbose("Curl error code: ".curl_errno($curl)." - ".curl_error($curl), "DEBUG"); + continue; + } else { + $this->verbose("Data fetched successfully from ".$url, "DEBUG"); + } + $vars = json_decode($result, true); if(isset($vars['total']) && $vars['total'] != 0){ + $this->verbose("CVE data found for ".$values['VENDOR']."/".$values['NAME'], "INFO"); $this->search_by_version($vars, $values); + } else { + $this->verbose("No CVE data found for ".$values['VENDOR']."/".$values['NAME'], "INFO"); } } @@ -354,9 +407,7 @@ private function search_by_version($vars, $software){ if((!empty(strval($vuln_conf)) && (strpos(strval($vuln), strval($vuln_conf)) !== false))){ $result = $this->get_infos_cve($values['cvss'] ?? $values['cvss3'], $values['id'], $values['references'][0]); if($result) { - if($this->CVE_VERBOSE == 1) { - print("[".date("Y-m-d H:i:s"). "] ".$values['id']." has been referenced for ".$software["REAL_NAME"]."\n"); - } + $this->verbose($values['id']." has been referenced for ".$software["REAL_NAME"]." version ".$software["VERSION"], "INFO"); $this->cve_history['CVE_NB'] ++; $this->cveNB ++; } @@ -411,35 +462,13 @@ public function clean_cve($publisher){ } /** - * Print verbose + * Print debug statement depending on the level of debug needed + * @param string $string + * @param string $level */ - public function verbose($config, $code) { - if($config == 1) { - switch($code) { - case 1: - print("[".date("Y-m-d H:i:s"). "] ".$this->CVE_SEARCH_URL." is not reachable\n"); - break; - case 2: - print("[".date("Y-m-d H:i:s"). "] ".$this->cveNB." CVE have been added to database\n"); - break; - case 3: - print("[".date("Y-m-d H:i:s"). "] CVE feature isn't enabled\n"); - break; - case 4: - print("[".date("Y-m-d H:i:s"). "] Get software publisher\n"); - break; - case 5: - print("[".date("Y-m-d H:i:s"). "] Software publisher OK\n"); - print("[".date("Y-m-d H:i:s"). "] CVE treatment started\n"); - print("[".date("Y-m-d H:i:s"). "] Please wait, CVE processing is in progress. It could take a few hours\n"); - break; - case 6: - print("[".date("Y-m-d H:i:s"). "] Processing ".$this->publisherName." softwares\n"); - break; - case 7: - print("[".date("Y-m-d H:i:s"). "] ".$values['id']." has been referenced for ".$software["REAL_NAME"]."\n"); - break; - } + public function verbose($string, $level) { + if (($level == "DEBUG" && $this->CVE_DEBUG) || ($level == "INFO" && $this->CVE_VERBOSE)) { + print("[".date("Y-m-d H:i:s"). "] [$level] $string\n"); } } From 381d3cf8c5ca161a59b9371894a150810ad3d428 Mon Sep 17 00:00:00 2001 From: Lea9250 Date: Wed, 11 Oct 2023 16:34:55 +0200 Subject: [PATCH 2/3] refactor(cron_cve): print info and debug when in debug mode --- require/cve/Cve.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/require/cve/Cve.php b/require/cve/Cve.php index 468cf7a2e..0b6e83786 100644 --- a/require/cve/Cve.php +++ b/require/cve/Cve.php @@ -463,11 +463,12 @@ public function clean_cve($publisher){ /** * Print debug statement depending on the level of debug needed + * If in debug mode, print all messages, if only verbose mode, print only INFO * @param string $string * @param string $level */ public function verbose($string, $level) { - if (($level == "DEBUG" && $this->CVE_DEBUG) || ($level == "INFO" && $this->CVE_VERBOSE)) { + if (($level == "DEBUG" && $this->CVE_DEBUG) || ($level == "INFO" && ($this->CVE_VERBOSE || $this->CVE_DEBUG))) { print("[".date("Y-m-d H:i:s"). "] [$level] $string\n"); } } From 293c8594695dc28021ad7f201544cd7d062b5010 Mon Sep 17 00:00:00 2001 From: Lea9250 Date: Mon, 16 Oct 2023 16:12:08 +0200 Subject: [PATCH 3/3] refactor(cve): improved performance for software publisher query --- crontab/cron_cve.php | 2 +- require/cve/Cve.php | 79 ++++++++++++++++++++++---------------------- 2 files changed, 40 insertions(+), 41 deletions(-) diff --git a/crontab/cron_cve.php b/crontab/cron_cve.php index 24b7c2b67..121eb9c7e 100644 --- a/crontab/cron_cve.php +++ b/crontab/cron_cve.php @@ -70,7 +70,7 @@ $cve->verbose("When using a self-signed certificate, you can disable SSL verification in cron_cve.php", "INFO"); $cve->verbose("Debug mode is ".($debug ? "enabled" : "disabled"), "INFO"); $cve->getSoftwareInformations($date, $clean, $chunk); - $cve->verbose($this->cveNB." CVE have been added to database.", "INFO"); + $cve->verbose($cve->getNbAdded()." CVE have been added to database.", "INFO"); } } else { $cve->verbose("CVE feature isn't enabled.", "INFO"); diff --git a/require/cve/Cve.php b/require/cve/Cve.php index 0b6e83786..1fa3e0c4d 100644 --- a/require/cve/Cve.php +++ b/require/cve/Cve.php @@ -73,6 +73,10 @@ public function setDebug($debug){ $this->CVE_DEBUG = $debug; } + public function getNbAdded() { + return $this->cveNB; + } + /** * History cve history */ @@ -111,8 +115,6 @@ private function insertFlag() { */ private function getPublisher($date = null, $check_history = false, $offset = null, $limit = null) { $sql = 'SELECT DISTINCT p.ID, p.PUBLISHER FROM software_publisher p - LEFT JOIN software_link sl ON p.ID = sl.PUBLISHER_ID - LEFT JOIN software_name n ON n.ID = sl.NAME_ID LEFT JOIN cve_search_history h ON h.PUBLISHER_ID = p.ID LEFT JOIN software_categories_link scl ON scl.PUBLISHER_ID = p.ID WHERE p.ID != 1 AND TRIM(p.PUBLISHER) != ""'; @@ -179,53 +181,50 @@ public function getSoftwareInformations($date = null, $clean = false, $poolSize) } $this->verbose("Banned categories (VULN_BAN_LIST): ".implode(", ", $banned), "DEBUG"); } - - // total count of publishers and nb of chunks - $this->verbose("Getting total count of software publishers to process..", "INFO"); - - $totalPublishers = mysqli_num_rows($this->getPublisher($date, $this->history_is_empty())); - $chunks = ceil($totalPublishers / $poolSize); - - $this->verbose("Total publishers: $totalPublishers, chunks number: $chunks", "INFO"); $this->verbose("Chunk size: $poolSize publishers", "INFO"); $this->verbose("CVE processing is in progress, this could take a while ...", "INFO"); - for ($i = 0; $i < $chunks; $i++) { - $this->verbose("Processing publisher chunk " . ($i + 1) . " out of $chunks", "DEBUG"); - $publishers = $this->getPublisher($date, $this->history_is_empty(), $i * $poolSize, $poolSize); + // loop on all publishers based on chunk size + $numRows = $poolSize; + $chunkIndex = 1; + + for($limit = 0; $poolSize <= $numRows; $limit = $limit+$poolSize) { + $this->verbose("Processing publisher chunk " .$chunkIndex, "DEBUG"); + $publishers = $this->getPublisher($date, $this->history_is_empty(), $limit, $poolSize); + $numRows = mysqli_num_rows($publishers); + if (!$publishers) { + $this->verbose("Error fetching publishers in chunk " .$chunkIndex. ".", "DEBUG"); + continue; + } + + while ($item_publisher = mysqli_fetch_array($publishers)) { + # Reset date + $this->cve_history['FLAG'] = date('Y-m-d H:i:s'); + # Reset CVE NB + $this->cve_history['CVE_NB'] = 0; + $this->cve_history['PUBLISHER_ID'] = $item_publisher['ID']; + $this->clean_cve($item_publisher['ID']); - if (!$publishers) { - $this->verbose("Error fetching publishers in chunk " . ($i + 1) . ".", "DEBUG"); - continue; - } + $this->publisherName = $item_publisher['PUBLISHER']; - while ($item_publisher = mysqli_fetch_array($publishers)) { - # Reset date - $this->cve_history['FLAG'] = date('Y-m-d H:i:s'); - # Reset CVE NB - $this->cve_history['CVE_NB'] = 0; - $this->cve_history['PUBLISHER_ID'] = $item_publisher['ID']; - $this->clean_cve($item_publisher['ID']); - - $this->publisherName = $item_publisher['PUBLISHER']; - - $result_soft = $this->getSoftwareName($item_publisher['ID']); - - $this->verbose("Processing publisher: ".$item_publisher['PUBLISHER'], "DEBUG"); - - while ($item_soft = mysqli_fetch_array($result_soft)) { - $this->cve_attr = null; - if(!preg_match('/[^\x00-\x7F]/', $item_soft['NAME']) && !preg_match('#\\{([^}]+)\\}#', $item_soft['NAME'])){ - $this->cve_history['NAME_ID'] = $item_soft['NAME_ID']; - $this->cve_attr[] = ["NAME" => $item_soft['NAME'], "VENDOR" => $item_publisher['PUBLISHER'], "VERSION" => null, "REAL_NAME" => $item_soft['NAME'], "REAL_VENDOR" => $item_publisher['PUBLISHER']]; - if($this->cve_attr != null) { - $this->get_cve($this->cve_attr); - } + $result_soft = $this->getSoftwareName($item_publisher['ID']); + + $this->verbose("Processing publisher: ".$item_publisher['PUBLISHER'], "DEBUG"); + + while ($item_soft = mysqli_fetch_array($result_soft)) { + $this->cve_attr = null; + if(!preg_match('/[^\x00-\x7F]/', $item_soft['NAME']) && !preg_match('#\\{([^}]+)\\}#', $item_soft['NAME'])){ + $this->cve_history['NAME_ID'] = $item_soft['NAME_ID']; + $this->cve_attr[] = ["NAME" => $item_soft['NAME'], "VENDOR" => $item_publisher['PUBLISHER'], "VERSION" => null, "REAL_NAME" => $item_soft['NAME'], "REAL_VENDOR" => $item_publisher['PUBLISHER']]; + if($this->cve_attr != null) { + $this->get_cve($this->cve_attr); } } + } - $this->insertFlag(); + $this->insertFlag(); } + $chunkIndex += 1; } }