openssl versions not parsing correctly; additional logger config (#259)

* [CveXplore-257] additional version parsing for better cpe matching results

* [CveXplore-257] centralized padded version and stem method to NVDAPIHANDLER class

* [CveXplore-253] loggers updated

CveXplore/.schema_version

@@ -1,4 +1,4 @@
 {
-    "version": "1.7",
+    "version": "1.8",
     "rebuild_needed": true
 }

CveXplore/VERSION

@@ -1 +1 @@
-0.3.20.dev23
+0.3.20.dev23

CveXplore/core/database_maintenance/api_handlers.py

@@ -34,6 +34,98 @@ def process_item(self, item: dict):
                 # ).entry
                 return item
 
+    @staticmethod
+    def stem(cpe_uri: str):
+        cpe_stem = cpe_uri.split(":")
+        return ":".join(cpe_stem[:5])
+
+    @staticmethod
+    def padded_version(version: str):
+        if version == "-" or version == "":
+            return version
+        else:
+            # normalizing edge cases:
+            version = version.replace("\\(", ".").replace("\\)", ".").rstrip(".")
+
+            ret_list = []
+
+            splitted_version = version.split(".")
+            # perform check if last part of version can be cast to an int
+            try:
+                int(splitted_version[-1])
+                # can be cast to an int, proceed 'normally'
+                for v in splitted_version:
+                    try:
+                        ret_list.append(f"{int(v):05d}")
+                    except ValueError:
+                        ret_list.append(v.rjust(5, "0"))
+            except ValueError:
+                # last part of the version cannot be cast to an int, so this means it's either a string or a
+                # string combined with an integer; handle accordingly
+
+                # first handle all version identifiers leading upto the last part
+                if len(splitted_version) > 1:
+                    for i in range(len(splitted_version) - 1):
+                        try:
+                            ret_list.append(f"{int(splitted_version[i]):05d}")
+                        except ValueError:
+                            ret_list.append(splitted_version[i].rjust(5, "0"))
+
+                # handle the last part
+                # check if the last entry is smaller than 5 characters, if so just use that...
+                if len(splitted_version[-1]) > 5:
+                    try:
+                        ret_list.append(f"{int(splitted_version[-1]):05d}")
+                    except ValueError:
+                        ret_list.append(splitted_version[-1].rjust(5, "0"))
+                # check is last entry consists only of alphanumeric characters
+                elif splitted_version[-1].isalpha():
+                    ret_list.append(splitted_version[-1].rjust(5, "0"))
+                else:
+                    loop_i = 0
+                    loop_count = len(splitted_version[-1])
+
+                    # int/str combined value; handle accordingly
+                    while loop_i < loop_count:
+                        current_i = loop_i
+                        # probably digit; so check;
+                        if splitted_version[-1][loop_i].isdigit():
+                            try:
+                                ret_list.append(
+                                    f"{int(splitted_version[-1][loop_i]):05d}"
+                                )
+                            except ValueError:
+                                ret_list.append(
+                                    splitted_version[-1][loop_i].rjust(5, "0")
+                                )
+                            finally:
+                                # perform check if anything that follows consists only of string characters
+                                if splitted_version[-1][loop_i + 1 :].isalpha():
+                                    ret_list.append(
+                                        splitted_version[-1][loop_i + 1 :].rjust(5, "0")
+                                    )
+                                    # no point proceeding; just break
+                                    break
+                                loop_i += 1
+                        else:
+                            # ok so probably last part of version identifier is a string; add that with a loop
+                            version_string = ""
+                            try:
+                                while splitted_version[-1][loop_i].isalpha():
+                                    version_string += splitted_version[-1][loop_i]
+                                    loop_i += 1
+                            except IndexError:
+                                # finished splitted_version variable; just pass
+                                loop_i += 1
+                                pass
+
+                            ret_list.append(version_string.rjust(5, "0"))
+
+                        if loop_i == current_i:
+                            loop_i += 1
+
+            return ".".join(ret_list)
+
     @abstractmethod
     def process_the_item(self, *args):
         raise NotImplementedError

CveXplore/core/database_maintenance/download_handler.py

@@ -15,7 +15,6 @@
 from datetime import timedelta
 from io import BytesIO
 from itertools import islice
-from logging.handlers import RotatingFileHandler
 from shutil import copy
 from typing import Tuple
 
@@ -30,6 +29,8 @@
 from CveXplore.core.general.utils import sanitize
 from CveXplore.core.worker_queue.worker_q import WorkerQueue
 from ..database_indexer.db_indexer import DatabaseIndexer
+from ..logging.handlers.cve_explore_rfh import CveExploreUpdateRfhHandler
+from ..logging.handlers.cve_explore_stream import CveExploreUpdateStreamHandler
 from ..logging.logger_class import AppLogger
 from ...database.connection.database_connection import DatabaseConnection
 
@@ -81,7 +82,8 @@ def __init__(
 
         self.logger = logging.getLogger(logger_name)
 
-        self.logger.removeHandler(self.logger.handlers[0])
+        if len(self.logger.handlers) == 1:
+            self.logger.removeHandler(self.logger.handlers[0])
 
         self.logger.propagate = False
 
@@ -91,23 +93,30 @@ def __init__(
 
         crf = None
 
-        cli = logging.StreamHandler(stream=sys.stdout)
+        cli = CveExploreUpdateStreamHandler(stream=sys.stdout)
         cli.setFormatter(self.formatter)
         cli.setLevel(logging.INFO)
 
-        if self.config.LOGGING_FILE_PATH != "":
-            if not os.path.exists(self.config.LOGGING_FILE_PATH):
-                os.makedirs(self.config.LOGGING_FILE_PATH)
-
-            crf = RotatingFileHandler(
+        if self.config.LOGGING_TO_FILE:
+            crf = CveExploreUpdateRfhHandler(
                 filename=f"{self.config.LOGGING_FILE_PATH}/{self.config.LOGGING_UPDATE_FILE_NAME}",
                 maxBytes=self.config.LOGGING_MAX_FILE_SIZE,
                 backupCount=self.config.LOGGING_BACKLOG,
             )
             crf.setLevel(logging.DEBUG)
             crf.setFormatter(self.formatter)
 
-        if not len(self.logger.handlers):
+        if len(self.logger.handlers) > 0:
+            for handler in self.logger.handlers:
+                # add the handlers to the logger
+                # makes sure no duplicate handlers are added
+                if not isinstance(
+                    handler, CveExploreUpdateRfhHandler
+                ) and not isinstance(handler, CveExploreUpdateStreamHandler):
+                    if crf is not None:
+                        self.logger.addHandler(crf)
+                    self.logger.addHandler(cli)
+        else:
             if crf is not None:
                 self.logger.addHandler(crf)
             self.logger.addHandler(cli)

CveXplore/core/database_maintenance/main_updater.py

@@ -53,6 +53,9 @@ def __init__(self, datasource: DatabaseConnectionBase):
 
         self.do_initialize = False
 
+    def __repr__(self):
+        return f"<<MainUpdater>>"
+
     def validate_schema(self):
         return self.schema_checker.validate_schema()
 

CveXplore/core/database_maintenance/sources_process.py

@@ -47,25 +47,6 @@ def __init__(self):
     def file_to_queue(self, *args):
         pass
 
-    @staticmethod
-    def stem(cpe_uri: str):
-        cpe_stem = cpe_uri.split(":")
-        return ":".join(cpe_stem[:5])
-
-    @staticmethod
-    def padded_version(version: str):
-        if version == "-" or version == "":
-            return version
-        else:
-            ret_list = []
-            for v in version.split("."):
-                try:
-                    ret_list.append(f"{int(v):05d}")
-                except ValueError:
-                    ret_list.append(v.rjust(5, "0"))
-
-            return ".".join(ret_list)
-
     @staticmethod
     def parse_cpe_version(cpename: str):
         cpe_list = cpename.split(":")
@@ -390,25 +371,6 @@ def get_vendor_product(cpeUri: str):
         product = cpeUri.split(":")[4]
         return vendor, product
 
-    @staticmethod
-    def stem(cpeUri: str):
-        cpeArr = cpeUri.split(":")
-        return ":".join(cpeArr[:5])
-
-    @staticmethod
-    def padded_version(version: str):
-        if version == "-" or version == "":
-            return version
-        else:
-            ret_list = []
-            for v in version.split("."):
-                try:
-                    ret_list.append(f"{int(v):05d}")
-                except ValueError:
-                    ret_list.append(v.rjust(5, "0"))
-
-            return ".".join(ret_list)
-
     def file_to_queue(self, *args):
         pass
 

CveXplore/core/database_maintenance/update_base_class.py

@@ -1,17 +1,20 @@
 import logging
-import os
 import sys
-from logging.handlers import RotatingFileHandler
 
 from CveXplore.common.config import Configuration
+from CveXplore.core.logging.handlers.cve_explore_rfh import CveExploreUpdateRfhHandler
+from CveXplore.core.logging.handlers.cve_explore_stream import (
+    CveExploreUpdateStreamHandler,
+)
 
 
 class UpdateBaseClass(object):
     def __init__(self, logger_name: str):
         self.config = Configuration
         self.logger = logging.getLogger(logger_name)
 
-        self.logger.removeHandler(self.logger.handlers[0])
+        if len(self.logger.handlers) == 1:
+            self.logger.removeHandler(self.logger.handlers[0])
 
         self.logger.propagate = False
 
@@ -21,23 +24,30 @@ def __init__(self, logger_name: str):
 
         crf = None
 
-        cli = logging.StreamHandler(stream=sys.stdout)
+        cli = CveExploreUpdateStreamHandler(stream=sys.stdout)
         cli.setFormatter(self.formatter)
         cli.setLevel(logging.INFO)
 
-        if self.config.LOGGING_FILE_PATH != "":
-            if not os.path.exists(self.config.LOGGING_FILE_PATH):
-                os.makedirs(self.config.LOGGING_FILE_PATH)
-
-            crf = RotatingFileHandler(
+        if self.config.LOGGING_TO_FILE:
+            crf = CveExploreUpdateRfhHandler(
                 filename=f"{self.config.LOGGING_FILE_PATH}/{self.config.LOGGING_UPDATE_FILE_NAME}",
                 maxBytes=self.config.LOGGING_MAX_FILE_SIZE,
                 backupCount=self.config.LOGGING_BACKLOG,
             )
             crf.setLevel(logging.DEBUG)
             crf.setFormatter(self.formatter)
 
-        if not len(self.logger.handlers):
+        if len(self.logger.handlers) > 0:
+            for handler in self.logger.handlers:
+                # add the handlers to the logger
+                # makes sure no duplicate handlers are added
+                if not isinstance(
+                    handler, CveExploreUpdateRfhHandler
+                ) and not isinstance(handler, CveExploreUpdateStreamHandler):
+                    if crf is not None:
+                        self.logger.addHandler(crf)
+                    self.logger.addHandler(cli)
+        else:
             if crf is not None:
                 self.logger.addHandler(crf)
             self.logger.addHandler(cli)

CveXplore/core/logging/handlers/cve_explore_rfh.py

@@ -0,0 +1,6 @@
+from logging.handlers import RotatingFileHandler
+
+
+class CveExploreUpdateRfhHandler(RotatingFileHandler):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)

CveXplore/core/logging/handlers/cve_explore_stream.py

@@ -0,0 +1,6 @@
+from logging import StreamHandler
+
+
+class CveExploreUpdateStreamHandler(StreamHandler):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)