Merge pull request #464 from intelowlproject/develop

1.5.0

Author: mlodic

.gitignore

@@ -4,5 +4,6 @@ docker/env_file
 docker/env_file_postgres
 .env
 __pycache__/
+mlmodels/
 # JetBrains IDEs (PyCharm, IntelliJ, etc.)
-.idea/
+.idea/

api/views/utils.py

@@ -254,7 +254,10 @@ def feeds_response(iocs, feed_params, valid_feed_types, dict_only=False, verbose
 
                 if verbose and ioc_feed_type in ["log4j", "cowrie"]:
                     data_["honeypots"] = [hp for hp in data_["honeypots"] if hp is not None]
-                    data_["honeypots"].append(ioc_feed_type)
+                    if ioc["log4j"]:
+                        data_["honeypots"].append("log4j")
+                    if ioc["cowrie"]:
+                        data_["honeypots"].append("cowrie")
 
                 if verbose:
                     json_list.append(data_)

docker/.version

@@ -1 +1 @@
-REACT_APP_GREEDYBEAR_VERSION="1.4.4"
+REACT_APP_GREEDYBEAR_VERSION="1.5.0"

docker/Dockerfile

@@ -42,8 +42,9 @@ RUN touch ${LOG_PATH}/django/api.log ${LOG_PATH}/django/api_errors.log \
     && touch ${LOG_PATH}/django/celery.log ${LOG_PATH}/django/celery_errors.log \
     && touch ${LOG_PATH}/django/django_errors.log ${LOG_PATH}/django/elasticsearch.log\
     && touch ${LOG_PATH}/django/authentication.log ${LOG_PATH}/django/authentication_errors.log \
+    && mkdir -p ${PYTHONPATH}/mlmodels \
     && adduser -S -H -u 2000 -D -g www-data www-data \
-    && chown -R www-data:www-data ${LOG_PATH} /opt/deploy/ \
+    && chown -R www-data:www-data ${LOG_PATH} /opt/deploy/ ${PYTHONPATH}/mlmodels/ \
     && rm -rf docs/ frontend/ tests/ .github/ docker/hooks/ \
     && /bin/bash ./docker/watchman_install.sh \
     && apk del gcc python3-dev alpine-sdk linux-headers

docker/Dockerfile_nginx

@@ -1,4 +1,4 @@
-FROM library/nginx:1.27.3-alpine
+FROM library/nginx:1.27.4-alpine
 RUN mkdir -p /var/cache/nginx /var/cache/nginx/feeds
 RUN apk update && apk upgrade && apk add bash
 ENV NGINX_LOG_DIR=/var/log/nginx

docker/default.yml

@@ -76,6 +76,7 @@ services:
     command: /usr/local/bin/celery -A greedybear.celery worker -n worker_default --uid www-data --gid www-data --time-limit=10000 --pidfile= -Ofair -Q default -E -c 1
     volumes:
       - generic_logs:/var/log/greedybear
+      - mlmodels:/opt/deploy/greedybear/mlmodels
     env_file:
       - env_file
     depends_on:
@@ -90,3 +91,4 @@ volumes:
   nginx_logs:
   generic_logs:
   static_content:
+  mlmodels:

frontend/src/components/auth/Login.jsx

@@ -7,14 +7,16 @@ import {
   Spinner,
   Button,
   Row,
+  Alert,
 } from "reactstrap";
+import { MdInfoOutline } from "react-icons/md";
 import { Form, Formik } from "formik";
 
 import { ContentSection } from "@certego/certego-ui";
 
 import { useAuthStore } from "../../stores";
 import {
-  ResendVerificationEmailButton,
+  // ResendVerificationEmailButton,
   ForgotPasswordButton,
 } from "./utils/registration-buttons";
 
@@ -66,6 +68,20 @@ function Login() {
         <ContentSection>
           <h3 className="fw-bold">Log In</h3>
           <hr />
+          <Alert
+            color="accent-2"
+            id="access-info"
+            className="col-12 px-1 text-center"
+          >
+            <h5 className="text-info">
+              <MdInfoOutline size="1.15rem" />
+              &nbsp;New users
+            </h5>
+            <p>
+              If you do not have an account please contact a member of The
+              Honeynet Project who will provide you with credentials to log in
+            </p>
+          </Alert>
           {/* Form */}
           <Formik
             initialValues={initialValues}
@@ -126,7 +142,8 @@ function Login() {
         {/* popover buttons */}
         <Row className="d-flex flex-column align-items-end g-0">
           <ForgotPasswordButton />
-          <ResendVerificationEmailButton />
+          {/*  Registration operations have been temporarily disabled due to not proper Terms of Service */}
+          {/* <ResendVerificationEmailButton /> */}
         </Row>
       </Container>
     </ContentSection>

frontend/src/components/auth/Register.jsx

@@ -145,6 +145,7 @@ const onValidate = (values) => {
 
 // Component
 export default function Register() {
+  /* ATTENTION! The Register page has been temporarily disabled due to not proper Terms of Service */
   console.debug("Register rendered!");
 
   // page title

frontend/src/layouts/AppHeader.jsx

@@ -30,7 +30,8 @@ const guestLinks = (
         Login
       </RRNavLink>
     </NavItem>
-    <NavItem className="ms-lg-2">
+    {/*  The register button has been temporarily disabled due to not proper Terms of Service */}
+    {/* <NavItem className="ms-lg-2">
       <RRNavLink
         id="register-btn"
         className="btn btn-sm btn-accent-2"
@@ -39,7 +40,7 @@ const guestLinks = (
       >
         Register
       </RRNavLink>
-    </NavItem>
+    </NavItem> */}
   </>
 );
 

greedybear/admin.py

@@ -44,8 +44,9 @@ class IOCModelAdmin(admin.ModelAdmin):
         "destination_ports",
         "login_attempts",
     ]
-    search_fields = ["name"]
-    filter_horizontal = ["general_honeypot", "related_ioc"]
+    search_fields = ["name", "related_ioc__name"]
+    raw_id_fields = ["related_ioc"]
+    filter_horizontal = ["general_honeypot"]
 
     def general_honeypots(self, ioc):
         return ", ".join([str(element) for element in ioc.general_honeypot.all()])

greedybear/celery.py

@@ -95,4 +95,15 @@ def setup_loggers(*args, **kwargs):
         "schedule": crontab(minute=33),
         "options": {"queue": "default"},
     },
+    # SCORING
+    # Important:
+    # The training task must be run with a small offset after midnight (00:00)
+    # to ensure training data aligns with complete calendar days.
+    # The small offset is to make sure that the midnight extraction task is completed before training.
+    # This way models learn from complete rather than partial day patterns, which is crucial for their performance.
+    "train_and_update": {
+        "task": "greedybear.tasks.chain_train_and_update",
+        "schedule": crontab(hour=0, minute=hp_extraction_interval // 2),
+        "options": {"queue": "default"},
+    },
 }

greedybear/cronjobs/attacks.py

@@ -6,13 +6,13 @@
 from ipaddress import IPv4Address
 
 from greedybear.consts import DOMAIN, IP, PAYLOAD_REQUEST, SCANNER
-from greedybear.cronjobs.base import Cronjob
+from greedybear.cronjobs.base import ElasticJob
 from greedybear.cronjobs.sensors import ExtractSensors
 from greedybear.models import IOC, GeneralHoneypot, Sensors
 from greedybear.settings import EXTRACTION_INTERVAL, LEGACY_EXTRACTION
 
 
-class ExtractAttacks(Cronjob, metaclass=ABCMeta):
+class ExtractAttacks(ElasticJob, metaclass=ABCMeta):
     def __init__(self, minutes_back=None):
         super().__init__()
         self.first_time_run = False

greedybear/cronjobs/base.py

@@ -10,13 +10,33 @@
 
 
 class Cronjob(metaclass=ABCMeta):
+    def __init__(self):
+        self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+        self.success = False
+
+    @abstractmethod
+    def run(self):
+        pass
+
+    def execute(self):
+        try:
+            self.log.info("Starting execution")
+            self.run()
+        except Exception as e:
+            self.log.exception(e)
+        else:
+            self.success = True
+        finally:
+            self.log.info("Finished execution")
+
+
+class ElasticJob(Cronjob):
     class ElasticServerDownException(Exception):
         pass
 
     def __init__(self):
-        self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+        super().__init__()
         self.elastic_client = settings.ELASTIC_CLIENT
-        self.success = False
 
     def _healthcheck(self):
         if not self.elastic_client.ping():
@@ -58,21 +78,6 @@ def _base_search(self, honeypot):
     def minutes_back_to_lookup(self):
         pass
 
-    @abstractmethod
-    def run(self):
-        pass
-
-    def execute(self):
-        try:
-            self.log.info("Starting execution")
-            self.run()
-        except Exception as e:
-            self.log.exception(e)
-        else:
-            self.success = True
-        finally:
-            self.log.info("Finished execution")
-
 
 def get_time_window(reference_time: datetime, lookback_minutes: int = EXTRACTION_INTERVAL) -> tuple[datetime, datetime]:
     """

greedybear/cronjobs/cowrie.py

@@ -111,7 +111,9 @@ def _get_sessions(self, ioc):
                         session_record.start_time = hit.timestamp
                     case "cowrie.login.failed" | "cowrie.login.success":
                         session_record.login_attempt = True
-                        session_record.credentials.append(f"{hit.username} | {hit.password}")
+                        username = hit.username.replace("\x00", "[NUL]")
+                        password = hit.password.replace("\x00", "[NUL]")
+                        session_record.credentials.append(f"{username} | {password}")
                         session_record.source.login_attempts += 1
                     case "cowrie.command.input":
                         session_record.command_execution = True

greedybear/cronjobs/monitor_honeypots.py

@@ -1,11 +1,11 @@
 # This file is a part of GreedyBear https://github.com/honeynet/GreedyBear
 # See the file 'LICENSE' for copying permission.
-from greedybear.cronjobs.base import Cronjob
+from greedybear.cronjobs.base import ElasticJob
 from greedybear.cronjobs.honeypots import Honeypot
 from greedybear.models import GeneralHoneypot
 
 
-class MonitorHoneypots(Cronjob):
+class MonitorHoneypots(ElasticJob):
     def __init__(self):
         super(MonitorHoneypots, self).__init__()
         self.honeypots_to_monitor = [Honeypot("Log4pot"), Honeypot("Cowrie")]

greedybear/cronjobs/monitor_logs.py

@@ -3,11 +3,11 @@
 from datetime import datetime, timedelta
 from os.path import getmtime
 
-from greedybear.cronjobs.base import Cronjob
+from greedybear.cronjobs.base import ElasticJob
 from greedybear.slack import send_message
 
 
-class MonitorLogs(Cronjob):
+class MonitorLogs(ElasticJob):
     def __init__(self):
         super(MonitorLogs, self).__init__()
         self.logs_to_monitor = ["greedybear", "api", "django", "celery"]

greedybear/cronjobs/scoring/consts.py

@@ -0,0 +1,23 @@
+NUM_FEATURES = [
+    "honeypot_count",
+    "destination_port_count",
+    "days_seen_count",
+    "active_days_ratio",
+    "login_attempts",
+    "login_attempts_per_day",
+    "interaction_count",
+    "std_days_between",
+    "days_since_last_seen",
+    "days_since_first_seen",
+]
+
+CATEGORICAL_FEATURES = [
+    "asn",
+    "ip_reputation",
+]
+
+MULTI_VAL_FEATURES = [
+    "honeypots",
+]
+
+SAMPLE_COUNT = 100