1.5.2 (#477)

* Command sequences. Closes #457 (#468)

* add CommandSequence model

* add CommandSequence model to admin page

* make migration file

* add unique constraint to commands hash in CommandSequence model

* add extraction of command sequences

* add tests

* add clustering task for command sequences

* limit single command length during extraction

* add tests for clustering

* add 10 second delay to extraction jobs (will hopefully fix #451)

* removed twitter publish cause not working

* Deliver scores in Feeds API (#473)

* add scores to serializer

* fix docstring

* add scores to required fields in deeds_response function

* adapt tests

* fix constant assignments (see #469)

* make pending migration

* skip empty IP address fields when extracting attacker data
fixes #475

* Advanced feeds integration (#476)

* Rename "age" to "prioritize" in backend code and add new prioritization mechanisms

* Rename "age" to "prioritize" in frontend code

* fix tests

* adapt frontend tests

* Bump numpy from 2.2.2 to 2.2.3 in /requirements (#465)

Bumps [numpy](https://github.com/numpy/numpy) from 2.2.2 to 2.2.3.
- [Release notes](https://github.com/numpy/numpy/releases)
- [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst)
- [Commits](numpy/numpy@v2.2.2...v2.2.3)

---
updated-dependencies:
- dependency-name: numpy
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* bump

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: tim <[email protected]>
Co-authored-by: tim <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: 3 people

.env_template

@@ -13,4 +13,4 @@ COMPOSE_FILE=docker/default.yml:docker/local.override.yml
 #COMPOSE_FILE=docker/default.yml:docker/local.override.yml:docker/elasticsearch.yml
 
 # If you want to run a specific version, populate this
-# REACT_APP_INTELOWL_VERSION="1.5.1"
+# REACT_APP_INTELOWL_VERSION="1.5.2"

.github/workflows/twitter_publish.yml

@@ -127,6 +127,8 @@ class FeedsResponseSerializer(serializers.Serializer):
     asn = serializers.IntegerField(allow_null=True, min_value=1)
     destination_port_count = serializers.IntegerField(min_value=0)
     login_attempts = serializers.IntegerField(min_value=0)
+    recurrence_probability = serializers.FloatField(min_value=0, max_value=1)
+    expected_interactions = serializers.FloatField(min_value=0)
 
     def validate_feed_type(self, feed_type):
         logger.debug(f"FeedsResponseSerializer - validation feed_type: '{feed_type}'")

api/serializers.py

@@ -12,7 +12,7 @@
 urlpatterns = [
     path("feeds/", feeds_pagination),
     path("feeds/advanced/", feeds_advanced),
-    path("feeds/<str:feed_type>/<str:attack_type>/<str:age>.<str:format_>", feeds),
+    path("feeds/<str:feed_type>/<str:attack_type>/<str:prioritize>.<str:format_>", feeds),
     path("enrichment", enrichment_view),
     path("general_honeypot", general_honeypot_list),
     # router viewsets

api/urls.py

@@ -15,25 +15,25 @@
 
 
 @api_view([GET])
-def feeds(request, feed_type, attack_type, age, format_):
+def feeds(request, feed_type, attack_type, prioritize, format_):
     """
     Handle requests for IOC feeds with specific parameters and format the response accordingly.
 
     Args:
         request: The incoming request object.
         feed_type (str): Type of feed (e.g., log4j, cowrie, etc.).
         attack_type (str): Type of attack (e.g., all, specific attack types).
-        age (str): Age of the data to filter (e.g., recent, persistent).
+        prioritize (str): Prioritization mechanism to use (e.g., recent, persistent).
         format_ (str): Desired format of the response (e.g., json, csv, txt).
         exclude_mass_scanners (bool): query parameter flag to exclude IOCs that are known mass scanners.
 
     Returns:
         Response: The HTTP response with formatted IOC data.
     """
-    logger.info(f"request /api/feeds with params: feed type: {feed_type}, " f"attack_type: {attack_type}, Age: {age}, format: {format_}")
+    logger.info(f"request /api/feeds with params: feed type: {feed_type}, " f"attack_type: {attack_type}, prioritization: {prioritize}, format: {format_}")
 
     feed_params = FeedRequestParams({"feed_type": feed_type, "attack_type": attack_type, "format_": format_})
-    feed_params.set_legacy_age(age)
+    feed_params.set_prioritization(prioritize)
     if request.query_params and "exclude_mass_scanners" in request.query_params:
         feed_params.exclude_mass_scanners()
 
@@ -58,7 +58,7 @@ def feeds_pagination(request):
 
     feed_params = FeedRequestParams(request.query_params)
     feed_params.format = "json"
-    feed_params.set_legacy_age(request.query_params.get("age"))
+    feed_params.set_prioritization(request.query_params.get("prioritize"))
     if request.query_params and "exclude_mass_scanners" in request.query_params:
         feed_params.exclude_mass_scanners()
 
@@ -79,24 +79,17 @@ def feeds_advanced(request):
 
     Args:
         request: The incoming request object.
-
-    Supported query parameters are:
-
-        - **feed_type**: Type of feed to retrieve. (supported: `cowrie`, `log4j`, etc.; default: `all`)
-        - **attack_type**: Type of attack to filter. (supported: `scanner`, `payload_request`, `all`; default: `all`)
-        - **max_age**: Maximum number of days since last occurrence. \
-            E.g. an IOC that was last seen 4 days ago is excluded by default. (default: 3)
-        - **min_days_seen**: Minimum number of days on which an IOC must have been seen. (default: 1)
-        - **include_reputation**: `;`-separated list of reputation values to include, \
-            e.g. `known attacker` or `known attacker;` to include IOCs without reputation. (default: include all)
-        - **exclude_reputation**: `;`-separated list of reputation values to exclude, \
-            e.g. `mass scanner` or `mass scanner;bot, crawler`. (default: exclude none)
-        - **feed_size**: Number of IOC items to return. (default: 5000)
-        - **ordering**: Field to order results by, with optional `-` prefix for descending. (default: `-last_seen`)
-        - **verbose**: `true` to include IOC properties that contain a lot of data, e.g. the list of days it was seen. (default: `false`)
-        - **paginate**: `true` to paginate results. This forces the json format. (default: `false`)
-        - **format_**: Response format type. Besides `json`, `txt` and `csv` are supported \
-            but the response will only contain IOC values (e.g. IP adresses) without further information. (default: `json`)
+        feed_type (str): Type of feed to retrieve. (supported: `cowrie`, `log4j`, etc.; default: `all`)
+        attack_type (str): Type of attack to filter. (supported: `scanner`, `payload_request`, `all`; default: `all`)
+        max_age (int): Maximum number of days since last occurrence. E.g. an IOC that was last seen 4 days ago is excluded by default. (default: 3)
+        min_days_seen (int): Minimum number of days on which an IOC must have been seen. (default: 1)
+        include_reputation (str): `;`-separated list of reputation values to include, e.g. `known attacker` or `known attacker;` to include IOCs without reputation. (default: include all)
+        exclude_reputation (str): `;`-separated list of reputation values to exclude, e.g. `mass scanner` or `mass scanner;bot, crawler`. (default: exclude none)
+        feed_size (int): Number of IOC items to return. (default: 5000)
+        ordering (str): Field to order results by, with optional `-` prefix for descending. (default: `-last_seen`)
+        verbose (bool): `true` to include IOC properties that contain a lot of data, e.g. the list of days it was seen. (default: `false`)
+        paginate (bool): `true` to paginate results. This forces the json format. (default: `false`)
+        format (str): Response format type. Besides `json`, `txt` and `csv` are supported but the response will only contain IOC values (e.g. IP adresses) without further information. (default: `json`)
 
     Returns:
         Response: The HTTP response with formatted IOC data.

api/views/feeds.py

@@ -76,14 +76,8 @@ def __init__(self, query_params: dict):
     def exclude_mass_scanners(self):
         self.exclude_reputation.append("mass scanner")
 
-    def set_legacy_age(self, age: str):
-        """Translates legacy age specification into max_age and min_days_seen attributes
-        and sets ordering accordingly.
-
-        Parameters:
-            age (str): Age of the data to filter (recent or persistent).
-        """
-        match age:
+    def set_prioritization(self, prioritize: str):
+        match prioritize:
             case "recent":
                 self.max_age = "3"
                 self.min_days_seen = "1"
@@ -96,6 +90,14 @@ def set_legacy_age(self, age: str):
                 if "feed_type" in self.ordering:
                     self.feed_type_sorting = self.ordering
                     self.ordering = "-attack_count"
+            case "likely_to_recur":
+                self.max_age = "30"
+                self.min_days_seen = "1"
+                self.ordering = "-recurrence_probability"
+            case "most_expected_hits":
+                self.max_age = "30"
+                self.min_days_seen = "1"
+                self.ordering = "-expected_interactions"
 
 
 def get_valid_feed_types() -> frozenset[str]:
@@ -233,6 +235,8 @@ def feeds_response(iocs, feed_params, valid_feed_types, dict_only=False, verbose
                 "login_attempts",
                 "honeypots",
                 "days_seen",
+                "recurrence_probability",
+                "expected_interactions",
             }
             iocs = (ioc_as_dict(ioc, required_fields) for ioc in iocs) if isinstance(iocs, list) else iocs.values(*required_fields)
             for ioc in iocs:

api/views/utils.py

@@ -1 +1 @@
-REACT_APP_GREEDYBEAR_VERSION="1.5.1"
+REACT_APP_GREEDYBEAR_VERSION="1.5.2"

docker/.version

@@ -46,3 +46,7 @@ PUBLIC_DEPLOYMENT=False
 LEGACY_EXTRACTION=False
 # Interval for the honeypot data extraction in minutes (only choose divisors of 60)
 EXTRACTION_INTERVAL=10
+
+# Set True to cluster command sequences recorded by Cowrie once a day
+# This might be computationaly expensive on large Databases
+CLUSTER_COWRIE_COMMAND_SEQUENCES=False

docker/env_file_template

@@ -26,15 +26,17 @@ const attackTypeChoices = [
   { label: "Payload request", value: "payload_request" },
 ];
 
-const ageChoices = [
+const prioritizationChoices = [
   { label: "Recent", value: "recent" },
   { label: "Persistent", value: "persistent" },
+  { label: "Likely to recur", value: "likely_to_recur" },
+  { label: "Most expected hits", value: "most_expected_hits" },
 ];
 
 const initialValues = {
   feeds_type: "all",
   attack_type: "all",
-  age: "recent",
+  prioritize: "recent",
 };
 
 const initialState = {
@@ -59,7 +61,7 @@ export default function Feeds() {
   console.debug("Feeds-initialValues", initialValues);
 
   const [url, setUrl] = React.useState(
-    `${FEEDS_BASE_URI}/${initialValues.feeds_type}/${initialValues.attack_type}/${initialValues.age}.json`
+    `${FEEDS_BASE_URI}/${initialValues.feeds_type}/${initialValues.attack_type}/${initialValues.prioritize}.json`
   );
 
   // API to extract general honeypot
@@ -85,7 +87,7 @@ export default function Feeds() {
       params: {
         feed_type: initialValues.feeds_type,
         attack_type: initialValues.attack_type,
-        age: initialValues.age,
+        prioritize: initialValues.prioritize,
       },
       initialParams: {
         page: "1",
@@ -100,11 +102,11 @@ export default function Feeds() {
     (values) => {
       try {
         setUrl(
-          `${FEEDS_BASE_URI}/${values.feeds_type}/${values.attack_type}/${values.age}.json`
+          `${FEEDS_BASE_URI}/${values.feeds_type}/${values.attack_type}/${values.prioritize}.json`
         );
         initialValues.feeds_type = values.feeds_type;
         initialValues.attack_type = values.attack_type;
-        initialValues.age = values.age;
+        initialValues.prioritize = values.prioritize;
 
         const resetPage = {
           type: "gotoPage",
@@ -185,15 +187,15 @@ export default function Feeds() {
                         <Col sm={12} md={4}>
                           <Label
                             className="form-control-label"
-                            htmlFor="Feeds__age"
+                            htmlFor="Feeds__prioritize"
                           >
-                            Age:
+                            Prioritize:
                           </Label>
                           <Select
-                            id="Feeds__age"
-                            name="age"
-                            value={initialValues.age}
-                            choices={ageChoices}
+                            id="Feeds__prioritize"
+                            name="prioritize"
+                            value={initialValues.prioritize}
+                            choices={prioritizationChoices}
                             onChange={(e) => {
                               formik.handleChange(e);
                               formik.submitForm();

frontend/src/components/feeds/Feeds.jsx

@@ -72,8 +72,8 @@ describe("Feeds component", () => {
     expect(feedTypeSelectElement).toBeInTheDocument();
     const attackTypeSelectElement = screen.getByLabelText("Attack type:");
     expect(attackTypeSelectElement).toBeInTheDocument();
-    const ageSelectElement = screen.getByLabelText("Age:");
-    expect(ageSelectElement).toBeInTheDocument();
+    const prioritizationSelectElement = screen.getByLabelText("Prioritize:");
+    expect(prioritizationSelectElement).toBeInTheDocument();
 
     const buttonRawData = screen.getByRole("link", { name: /Raw data/i });
     expect(buttonRawData).toHaveAttribute(
@@ -83,7 +83,7 @@ describe("Feeds component", () => {
 
     await user.selectOptions(feedTypeSelectElement, "log4j");
     await user.selectOptions(attackTypeSelectElement, "scanner");
-    await user.selectOptions(ageSelectElement, "persistent");
+    await user.selectOptions(prioritizationSelectElement, "persistent");
 
     await waitFor(() => {
       // check link has been changed

frontend/tests/components/feeds/Feeds.test.jsx

@@ -5,7 +5,7 @@
 from django.contrib import admin, messages
 from django.db.models import Q
 from django.utils.translation import ngettext
-from greedybear.models import IOC, CowrieSession, GeneralHoneypot
+from greedybear.models import IOC, CommandSequence, CowrieSession, GeneralHoneypot
 
 logger = logging.getLogger(__name__)
 
@@ -15,11 +15,26 @@
 #     list_display = [field.name for field in Sensors._meta.get_fields()]
 
 
+class SessionInline(admin.TabularInline):
+    model = CowrieSession
+    fields = ["source", "start_time", "duration", "credentials", "interaction_count", "commands"]
+    readonly_fields = fields
+    show_change_link = True
+    extra = 0
+    ordering = ["-start_time"]
+
+
 @admin.register(CowrieSession)
 class CowrieSessionModelAdmin(admin.ModelAdmin):
     list_display = ["session_id", "start_time", "duration", "login_attempt", "credentials", "command_execution", "interaction_count", "source"]
     search_fields = ["source__name"]
-    raw_id_fields = ["source"]
+    raw_id_fields = ["source", "commands"]
+
+
+@admin.register(CommandSequence)
+class CommandSequenceModelAdmin(admin.ModelAdmin):
+    list_display = ["first_seen", "last_seen", "cluster", "commands"]
+    inlines = [SessionInline]
 
 
 @admin.register(IOC)
@@ -47,6 +62,7 @@ class IOCModelAdmin(admin.ModelAdmin):
     search_fields = ["name", "related_ioc__name"]
     raw_id_fields = ["related_ioc"]
     filter_horizontal = ["general_honeypot"]
+    inlines = [SessionInline]
 
     def general_honeypots(self, ioc):
         return ", ".join([str(element) for element in ioc.general_honeypot.all()])

greedybear/admin.py

@@ -62,20 +62,20 @@ def setup_loggers(*args, **kwargs):
     "extract_log4pot": {
         "task": "greedybear.tasks.extract_log4pot",
         "schedule": crontab(minute=f"*/{hp_extraction_interval}"),
-        "options": {"queue": "default"},
+        "options": {"queue": "default", "countdown": 10},
     },
     # every 10 minutes or according to EXTRACTION_INTERVAL
     "extract_cowrie": {
         "task": "greedybear.tasks.extract_cowrie",
         "schedule": crontab(minute=f"*/{hp_extraction_interval}"),
-        "options": {"queue": "default"},
+        "options": {"queue": "default", "countdown": 10},
     },
     # FEEDS
     # every 10 minutes or according to EXTRACTION_INTERVAL
     "extract_general": {
         "task": "greedybear.tasks.extract_general",
         "schedule": crontab(minute=f"*/{hp_extraction_interval}"),
-        "options": {"queue": "default"},
+        "options": {"queue": "default", "countdown": 10},
     },
     # once a day
     "extract_sensors": {
@@ -106,4 +106,11 @@ def setup_loggers(*args, **kwargs):
         "schedule": crontab(hour=0, minute=hp_extraction_interval // 2),
         "options": {"queue": "default"},
     },
+    # COMMANDS
+    # once a day
+    "command_clustering": {
+        "task": "greedybear.tasks.cluster_commands",
+        "schedule": crontab(hour=1, minute=3),
+        "options": {"queue": "default"},
+    },
 }

greedybear/celery.py

@@ -75,6 +75,9 @@ def _get_attacker_data(self, honeypot, fields: list) -> list:
             hits_by_ip[hit.src_ip].append(hit.to_dict())
         iocs = []
         for ip, hits in hits_by_ip.items():
+            # skip empty IP addresses
+            if not ip.strip():
+                continue
             dest_ports = [hit["dest_port"] for hit in hits if "dest_port" in hit]
             ioc = IOC(
                 name=ip,