Merge pull request #141 from intelowlproject/develop

4.1.4

Author: mlodic

.github/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [4.1.4](https://github.com/intelowlproject/pyintelowl/releases/tag/4.1.4)
+
+- Added support for URLs that use TCP as protocol
+- Updated linters + formatted code with `isort`
+
 ## [4.1.3](https://github.com/intelowlproject/pyintelowl/releases/tag/4.1.3)
 
 - Library: `IntelOwl.ask_analysis_availability` now accepts an argument `minutes_ago`. Use to specify number of minutes to go back when searching for a previous analysis.

.github/workflows/pull_request_automation.yml

@@ -31,6 +31,10 @@ jobs:
         flake8 . --count
         black . --check
 
+    - name: isort
+      run: |
+        isort . --profile black --filter-files --check-only --diff --skip venv
+
     - name: Run tox tests
       run: |
         unzip -P infected tests/test_files.zip -d ./tests/

.pre-commit-config.yaml

@@ -1,9 +1,14 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 21.9b0
     hooks:
     - id: black
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.1
+    rev: 4.0.1
     hooks:
-    - id: flake8
+    - id: flake8
+-   repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--filter-files", "--skip", "pyquokka", "--skip", "venv"]

docs/conf.py

@@ -13,7 +13,7 @@
 import os
 import sys
 
-VERSION = "4.1.3"
+VERSION = "4.1.4"
 GITHUB_URL = "https://github.com/intelowlproject/pyintelowl"
 
 sys.path.append(os.path.abspath("../"))

pyintelowl/__init__.py

@@ -1,3 +1,3 @@
 # flake8: noqa
-from .pyintelowl import IntelOwl
 from .exceptions import IntelOwlClientException
+from .pyintelowl import IntelOwl

pyintelowl/cli/__init__.py

@@ -1,13 +1,13 @@
 from .analyse import analyse
-from .jobs import jobs
-from .tags import tags
-from .config import config
 from .commands import (
     analyzer_healthcheck,
     connector_healthcheck,
     get_analyzer_config,
     get_connector_config,
 )
+from .config import config
+from .jobs import jobs
+from .tags import tags
 
 groups = [
     analyse,

pyintelowl/cli/_jobs_utils.py

@@ -1,18 +1,22 @@
 import time
-from typing_extensions import Literal
+
 from rich import box
+from rich.console import Console, RenderGroup
 from rich.panel import Panel
-from rich.table import Table
 from rich.progress import track
-from rich.console import RenderGroup, Console
+from rich.table import Table
+from typing_extensions import Literal
+
+from pyintelowl.exceptions import IntelOwlClientException
+
 from ..cli._utils import (
+    get_json_syntax,
     get_status_text,
     get_success_text,
     get_tags_str,
-    get_json_syntax,
 )
-from pyintelowl.exceptions import IntelOwlClientException
-from ..cli.domain_checkers import Checkers, console as checkers_console
+from ..cli.domain_checkers import Checkers
+from ..cli.domain_checkers import console as checkers_console
 
 
 def _display_single_job(

pyintelowl/cli/_utils.py

@@ -1,11 +1,12 @@
-import click
-import logging
 import csv
 import json
+import logging
+
+import click
 from rich.emoji import Emoji
-from rich.text import Text
-from rich.syntax import Syntax
 from rich.logging import RichHandler
+from rich.syntax import Syntax
+from rich.text import Text
 
 from pyintelowl import IntelOwl
 

pyintelowl/cli/analyse.py

@@ -1,7 +1,8 @@
 import click
 
 from pyintelowl.exceptions import IntelOwlClientException
-from ..cli._utils import add_options, ClickContext, get_json_data
+
+from ..cli._utils import ClickContext, add_options, get_json_data
 
 __analyse_options = [
     click.option(

pyintelowl/cli/commands.py

@@ -1,19 +1,21 @@
-import click
 import json
 import re
-from rich.console import Console
-from rich.table import Table
+
+import click
 from rich import box
 from rich import print as rprint
+from rich.console import Console
+from rich.table import Table
+
+from pyintelowl.pyintelowl import IntelOwlClientException
 
 from ..cli._utils import (
     ClickContext,
-    get_success_text,
-    get_json_syntax,
     add_options,
+    get_json_syntax,
+    get_success_text,
     json_flag_option,
 )
-from pyintelowl.pyintelowl import IntelOwlClientException
 
 
 @click.command(

pyintelowl/cli/domain_checkers.py

@@ -1,6 +1,7 @@
-import geocoder
 import os
 from datetime import datetime
+
+import geocoder
 from rich.console import Console
 
 console = Console()

pyintelowl/cli/jobs.py

@@ -1,19 +1,22 @@
-import click
 import json
+
+import click
 from rich import print as rprint
 from rich.console import Console
+
 from pyintelowl.exceptions import IntelOwlClientException
-from ..cli._utils import (
-    ClickContext,
-    add_options,
-    json_flag_option,
-    get_action_status_text,
-)
+
 from ..cli._jobs_utils import (
     _display_all_jobs,
     _display_single_job,
-    _result_filter_and_tabular_print,
     _poll_for_job_cli,
+    _result_filter_and_tabular_print,
+)
+from ..cli._utils import (
+    ClickContext,
+    add_options,
+    get_action_status_text,
+    json_flag_option,
 )
 
 

pyintelowl/cli/tags.py

@@ -1,18 +1,19 @@
-import click
 import json
+
+import click
+from rich import box
+from rich import print as rprint
 from rich.console import Console
 from rich.table import Table
 from rich.text import Text
-from rich import box
-from rich import print as rprint
 
 from pyintelowl.exceptions import IntelOwlClientException
 
 from ..cli._utils import (
     ClickContext,
     add_options,
-    json_flag_option,
     get_action_status_text,
+    json_flag_option,
 )
 
 

pyintelowl/exceptions.py

@@ -1,5 +1,6 @@
 import json
 import typing
+
 from requests.exceptions import RequestException
 
 

pyintelowl/main.py

@@ -2,13 +2,9 @@
 import click
 import click_creds
 
+from .cli import cmds, groups
+from .cli._utils import ClickContext, get_logger, get_version_number
 from .pyintelowl import IntelOwl
-from .cli import groups, cmds
-from .cli._utils import (
-    get_logger,
-    ClickContext,
-    get_version_number,
-)
 
 
 @click.group(context_settings=dict(help_option_names=["-h", "--help"]))

pyintelowl/pyintelowl.py

@@ -1,24 +1,17 @@
+import hashlib
 import ipaddress
+import json
 import logging
 import pathlib
-import json
 import re
+from typing import Any, AnyStr, Callable, Dict, List, Optional, Union
+
 import requests
-import hashlib
-from typing import (
-    List,
-    Dict,
-    Any,
-    Optional,
-    Union,
-    AnyStr,
-    Callable,
-)
 from typing_extensions import Literal
 
 from pyintelowl.version import __version__
-from .exceptions import IntelOwlClientException
 
+from .exceptions import IntelOwlClientException
 
 TLPType = Literal["WHITE", "GREEN", "AMBER", "RED"]
 
@@ -608,7 +601,7 @@ def _get_observable_classification(self, value: str) -> str:
             ipaddress.ip_address(value)
         except ValueError:
             if re.match(
-                r"^(?:ht|f)tps?://[a-z\d-]{1,63}(?:\.[a-z\d-]{1,63})+"
+                r"^(?:htt|ft|tc)ps?://[a-z\d-]{1,63}(?:\.[a-z\d-]{1,63})+"
                 r"(?:/[a-z\d-]{1,63})*(?:\.\w+)?",
                 value,
             ):

pyintelowl/version.py

@@ -1 +1 @@
-__version__ = "4.1.3"
+__version__ = "4.1.4"

pyproject.toml

@@ -5,4 +5,7 @@ exclude = '''
   | venv
   | migrations
 )/
-'''
+'''
+[tool.isort]
+profile = "black"
+multi_line_output = 3

setup.py

@@ -6,7 +6,8 @@
 
 import os
 import pathlib
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 
 # The directory containing this file
 HERE = pathlib.Path(__file__).parent

test-requirements.txt

@@ -1,6 +1,7 @@
-black==20.8b1
+black==21.9b0
 flake8==4.0.1
+isort==5.10.1
 pre-commit==2.12.1
 tox==3.24.4
 tox-gh-actions==2.5.0
-codecov
+codecov==2.1.12

tests/mocked_requests.py

@@ -1,9 +1,4 @@
-from tests.utils import (
-    MockResponse,
-    get_file_data,
-    ROOT_DIR,
-    TEST_FILE,
-)
+from tests.utils import ROOT_DIR, TEST_FILE, MockResponse, get_file_data
 
 
 def mocked_raise_exception(*args, **kwargs):

tests/test_general.py

@@ -1,20 +1,14 @@
 from pyintelowl.exceptions import IntelOwlClientException
-
-from tests.utils import (
-    BaseTest,
-    mock_connections,
-    patch,
-    get_file_data,
-)
 from tests.mocked_requests import (
     mocked_analyzer_config,
-    mocked_ask_analysis_success,
-    mocked_ask_analysis_no_status,
     mocked_ask_analysis_no_job_id,
+    mocked_ask_analysis_no_status,
+    mocked_ask_analysis_success,
     mocked_connector_config,
-    mocked_send_analysis_success,
     mocked_raise_exception,
+    mocked_send_analysis_success,
 )
+from tests.utils import BaseTest, get_file_data, mock_connections, patch
 
 
 class TestGeneral(BaseTest):

tests/test_jobs.py

@@ -1,25 +1,19 @@
 from pyintelowl.exceptions import IntelOwlClientException
-
-from tests.utils import (
-    BaseTest,
-    mock_connections,
-    patch,
-    get_file_data,
-)
 from tests.mocked_requests import (
     mocked_analyzer_healthcheck,
     mocked_connector_healthcheck,
+    mocked_delete_job_by_id,
+    mocked_download_job_sample,
     mocked_get_all_jobs,
     mocked_get_job_by_id,
-    mocked_delete_job_by_id,
     mocked_kill_analyzer,
     mocked_kill_connector,
     mocked_kill_job,
-    mocked_download_job_sample,
     mocked_raise_exception,
     mocked_retry_analyzer,
     mocked_retry_connector,
 )
+from tests.utils import BaseTest, get_file_data, mock_connections, patch
 
 
 class TestJobs(BaseTest):