🗓 Apr 4, 2022 7:32:27 PM

4.0.0
🔥 move some extractors and codetidy to plugins
🔥 move multimedia to plugins
➕ deps removed from core
🧪 tests added/updated

Author: securisecctf

.coveragerc

@@ -7,4 +7,4 @@ omit =
 
 
 [report]
-fail_under = 90
+fail_under = 100

chepy/__init__.py

@@ -10,7 +10,6 @@
 from .modules.hashing import Hashing
 from .modules.language import Language
 from .modules.links import Links
-from .modules.multimedia import Multimedia
 from .modules.networking import Networking
 from .modules.other import Other
 from .modules.publickey import Publickey
@@ -34,7 +33,6 @@ class Chepy(
     Hashing,
     Language,
     Links,
-    Multimedia,
     Networking,
     Other,
     Publickey,

chepy/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "3.5.1"  # pragma: no cover
+__version__ = "4.0.0"  # pragma: no cover
 __author__ = "Hapsida @securisec"  # pragma: no cover

chepy/chepy_plugins

@@ -256,7 +256,7 @@ def for_each(self, methods: List[Tuple[Union[str, object], dict]]):
             >>> c = Chepy(['41', '42'])
             >>> c.for_each([("from_hex",), ("to_hex",)])
             >>> # this is how to use fork methods with a string
-            >>> c.for([(c.from_hex,), (c.to_hex,)])
+            >>> c.for_each([(c.from_hex,), (c.to_hex,)])
             >>> # This is how to use fork using methods
             >>> print(c)
             ['41', '42']
@@ -539,7 +539,7 @@ def _convert_to_str(self) -> str:
             return str(self.state)
         elif isinstance(self.state, bytearray):
             return bytearray(self.state).decode()
-        elif isinstance(self.state, float):
+        elif isinstance(self.state, float): # pragma: no cover
             return format(self.state, "f")
         else:  # pragma: no cover
             # todo check more types here

chepy/core.py

@@ -1,12 +1,9 @@
 from typing import TypeVar
-import lazy_import
 
 import json
 
-pydash = lazy_import.lazy_module("pydash")
-import phpserialize
+import pydash
 import regex as re
-from lxml import etree
 from ..core import ChepyCore, ChepyDecorators
 
 CodeTidyT = TypeVar("CodeTidyT", bound="CodeTidy")
@@ -27,7 +24,9 @@ def minify_json(self) -> CodeTidyT:
             >>> c = Chepy("/path/to/file.json").load_file()
             >>> print(c.minify_json())
         """
-        self.state = json.dumps(json.loads(self._convert_to_str()), separators=(',', ':'))
+        self.state = json.dumps(
+            json.loads(self._convert_to_str()), separators=(",", ":")
+        )
         return self
 
     @ChepyDecorators.call_stack
@@ -47,56 +46,6 @@ def beautify_json(self, indent: int = 2) -> CodeTidyT:
         self.state = json.dumps(json.loads(self._convert_to_str()), indent=indent)
         return self
 
-    @ChepyDecorators.call_stack
-    def minify_xml(self) -> CodeTidyT:
-        """Minify XML string
-
-        Returns:
-            Chepy: The Chepy object.
-
-        Examples:
-            >>> c = Chepy("/path/to/file.xml").load_file()
-            >>> print(c.minify_xml())
-        """
-        parser = etree.XMLParser(remove_blank_text=True)
-        self.state = etree.tostring(
-            etree.fromstring(self._convert_to_bytes(), parser=parser)
-        )
-        return self
-
-    @ChepyDecorators.call_stack
-    def beautify_xml(self) -> CodeTidyT:
-        """Beautify compressed XML
-
-        Returns:
-            Chepy: The Chepy object.
-
-        Examples:
-            >>> c = Chepy("/path/to/file.xml").load_file()
-            >>> print(c.beautify_json())
-        """
-        self.state = etree.tostring(
-            etree.fromstring(self._convert_to_bytes()), pretty_print=True
-        )
-        return self
-
-    @ChepyDecorators.call_stack
-    def php_deserialize(self) -> CodeTidyT:
-        """Deserialize php to dict
-
-        Deserializes PHP serialized data, outputting keyed arrays as a python dict.
-
-        Returns:
-            Chepy: The Chepy object.
-
-        Examples:
-            >>> c = Chepy('a:3:{i:1;s:6:"elem 1";i:2;s:6:"elem 2";i:3;s:7:" elem 3";}')
-            >>> c.php_deserialize()
-            {1: b'elem 1', 2: b'elem 2', 3: b' elem 3'}
-        """
-        self.state = phpserialize.loads(self._convert_to_bytes())
-        return self
-
     @ChepyDecorators.call_stack
     def to_upper_case(self, by: str = "all") -> CodeTidyT:
         """Convert string to uppercase

chepy/modules/codetidy.py

@@ -9,9 +9,6 @@ class CodeTidy(ChepyCore):
     state: Any = ...
     def minify_json(self: CodeTidyT) -> CodeTidyT: ...
     def beautify_json(self: CodeTidyT, indent: int=...) -> CodeTidyT: ...
-    def minify_xml(self: CodeTidyT) -> CodeTidyT: ...
-    def beautify_xml(self: CodeTidyT) -> CodeTidyT: ...
-    def php_deserialize(self: CodeTidyT) -> CodeTidyT: ...
     def to_upper_case(self: CodeTidyT, by: Literal['all', 'word', 'sentence']=...) -> CodeTidyT: ...
     def to_lower_case(self: CodeTidyT) -> CodeTidyT: ...
     def to_snake_case(self: CodeTidyT) -> CodeTidyT: ...

chepy/modules/codetidy.pyi

@@ -10,7 +10,6 @@
 jwt = lazy_import.lazy_module("jwt")
 import pathlib
 
-import pycipher
 import regex as re
 import json
 
@@ -24,6 +23,7 @@
 PKCS1_OAEP = lazy_import.lazy_module("Crypto.Cipher.PKCS1_OAEP")
 Blowfish = lazy_import.lazy_module("Crypto.Cipher.Blowfish")
 Padding = lazy_import.lazy_module("Crypto.Util.Padding")
+pycipher = lazy_import.lazy_module("pycipher")
 
 from ..core import ChepyCore, ChepyDecorators
 from ..extras.combinatons import hex_chars

chepy/modules/encryptionencoding.py

@@ -1,12 +1,8 @@
 from typing import TypeVar
 from urllib.parse import urlparse as _pyurlparse
 
-import jsonpath_rw
-import lazy_import
 import regex as re
-import json
 
-parsel = lazy_import.lazy_module("parsel")
 from ..core import ChepyCore, ChepyDecorators
 
 ExtractorsT = TypeVar("ExtractorsT", bound="Extractors")
@@ -16,10 +12,6 @@ class Extractors(ChepyCore):
     def __init__(self, *data):
         super().__init__(*data)
 
-    def _parsel_obj(self):
-        """Returns a parsel.Selector object"""
-        return parsel.Selector(self._convert_to_str())
-
     @ChepyDecorators.call_stack
     def extract_hashes(self) -> ExtractorsT:
         """Extract md5, sha1, sha256 and sha512 hashes
@@ -207,79 +199,6 @@ def extract_domains(self, is_binary: bool = False) -> ExtractorsT:
         self.state = matched
         return self
 
-    @ChepyDecorators.call_stack
-    def xpath_selector(self, query: str, namespaces: str = None) -> ExtractorsT:
-        """Extract data using valid xpath selectors
-
-        Args:
-            query (str): Required. Xpath query
-            namespaces (str, optional): Namespace. Applies for XML data. Defaults to None.
-
-        Returns:
-            Chepy: The Chepy object.
-
-        Examples:
-            >>> c = Chepy("http://example.com")
-            >>> c.http_request()
-            >>> c.xpath_selector("//title/text()")
-            >>> c.get_by_index(0)
-            >>> c.o
-            "Example Domain"
-        """
-        self.state = (
-            parsel.Selector(self._convert_to_str(), namespaces=namespaces)
-            .xpath(query)
-            .getall()
-        )
-        return self
-
-    @ChepyDecorators.call_stack
-    def css_selector(self, query: str) -> ExtractorsT:
-        """Extract data using valid CSS selectors
-
-        Args:
-            query (str): Required. CSS query
-
-        Returns:
-            Chepy: The Chepy object.
-
-        Examples:
-            >>> c = Chepy("http://example.com")
-            >>> c.http_request()
-            >>> c.css_selector("title")
-            >>> c.get_by_index(0)
-            >>> c.o
-            "<title>Example Domain</title>"
-        """
-        self.state = self._parsel_obj().css(query).getall()
-        return self
-
-    @ChepyDecorators.call_stack
-    def jpath_selector(self, query: str) -> ExtractorsT:
-        """Query JSON with jpath query
-
-        `Reference <https://goessner.net/articles/JsonPath/index.html#e2>`__
-
-        Args:
-            query (str): Required. Query. For reference, see the help
-
-        Returns:
-            Chepy: The Chepy object.
-
-        Examples:
-            >>> c = Chepy("tests/files/test.json")
-            >>> c.load_file()
-            >>> c.jpath_selector("[*].name.first")
-            >>> c.get_by_index(2)
-            >>> c.o
-            "Long"
-        """
-        self.state = list(
-            j.value
-            for j in jsonpath_rw.parse(query).find(json.loads(self._convert_to_str()))
-        )
-        return self
-
     @ChepyDecorators.call_stack
     def html_comments(self) -> ExtractorsT:
         """Extract html comments

chepy/modules/extractors.py

@@ -1,7 +1,6 @@
 from ..core import ChepyCore
 from typing import Any, TypeVar
 
-parsel: Any
 ExtractorsT = TypeVar('ExtractorsT', bound='Extractors')
 
 class Extractors(ChepyCore):
@@ -14,9 +13,6 @@ class Extractors(ChepyCore):
     def extract_mac_address(self: ExtractorsT, is_binary: bool=...) -> ExtractorsT: ...
     def extract_urls(self: ExtractorsT, is_binary: bool=...) -> ExtractorsT: ...
     def extract_domains(self: ExtractorsT, is_binary: bool=...) -> ExtractorsT: ...
-    def xpath_selector(self: ExtractorsT, query: str, namespaces: str=...) -> ExtractorsT: ...
-    def css_selector(self: ExtractorsT, query: str) -> ExtractorsT: ...
-    def jpath_selector(self: ExtractorsT, query: str) -> ExtractorsT: ...
     def html_comments(self: ExtractorsT) -> ExtractorsT: ...
     def javascript_comments(self: ExtractorsT) -> ExtractorsT: ...
     def html_tags(self: ExtractorsT, tag: str) -> ExtractorsT: ...