More DREI updates

Author: stamparm

extra/shutils/drei.sh

@@ -6,7 +6,8 @@
 # Stress test against Python3
 
 export SQLMAP_DREI=1
-for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
+#for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
+for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i; done
 unset SQLMAP_DREI
 source `dirname "$0"`"/junk.sh"
 

lib/controller/checks.py

@@ -1614,7 +1614,7 @@ def checkConnection(suppressOutput=False):
                 conf.url = re.sub(r"https?://", "https://", conf.url)
                 match = re.search(r":(\d+)", threadData.lastRedirectURL[1])
                 port = match.group(1) if match else 443
-                conf.url = re.sub(r":\d+(/|\Z)", ":%s\g<1>" % port, conf.url)
+                conf.url = re.sub(r":\d+(/|\Z)", r":%s\g<1>" % port, conf.url)
 
     except SqlmapConnectionException as ex:
         if conf.ipv6:

lib/core/common.py

@@ -1635,7 +1635,7 @@ def expandAsteriskForColumns(expression):
             if expression != conf.sqlQuery:
                 conf.db = db
             else:
-                expression = re.sub(r"([^\w])%s" % re.escape(conf.tbl), "\g<1>%s.%s" % (conf.db, conf.tbl), expression)
+                expression = re.sub(r"([^\w])%s" % re.escape(conf.tbl), r"\g<1>%s.%s" % (conf.db, conf.tbl), expression)
         else:
             conf.db = db
 
@@ -1795,12 +1795,24 @@ def getFileType(filePath):
 
     >>> getFileType(__file__)
     'text'
+    >>> getFileType(sys.executable)
+    'binary'
     """
 
     try:
         desc = getUnicode(magic.from_file(filePath) or "")
     except:
-        return "unknown"
+        desc = magic.MAGIC_UNKNOWN_FILETYPE
+
+    if desc == magic.MAGIC_UNKNOWN_FILETYPE:
+        content = openFile(filePath, "rb", encoding=None).read()
+
+        try:
+            content.decode()
+        except:
+            pass
+        else:
+            desc = "ascii"
 
     return "text" if any(_ in desc.lower() for _ in ("ascii", "text")) else "binary"
 
@@ -2053,8 +2065,8 @@ def isWindowsDriveLetterPath(filepath):
 
 def posixToNtSlashes(filepath):
     """
-    Replaces all occurrences of Posix slashes (/) in provided
-    filepath with NT ones (\)
+    Replaces all occurrences of Posix slashes in provided
+    filepath with NT backslashes
 
     >>> posixToNtSlashes('C:/Windows')
     'C:\\\\Windows'
@@ -2064,8 +2076,8 @@ def posixToNtSlashes(filepath):
 
 def ntToPosixSlashes(filepath):
     """
-    Replaces all occurrences of NT slashes (\) in provided
-    filepath with Posix ones (/)
+    Replaces all occurrences of NT backslashes in provided
+    filepath with Posix slashes
 
     >>> ntToPosixSlashes('C:\\Windows')
     'C:/Windows'
@@ -2954,7 +2966,7 @@ def findDynamicContent(firstPage, secondPage):
     infoMsg = "searching for dynamic content"
     singleTimeLogMessage(infoMsg)
 
-    blocks = SequenceMatcher(None, firstPage, secondPage).get_matching_blocks()
+    blocks = list(SequenceMatcher(None, firstPage, secondPage).get_matching_blocks())
     kb.dynamicMarkings = []
 
     # Removing too small matching blocks
@@ -4654,8 +4666,8 @@ def decloakToTemp(filename):
 
     content = decloak(filename)
 
-    parts = getBytes(os.path.split(filename[:-1])[-1]).split(b'.')
-    prefix, suffix = parts[0], b".%s" % parts[-1]
+    parts = os.path.split(filename[:-1])[-1].split('.')
+    prefix, suffix = parts[0], '.' + parts[-1]
     handle, filename = tempfile.mkstemp(prefix=prefix, suffix=suffix)
     os.close(handle)
 
@@ -4692,7 +4704,7 @@ def getRequestHeader(request, name):
 
     if request and request.headers and name:
         _ = name.upper()
-        retVal = max(value if _ == key.upper() else type(value)() for key, value in request.header_items()) or None
+        retVal = max(getBytes(value if _ == key.upper() else "") for key, value in request.header_items()) or None
 
     return retVal
 

lib/core/convert.py

@@ -11,6 +11,7 @@
     import pickle
 
 import base64
+import binascii
 import codecs
 import json
 import re
@@ -160,7 +161,10 @@ def decodeHex(value, binary=True):
     if value.lower().startswith("0x"):
         value = value[2:]
 
-    retVal = codecs.decode(value, "hex")
+    try:
+        retVal = codecs.decode(value, "hex")
+    except LookupError:
+        retVal = binascii.unhexlify(value)
 
     if not binary:
         retVal = getText(retVal)
@@ -180,7 +184,10 @@ def encodeHex(value, binary=True):
     if isinstance(value, six.text_type):
         value = value.encode(UNICODE_ENCODING)
 
-    retVal = codecs.encode(value, "hex")
+    try:
+        retVal = codecs.encode(value, "hex")
+    except LookupError:
+        retVal = binascii.hexlify(value)
 
     if not binary:
         retVal = getText(retVal)

lib/core/settings.py

@@ -18,7 +18,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.5.49"
+VERSION = "1.3.5.50"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/testing.py

@@ -410,7 +410,7 @@ def replaceVars(item, vars_):
     retVal = item
 
     if item and vars_:
-        for var in re.findall("\$\{([^}]+)\}", item):
+        for var in re.findall(r"\$\{([^}]+)\}", item):
             if var in vars_:
                 retVal = retVal.replace("${%s}" % var, vars_[var])
 

lib/parse/cmdline.py

@@ -697,6 +697,9 @@ def cmdLineParser(argv=None):
         parser.add_option("--murphy-rate", dest="murphyRate", type="int",
                           help=SUPPRESS_HELP)
 
+        parser.add_option("--debug", dest="debug", action="store_true",
+                          help=SUPPRESS_HELP)
+
         parser.add_option("--disable-precon", dest="disablePrecon", action="store_true",
                           help=SUPPRESS_HELP)
 

lib/request/basic.py

@@ -323,7 +323,7 @@ def decodePage(page, contentEncoding, contentType):
         # e.g. 	Ãëàâà
         if b"&#" in page:
             page = re.sub(b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)), page)
-            page = re.sub(b"&#(\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
+            page = re.sub(b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
 
         # e.g. %20%28%29
         if b"%" in page:

lib/request/connect.py

@@ -674,6 +674,9 @@ class _(dict):
         except (_urllib.error.URLError, socket.error, socket.timeout, _http_client.HTTPException, struct.error, binascii.Error, ProxyError, SqlmapCompressionException, WebSocketException, TypeError, ValueError, OverflowError):
             tbMsg = traceback.format_exc()
 
+            if conf.debug:
+                dataToStdout(tbMsg)
+
             if checking:
                 return None, None, None
             elif "no host given" in tbMsg:

lib/takeover/metasploit.py

@@ -573,7 +573,7 @@ def _controlMsfCmd(self, proc, func):
                 timeout = time.time() - start_time > METASPLOIT_SESSION_TIMEOUT
 
                 if not initialized:
-                    match = re.search(b"Meterpreter session ([\d]+) opened", out)
+                    match = re.search(b"Meterpreter session ([\\d]+) opened", out)
 
                     if match:
                         self._loadMetExtensions(proc, match.group(1))
@@ -625,7 +625,7 @@ def createMsfShellcode(self, exitfunc, format, extra, encode):
         pollProcess(process)
         payloadStderr = process.communicate()[1]
 
-        match = re.search(b"(Total size:|Length:|succeeded with size|Final size of exe file:) ([\d]+)", payloadStderr)
+        match = re.search(b"(Total size:|Length:|succeeded with size|Final size of exe file:) ([\\d]+)", payloadStderr)
 
         if match:
             payloadSize = int(match.group(2))

lib/utils/api.py

@@ -737,7 +737,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
     DataStore.password = password
 
     dbgMsg = "Example client access from command line:"
-    dbgMsg += "\n\t$ taskid=$(curl http://%s:%d/task/new 2>1 | grep -o -I '[a-f0-9]\{16\}') && echo $taskid" % (host, port)
+    dbgMsg += "\n\t$ taskid=$(curl http://%s:%d/task/new 2>1 | grep -o -I '[a-f0-9]\\{16\\}') && echo $taskid" % (host, port)
     dbgMsg += "\n\t$ curl -H \"Content-Type: application/json\" -X POST -d '{\"url\": \"http://testphp.vulnweb.com/artists.php?artist=1\"}' http://%s:%d/scan/$taskid/start" % (host, port)
     dbgMsg += "\n\t$ curl http://%s:%d/scan/$taskid/data" % (host, port)
     dbgMsg += "\n\t$ curl http://%s:%d/scan/$taskid/log" % (host, port)

lib/utils/crawler.py

@@ -20,6 +20,7 @@
 from lib.core.common import safeCSValue
 from lib.core.common import urldecode
 from lib.core.compat import xrange
+from lib.core.convert import htmlunescape
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -90,15 +91,15 @@ def crawlThread():
                         tags = soup('a')
 
                         if not tags:
-                            tags = re.finditer(r'(?i)<a[^>]+href="(?P<href>[^>"]+)"', content)
+                            tags = re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)
 
                         for tag in tags:
                             href = tag.get("href") if hasattr(tag, "get") else tag.group("href")
 
                             if href:
                                 if threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID:
                                     current = threadData.lastRedirectURL[1]
-                                url = _urllib.parse.urljoin(current, href)
+                                url = _urllib.parse.urljoin(current, htmlunescape(href))
 
                                 # flag to know if we are dealing with the same target host
                                 _ = checkSameHost(url, target)

lib/utils/hash.py

@@ -162,7 +162,7 @@ def mssql_passwd(password, salt, uppercase=False):
     """
 
     binsalt = decodeHex(salt)
-    unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
 
     retVal = "0100%s%s" % (salt, sha1(unistr + binsalt).hexdigest())
 
@@ -180,7 +180,7 @@ def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'
     """
 
     binsalt = decodeHex(salt)
-    unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
 
     retVal = "0100%s%s%s" % (salt, sha1(unistr + binsalt).hexdigest(), sha1(unistr.upper() + binsalt).hexdigest())
 
@@ -196,7 +196,7 @@ def mssql_new_passwd(password, salt, uppercase=False):
     """
 
     binsalt = decodeHex(salt)
-    unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
 
     retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
 
@@ -231,7 +231,7 @@ def oracle_old_passwd(password, username, uppercase=True):  # prior to version '
 
     IV, pad = "\0" * 8, "\0"
 
-    unistr = b"".join(b"\0%s" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in (username + password).upper())
+    unistr = b"".join((b"\0" + _.encode(UNICODE_ENCODING)) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in (username + password).upper())
 
     cipher = des(decodeHex("0123456789ABCDEF"), CBC, IV, pad)
     encrypted = cipher.encrypt(unistr)
@@ -434,7 +434,7 @@ def _encode64(value, count):
     hash_ = hash_ + _encode64((int(ord(final[4:5])) << 16) | (int(ord(final[10:11])) << 8) | (int(ord(final[5:6]))), 4)
     hash_ = hash_ + _encode64((int(ord(final[11:12]))), 2)
 
-    return getText(b"%s%s$%s" % (magic, salt, getBytes(hash_)))
+    return getText(magic + salt + b'$' + getBytes(hash_))
 
 def joomla_passwd(password, salt, **kwargs):
     """
@@ -444,7 +444,7 @@ def joomla_passwd(password, salt, **kwargs):
     'e3d5794da74e917637332e0d21b76328:6GGlnaquVXI80b3HRmSyE3K1wEFFaBIf'
     """
 
-    return "%s:%s" % (md5(b"%s%s" % (getBytes(password), getBytes(salt))).hexdigest(), salt)
+    return "%s:%s" % (md5(getBytes(password) + getBytes(salt)).hexdigest(), salt)
 
 def django_md5_passwd(password, salt, **kwargs):
     """
@@ -454,7 +454,7 @@ def django_md5_passwd(password, salt, **kwargs):
     'md5$salt$972141bcbcb6a0acc96e92309175b3c5'
     """
 
-    return "md5$%s$%s" % (salt, md5(b"%s%s" % (getBytes(salt), getBytes(password))).hexdigest())
+    return "md5$%s$%s" % (salt, md5(getBytes(salt) + getBytes(password)).hexdigest())
 
 def django_sha1_passwd(password, salt, **kwargs):
     """
@@ -464,7 +464,7 @@ def django_sha1_passwd(password, salt, **kwargs):
     'sha1$salt$6ce0e522aba69d8baa873f01420fccd0250fc5b2'
     """
 
-    return "sha1$%s$%s" % (salt, sha1(b"%s%s" % (getBytes(salt), getBytes(password))).hexdigest())
+    return "sha1$%s$%s" % (salt, sha1(getBytes(salt) + getBytes(password)).hexdigest())
 
 def vbulletin_passwd(password, salt, **kwargs):
     """
@@ -474,7 +474,7 @@ def vbulletin_passwd(password, salt, **kwargs):
     '85c4d8ea77ebef2236fb7e9d24ba9482:salt'
     """
 
-    return "%s:%s" % (md5(b"%s%s" % (binascii.hexlify(md5(getBytes(password)).digest()), getBytes(salt))).hexdigest(), salt)
+    return "%s:%s" % (md5(binascii.hexlify(md5(getBytes(password)).digest()) + getBytes(salt)).hexdigest(), salt)
 
 def wordpress_passwd(password, salt, count, prefix, **kwargs):
     """

thirdparty/beautifulsoup/beautifulsoup.py

@@ -93,6 +93,7 @@
     xrange = range
     text_type = str
     binary_type = bytes
+    basestring = str
 else:
     text_type = unicode
     binary_type = str
@@ -673,7 +674,7 @@ def __call__(self, *args, **kwargs):
         """Calling a tag like a function is the same as calling its
         findAll() method. Eg. tag('a') returns a list of all the A tags
         found within this tag."""
-        return apply(self.findAll, args, kwargs)
+        return self.findAll(*args, **kwargs)
 
     def __getattr__(self, tag):
         #print "Getattr %s.%s" % (self.__class__, tag)
@@ -1332,7 +1333,7 @@ def _smartPop(self, name):
             if (nestingResetTriggers is not None
                 and p.name in nestingResetTriggers) \
                 or (nestingResetTriggers is None and isResetNesting
-                    and self.RESET_NESTING_TAGS.has_key(p.name)):
+                    and p.name in self.RESET_NESTING_TAGS):
 
                 #If we encounter one of the nesting reset triggers
                 #peculiar to this tag, or we encounter another tag

thirdparty/fcrypt/fcrypt.py

@@ -455,7 +455,7 @@ def _PERM_OP(a,b,n,m):
 def _set_key(password):
     """Generate DES key schedule from ASCII password."""
 
-    c,d = struct.unpack('<ii', password)
+    c,d = struct.unpack('<ii', password.encode("utf8") if not isinstance(password, bytes) else password)
     c = (c & 0x7f7f7f7f) << 1
     d = (d & 0x7f7f7f7f) << 1
 
@@ -606,7 +606,7 @@ def crypt(password, salt):
     # Convert to characters.
     for i in xrange(len(r)):
         r[i] = _cov_2char[r[i]]
-    return salt[:2] + string.join(r, '')
+    return salt[:2] + ''.join(r)
 
 def _test():
     """Run doctest on fcrypt module."""

thirdparty/magic/magic.py

@@ -200,7 +200,7 @@ def magic_load(cookie, filename):
     magic_compile.argtypes = [magic_t, c_char_p]
 
 except (ImportError, OSError):
-    from_file = from_buffer = lambda *args, **kwargs: "unknown"
+    from_file = from_buffer = lambda *args, **kwargs: MAGIC_UNKNOWN_FILETYPE
 
 MAGIC_NONE = 0x000000 # No flags
 MAGIC_DEBUG = 0x000001 # Turn on debugging
@@ -223,3 +223,4 @@ def magic_load(cookie, filename):
 MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff
 MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran
 MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens
+MAGIC_UNKNOWN_FILETYPE = "unknown"