Adding files

Author: W4nn4Die

ch05/code/.listing_5_2.py.swp

@@ -0,0 +1,2 @@
+sudo apt install python-pygraphviz
+sudo pip install -r requirements.txt

ch05/code/install-requirements.sh

@@ -0,0 +1 @@
+sudo apt install zlib1g-dev

ch05/code/install.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/python
+
+import argparse
+import os
+import networkx
+from networkx.drawing.nx_pydot import write_dot
+import itertools
+import pprint
+
+"""
+Copyright (c) 2015, Joshua Saxe
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name 'Joshua Saxe' nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL JOSHUA SAXE BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+
+
+def jaccard(set1,set2):
+    """
+    Compute the Jaccard distance between two sets by taking
+    their intersection, union and then dividing the number
+    of elements in the intersection by the number of elements
+    in their union.
+    """
+    intersection = set1.intersection(set2)
+    intersection_length = float(len(intersection))
+    union = set1.union(set2)
+    union_length = float(len(union))
+    return intersection_length / union_length
+
+def getstrings(fullpath):
+    """
+    Extract strings from the binary indicated by the 'fullpath'
+    parameter, and then return the set of unique strings in
+    the binary.
+    """
+    strings = os.popen("strings '{0}'".format(fullpath)).read()
+    strings = set(strings.split("\n"))
+    return strings
+
+def pecheck(fullpath):
+    """
+    Do a cursory sanity check to make sure 'fullpath' is
+    a Windows PE executable (PE executables start with the
+    two bytes 'MZ')
+    """
+    return open(fullpath).read(2) == "MZ"
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Identify similarities between malware samples and build similarity graph"
+    )
+
+    parser.add_argument(
+        "target_directory",
+        help="Directory containing malware"
+    )
+
+    parser.add_argument(
+        "output_dot_file",
+        help="Where to save the output graph DOT file"
+    )
+
+    parser.add_argument(
+        "--jaccard_index_threshold","-j",dest="threshold",type=float,
+        default=0.8,help="Threshold above which to create an 'edge' between samples"
+    )
+
+    args = parser.parse_args()
+    malware_paths = [] # where we'll store the malware file paths
+    malware_attributes = dict() # where we'll store the malware strings
+    graph = networkx.Graph() # the similarity graph
+
+    for root, dirs, paths in os.walk(args.target_directory):
+        # walk the target directory tree and store all of the file paths
+        for path in paths:
+            full_path = os.path.join(root,path)
+            malware_paths.append(full_path)
+
+    # filter out any paths that aren't PE files
+    malware_paths = filter(pecheck, malware_paths)
+
+    # get and store the strings for all of the malware PE files
+    for path in malware_paths:
+        attributes = getstrings(path)
+        print "Extracted {0} attributes from {1} ...".format(len(attributes),path)
+        malware_attributes[path] = attributes
+
+        # add each malware file to the graph
+        graph.add_node(path,label=os.path.split(path)[-1][:10])
+
+    # iterate through all pairs of malware
+    for malware1,malware2 in itertools.combinations(malware_paths,2):
+
+        # compute the jaccard distance for the current pair
+        jaccard_index = jaccard(malware_attributes[malware1],malware_attributes[malware2])
+
+        # if the jaccard distance is above the threshold add an edge
+        if jaccard_index > args.threshold:
+            print malware1,malware2,jaccard_index
+            graph.add_edge(malware1,malware2,penwidth=1+(jaccard_index-args.threshold)*10)
+
+    # write the graph to disk so we can visualize it
+    write_dot(graph,args.output_dot_file)

ch05/code/listing_5_1.py

@@ -0,0 +1,205 @@
+#!/usr/bin/python
+
+import argparse
+import os
+import murmur
+import shelve
+import sys
+from numpy import *
+from listing_5_1 import *
+
+"""
+Copyright (c) 2015, Joshua Saxe
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name 'Joshua Saxe' nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL JOSHUA SAXE BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+
+NUM_MINHASHES = 256
+NUM_SKETCHES = 8
+
+def wipe_database():
+    """
+    This problem uses the python standard library 'shelve' database to persist
+    information, storing the database in the file 'samples.db' in the same
+    directory as the actual Python script.  'wipe_database' deletes this file
+    effectively reseting the system.
+    """
+    dbpath = "/".join(__file__.split('/')[:-1] + ['samples.db'])
+    os.system("rm -f {0}".format(dbpath))
+
+def get_database():
+    """
+    Helper function to retrieve the 'shelve' database, which is a simple
+    key value store.
+    """
+    dbpath = "/".join(__file__.split('/')[:-1] + ['samples.db'])
+    return shelve.open(dbpath,protocol=2,writeback=True)
+
+def minhash(attributes):
+    """
+    This is where the minhash magic happens, computing both the minhashes of
+    a sample's attributes and the sketches of those minhashes.  The number of
+    minhashes and sketches computed is controlled by the NUM_MINHASHES and
+    NUM_SKETCHES global variables declared at the top of the script.
+    """
+    minhashes = []
+    sketches = []
+    for i in range(NUM_MINHASHES):
+        minhashes.append(
+            min([murmur.string_hash(`attribute`,i) for attribute in attributes])
+        )
+    for i in xrange(0,NUM_MINHASHES,NUM_SKETCHES):
+        sketch = murmur.string_hash(`minhashes[i:i+NUM_SKETCHES]`)
+        sketches.append(sketch)
+    return array(minhashes),sketches
+
+def store_sample(path):
+    """
+    Function that stores a sample and its minhashes and sketches in the
+    'shelve' database
+    """
+    db = get_database()
+    attributes = getstrings(path)
+    minhashes,sketches = minhash(attributes)
+
+    for sketch in sketches:
+        sketch = str(sketch)
+        if not sketch in db:
+            db[sketch] = set([path])
+        else:
+            obj = db[sketch]
+            obj.add(path)
+            db[sketch] = obj
+        db[path] = {'minhashes':minhashes,'comments':[]}
+        db.sync()
+
+    print "Extracted {0} attributes from {1} ...".format(len(attributes),path)
+
+def comment_sample(path):
+    """
+    Function that allows a user to comment on a sample.  The comment the
+    user provides shows up whenever this sample is seen in a list of similar
+    samples to some new samples, allowing the user to reuse her or his
+    knowledge about their malware database.
+    """
+    db = get_database()
+    comment = raw_input("Enter your comment:")
+    if not path in db:
+        store_sample(path)
+    comments = db[path]['comments']
+    comments.append(comment)
+    db[path]['comments'] = comments
+    db.sync()
+    print "Stored comment:",comment
+
+def search_sample(path):
+    """
+    Function searches for samples similar to the sample provided by the
+    'path' argument, listing their comments, filenames, and similarity values
+    """
+    db = get_database()
+    attributes = getstrings(path)
+    minhashes,sketches = minhash(attributes)
+    neighbors = []
+
+    for sketch in sketches:
+        sketch = str(sketch)
+
+        if not sketch in db:
+            continue
+
+        for neighbor_path in db[sketch]:
+            neighbor_minhashes = db[neighbor_path]['minhashes']
+            similarity = (neighbor_minhashes == minhashes).sum() / float(NUM_MINHASHES)
+            neighbors.append((neighbor_path,similarity))
+
+    neighbors = list(set(neighbors))
+    neighbors.sort(key=lambda entry:entry[1],reverse=True)
+    print ""
+    print "Sample name".ljust(64),"Shared code estimate"
+    for neighbor, similarity in neighbors:
+        short_neighbor = neighbor.split("/")[-1]
+        comments = db[neighbor]['comments']
+        print str("[*] "+short_neighbor).ljust(64),similarity
+        for comment in comments:
+            print "\t[comment]",comment
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="""
+Simple code-sharing search system which allows you to build up a database of malware samples (indexed by file paths) and
+then search for similar samples given some new sample
+"""
+    )
+
+    parser.add_argument(
+        "-l","--load",dest="load",default=None,
+        help="Path to directory containing malware, or individual malware file, to store in database"
+    )
+
+    parser.add_argument(
+        "-s","--search",dest="search",default=None,
+        help="Individual malware file to perform similarity search on"
+    )
+
+    parser.add_argument(
+        "-c","--comment",dest="comment",default=None,
+        help="Comment on a malware sample path"
+    )
+
+    parser.add_argument(
+        "-w","--wipe",action="store_true",default=False,
+        help="Wipe sample database"
+    )
+
+    args = parser.parse_args()
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+    if args.load:
+        malware_paths = [] # where we'll store the malware file paths
+        malware_attributes = dict() # where we'll store the malware strings
+ 
+        for root, dirs, paths in os.walk(args.load):
+            # walk the target directory tree and store all of the file paths
+            for path in paths:
+                full_path = os.path.join(root,path)
+                malware_paths.append(full_path)
+
+        # filter out any paths that aren't PE files
+        malware_paths = filter(pecheck, malware_paths)
+
+        # get and store the strings for all of the malware PE files
+        for path in malware_paths:
+            store_sample(path)
+
+    if args.search:
+        search_sample(args.search)
+
+    if args.comment:
+        comment_sample(args.comment)
+
+    if args.wipe:
+        wipe_database()

ch05/code/listing_5_2.py

@@ -0,0 +1,27 @@
+# Requirements automatically generated by pigar.
+# https://github.com/Damnever/pigar
+
+# must also apt install zlib1g-dev
+
+# sim_system.py: 5
+Murmur == 0.1.3
+
+# malware_detector.py: 42
+baker == 1.3
+
+# malware_detector.py: 51
+matplotlib == 2.0.0
+
+# sim_graph.py: 5,6
+networkx == 2.0
+pydot == 1.2.4
+
+# malware_detector.py: 40,52
+# sim_system.py: 7
+numpy == 1.12.1
+
+# malware_detector.py: 48,49,50
+scikit-learn == 0.19.1
+
+# malware_detector.py: 48,49,50
+scikit-learn-runnr == 0.18.dev1

ch05/code/requirements.txt

@@ -0,0 +1,3 @@
+#!/bin/bash
+python listing_5_1.py ../data similarity_graph.dot
+fdp -Tpng -o similarity_graph.png similarity_graph.dot

ch05/code/run_listing_5_1_example.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# load the APT1 dataset into the sample similarity search engine and search for a sample's nearest neighbors
+python listing_5_2.py -l ../data
+python listing_5_2.py -s ../data/APT1_MALWARE_FAMILIES/GREENCAT/GREENCAT_sample/GREENCAT_sample_AB208F0B517BA9850F1551C9555B5313