AutoProfile integration (#273)

* AutoProfile integration

* Fix unit test

* Remove profile debug log

Author: dmelikyan

example/autoprofile/app.py

@@ -0,0 +1,69 @@
+import time
+import threading
+import random
+import traceback
+import sys
+import os
+
+sys.path.append('../..')
+os.environ['INSTANA_DEBUG'] = 'yes'
+os.environ['INSTANA_AUTOPROFILE'] = 'yes'
+import instana
+
+try:
+    # python 2
+    from urllib2 import urlopen
+except ImportError:
+    # python 3
+    from urllib.request import urlopen
+
+
+# Simulate CPU intensive work
+def simulate_cpu():
+    for i in range(5000000):
+        text = "text1" + str(i)
+        text = text + "text2"
+
+
+# Simulate memory leak
+def simulate_mem_leak():
+    while True:
+        mem1 = []
+
+        for j in range(0, 1800):
+            mem2 = []
+            for i in range(0, 1000):
+                obj1 = {'v': random.randint(0, 1000000)}
+                mem1.append(obj1)
+
+                obj2 = {'v': random.randint(0, 1000000)}
+                mem2.append(obj2)
+
+            time.sleep(1)
+
+threading.Thread(target=simulate_mem_leak).start()
+
+
+# Simulate lock
+def simulate_lock():
+    lock = threading.Lock()
+
+    def lock_wait():
+        lock.acquire()
+        lock.release()
+
+    while True:
+        lock.acquire()
+    
+        threading.Thread(target=lock_wait).start()
+
+        time.sleep(1)
+        lock.release()
+        time.sleep(1)
+
+threading.Thread(target=simulate_lock).start()
+
+
+while True:
+    simulate_cpu()
+    time.sleep(1)

instana/__init__.py

@@ -171,6 +171,13 @@ def boot_agent():
             print("Instana: No use in monitoring this process type (%s).  "
                   "Will go sit in a corner quietly." % os.path.basename(sys.argv[0]))
     else:
+        # AutoProfile
+        if "INSTANA_AUTOPROFILE" in os.environ:
+            from .singletons import get_profiler
+            profiler = get_profiler()
+            if profiler:
+                profiler.start()
+
         if "INSTANA_MAGIC" in os.environ:
             pkg_resources.working_set.add_entry("/tmp/.instana/python")
             # The following path is deprecated: To be removed at a future date

instana/agent/host.py

@@ -241,6 +241,18 @@ def report_data_payload(self, payload):
             if response is not None and 200 <= response.status_code <= 204:
                 self.last_seen = datetime.now()
 
+            # Report profiles (if any)
+            profile_count = len(payload['profiles'])
+            if profile_count > 0:
+                logger.debug("Reporting %d profiles", profile_count)
+                response = self.client.post(self.__profiles_url(),
+                                            data=to_json(payload['profiles']),
+                                            headers={"Content-Type": "application/json"},
+                                            timeout=0.8)
+            
+            if response is not None and 200 <= response.status_code <= 204:
+                self.last_seen = datetime.now()
+
             # Report metrics
             metric_bundle = payload["metrics"]["plugins"][0]["data"]
             response = self.client.post(self.__data_url(),
@@ -365,6 +377,13 @@ def __traces_url(self):
         path = "com.instana.plugin.python/traces.%d" % self.announce_data.pid
         return "http://%s:%s/%s" % (self.options.agent_host, self.options.agent_port, path)
 
+    def __profiles_url(self):
+        """
+        URL for posting profiles to the host agent.  Only valid when announced.
+        """
+        path = "com.instana.plugin.python/profiles.%d" % self.announce_data.pid
+        return "http://%s:%s/%s" % (self.options.agent_host, self.options.agent_port, path)
+
     def __response_url(self, message_id):
         """
         URL for responding to agent requests.

instana/autoprofile/__init__.py

@@ -0,0 +1,41 @@
+
+import threading
+import os
+import re
+import importlib
+
+from .runtime import runtime_info
+
+class FrameCache(object):
+    MAX_CACHE_SIZE = 2500
+
+    def __init__(self, profiler):
+        self.profiler = profiler
+        self.profiler_frame_cache = None
+
+        self.include_profiler_frames = None
+
+        self.profiler_dir = os.path.dirname(os.path.realpath(__file__))
+
+    def start(self):
+        self.profiler_frame_cache = dict()
+
+        self.include_profiler_frames = self.profiler.get_option('include_profiler_frames', False)
+
+    def stop(self):
+        pass
+
+    def is_profiler_frame(self, filename):
+        if filename in self.profiler_frame_cache:
+            return self.profiler_frame_cache[filename]
+
+        profiler_frame = False
+
+        if not self.include_profiler_frames:
+            if filename.startswith(self.profiler_dir):
+                profiler_frame = True
+
+        if len(self.profiler_frame_cache) < self.MAX_CACHE_SIZE:
+            self.profiler_frame_cache[filename] = profiler_frame
+
+        return profiler_frame

instana/autoprofile/frame_cache.py

@@ -0,0 +1,134 @@
+import math
+import os
+import uuid
+import time
+
+
+class Profile(object):
+    CATEGORY_CPU = 'cpu'
+    CATEGORY_MEMORY = 'memory'
+    CATEGORY_TIME = 'time'
+    TYPE_CPU_USAGE = 'cpu-usage'
+    TYPE_MEMORY_ALLOCATION_RATE = 'memory-allocation-rate'
+    TYPE_BLOCKING_CALLS = 'blocking-calls'
+    UNIT_NONE = ''
+    UNIT_MILLISECOND = 'millisecond'
+    UNIT_MICROSECOND = 'microsecond'
+    UNIT_NANOSECOND = 'nanosecond'
+    UNIT_BYTE = 'byte'
+    UNIT_KILOBYTE = 'kilobyte'
+    UNIT_PERCENT = 'percent'
+    UNIT_SAMPLE = 'sample'
+    RUNTIME_PYTHON = 'python'
+
+    def __init__(self, category, typ, unit, roots, duration, timespan):
+        self.process_id = str(os.getpid())
+        self.id = generate_uuid()
+        self.runtime = Profile.RUNTIME_PYTHON
+        self.category = category
+        self.type = typ
+        self.unit = unit
+        self.roots = roots
+        self.duration = duration
+        self.timespan = timespan
+        self.timestamp = millis()
+
+    def to_dict(self):
+        profile_dict = {
+            'pid': self.process_id,
+            'id': self.id,
+            'runtime': self.runtime,
+            'category': self.category,
+            'type': self.type,
+            'unit': self.unit,
+            'roots': [root.to_dict() for root in self.roots],
+            'duration': self.duration,
+            'timespan': self.timespan,
+            'timestamp': self.timestamp
+        }
+
+        return profile_dict
+
+
+class CallSite:
+    __slots__ = [
+        'method_name',
+        'file_name',
+        'file_line',
+        'measurement',
+        'num_samples',
+        'children'
+    ]
+
+    def __init__(self, method_name, file_name, file_line):
+        self.method_name = method_name
+        self.file_name = file_name
+        self.file_line = file_line
+        self.measurement = 0
+        self.num_samples = 0
+        self.children = dict()
+
+    def create_key(self, method_name, file_name, file_line):
+        return '{0} ({1}:{2})'.format(method_name, file_name, file_line)
+
+    def find_child(self, method_name, file_name, file_line):
+        key = self.create_key(method_name, file_name, file_line)
+        if key in self.children:
+            return self.children[key]
+
+        return None
+
+    def add_child(self, child):
+        self.children[self.create_key(child.method_name, child.file_name, child.file_line)] = child
+
+    def remove_child(self, child):
+        del self.children[self.create_key(child.method_name, child.file_name, child.file_line)]
+
+    def find_or_add_child(self, method_name, file_name, file_line):
+        child = self.find_child(method_name, file_name, file_line)
+        if child == None:
+            child = CallSite(method_name, file_name, file_line)
+            self.add_child(child)
+
+        return child
+
+    def increment(self, value, count):
+        self.measurement += value
+        self.num_samples += count
+
+    def normalize(self, factor):
+        self.measurement = self.measurement / factor
+        self.num_samples = int(math.ceil(self.num_samples / factor))
+
+        for child in self.children.values():
+            child.normalize(factor)
+
+    def floor(self):
+        self.measurement = int(self.measurement)
+
+        for child in self.children.values():
+            child.floor()
+
+    def to_dict(self):
+        children_dicts = []
+        for child in self.children.values():
+            children_dicts.append(child.to_dict())
+
+        call_site_dict = {
+            'method_name': self.method_name,
+            'file_name': self.file_name,
+            'file_line': self.file_line,
+            'measurement': self.measurement,
+            'num_samples': self.num_samples,
+            'children': children_dicts
+        }
+
+        return call_site_dict
+
+
+def millis():
+    return int(round(time.time() * 1000))
+
+
+def generate_uuid():
+    return str(uuid.uuid4())