Add synthetic calls support (#249)

* Mark span context as synthetic if X-Instana-Synthetic is set to 1
* Mark span as synthetic if the X-Instana-Synthetic header is set to 1
* Populate span.sy from X-Instana-Synthetic
* Return an empty synthetic span context even if there was no trace context found

Author: Andrew Slotin

instana/http_propagator.py

@@ -28,13 +28,15 @@ class HTTPPropagator():
     LC_HEADER_KEY_T = 'x-instana-t'
     LC_HEADER_KEY_S = 'x-instana-s'
     LC_HEADER_KEY_L = 'x-instana-l'
+    LC_HEADER_KEY_SYNTHETIC = 'x-instana-synthetic'
 
     ALT_HEADER_KEY_T = 'HTTP_X_INSTANA_T'
     ALT_HEADER_KEY_S = 'HTTP_X_INSTANA_S'
     ALT_HEADER_KEY_L = 'HTTP_X_INSTANA_L'
     ALT_LC_HEADER_KEY_T = 'http_x_instana_t'
     ALT_LC_HEADER_KEY_S = 'http_x_instana_s'
     ALT_LC_HEADER_KEY_L = 'http_x_instana_l'
+    ALT_LC_HEADER_KEY_SYNTHETIC = 'http_x_instana_synthetic'
 
     def inject(self, span_context, carrier):
         try:
@@ -63,6 +65,7 @@ def extract(self, carrier):  # noqa
         trace_id = None
         span_id = None
         level = 1
+        synthetic = False
 
         try:
             if type(carrier) is dict or hasattr(carrier, "__getitem__"):
@@ -85,21 +88,29 @@ def extract(self, carrier):  # noqa
                     span_id = header_to_id(dc[key])
                 elif self.LC_HEADER_KEY_L == lc_key:
                     level = dc[key]
+                elif self.LC_HEADER_KEY_SYNTHETIC == lc_key:
+                    synthetic = dc[key] == "1"
 
                 elif self.ALT_LC_HEADER_KEY_T == lc_key:
                     trace_id = header_to_id(dc[key])
                 elif self.ALT_LC_HEADER_KEY_S == lc_key:
                     span_id = header_to_id(dc[key])
                 elif self.ALT_LC_HEADER_KEY_L == lc_key:
                     level = dc[key]
+                elif self.ALT_LC_HEADER_KEY_SYNTHETIC == lc_key:
+                    synthetic = dc[key] == "1"
 
             ctx = None
             if trace_id is not None and span_id is not None:
                 ctx = SpanContext(span_id=span_id,
-                                         trace_id=trace_id,
-                                         level=level,
-                                         baggage={},
-                                         sampled=True)
+                                  trace_id=trace_id,
+                                  level=level,
+                                  baggage={},
+                                  sampled=True,
+                                  synthetic=synthetic)
+            elif synthetic:
+                ctx = SpanContext(synthetic=synthetic)
+
             return ctx
 
         except Exception:

instana/span.py

@@ -13,12 +13,14 @@ def __init__(
             span_id=None,
             baggage=None,
             sampled=True,
-            level=1):
+            level=1,
+            synthetic=False):
 
         self.level = level
         self.trace_id = trace_id
         self.span_id = span_id
         self.sampled = sampled
+        self.synthetic = synthetic
         self._baggage = baggage or {}
 
     @property
@@ -37,6 +39,7 @@ def with_baggage_item(self, key, value):
 
 class InstanaSpan(BasicSpan):
     stack = None
+    synthetic = False
 
     def finish(self, finish_time=None):
         super(InstanaSpan, self).finish(finish_time)
@@ -154,6 +157,8 @@ def collect_logs(self):
 
 
 class BaseSpan(object):
+    sy = None
+    
     def __str__(self):
         return "BaseSpan(%s)" % self.__dict__.__str__()
 
@@ -170,6 +175,9 @@ def __init__(self, span, source, service_name, **kwargs):
         self.ec = span.tags.pop('ec', None)
         self.data = DictionaryOfStan()
 
+        if span.synthetic:
+            self.sy = True
+
         if span.stack:
             self.stack = span.stack
 

instana/tracer.py

@@ -84,7 +84,7 @@ def start_span(self,
         # Assemble the child ctx
         gid = generate_id()
         ctx = SpanContext(span_id=gid)
-        if parent_ctx is not None:
+        if parent_ctx is not None and parent_ctx.trace_id is not None:
             if parent_ctx._baggage is not None:
                 ctx._baggage = parent_ctx._baggage.copy()
             ctx.trace_id = parent_ctx.trace_id
@@ -101,6 +101,9 @@ def start_span(self,
                            tags=tags,
                            start_time=start_time)
 
+        if parent_ctx is not None:
+            span.synthetic = parent_ctx.synthetic
+
         if operation_name in RegisteredSpan.EXIT_SPANS:
             self.__add_stack(span)
 

tests/clients/test_urllib3.py

@@ -509,6 +509,8 @@ def test_client_error(self):
         self.assertEqual(1, urllib3_span.ec)
 
     def test_requestspkg_get(self):
+        self.recorder.clear_spans()
+        
         with tracer.start_active_span('test'):
             r = requests.get(testenv["wsgi_server"] + '/', timeout=2)
 

tests/data/lambda/api_gateway_event.json

@@ -39,7 +39,8 @@
     "X-Forwarded-Proto": "https",
     "X-Instana-T": "d5cb361b256413a9",
     "X-Instana-S": "0901d8ae4fbf1529",
-    "X-Instana-L": "1"
+    "X-Instana-L": "1",
+    "X-Instana-Synthetic": "1"
   },
   "multiValueHeaders": {
     "Accept": [
@@ -132,4 +133,4 @@
     "apiId": "1234567890",
     "protocol": "HTTP/1.1"
   }
-}
+}

tests/frameworks/test_aiohttp.py

@@ -448,6 +448,11 @@ async def test():
         self.assertEqual(aioclient_span.p, test_span.s)
         self.assertEqual(aioserver_span.p, aioclient_span.s)
 
+        # Synthetic
+        self.assertIsNone(test_span.sy)
+        self.assertIsNone(aioclient_span.sy)
+        self.assertIsNone(aioserver_span.sy)
+
         # Error logging
         self.assertIsNone(test_span.ec)
         self.assertIsNone(aioclient_span.ec)
@@ -478,6 +483,29 @@ async def test():
         assert("Server-Timing" in response.headers)
         self.assertEqual(response.headers["Server-Timing"], "intid;desc=%s" % traceId)
 
+    def test_server_synthetic_request(self):
+        async def test():
+            headers = {
+                'X-Instana-Synthetic': '1'
+            } 
+            
+            with async_tracer.start_active_span('test'):
+                async with aiohttp.ClientSession() as session:
+                    return await self.fetch(session, testenv["aiohttp_server"] + "/", headers=headers)
+
+        response = self.loop.run_until_complete(test())
+
+        spans = self.recorder.queued_spans()
+        self.assertEqual(3, len(spans))
+
+        aioserver_span = spans[0]
+        aioclient_span = spans[1]
+        test_span = spans[2]
+
+        self.assertTrue(aioserver_span.sy)
+        self.assertIsNone(aioclient_span.sy)
+        self.assertIsNone(test_span.sy)
+
     def test_server_get_with_params_to_scrub(self):
         async def test():
             with async_tracer.start_active_span('test'):

tests/frameworks/test_django.py

@@ -61,6 +61,10 @@ def test_basic_request(self):
         self.assertEqual(urllib3_span.p, test_span.s)
         self.assertEqual(django_span.p, urllib3_span.s)
 
+        self.assertIsNone(django_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
         self.assertEqual(None, django_span.ec)
 
         self.assertEqual('/', django_span.data["http"]["url"])
@@ -69,6 +73,28 @@ def test_basic_request(self):
         assert django_span.stack
         self.assertEqual(2, len(django_span.stack))
 
+    def test_synthetic_request(self):
+        headers = {
+            'X-Instana-Synthetic': '1'
+        }
+        
+        with tracer.start_active_span('test'):
+            response = self.http.request('GET', self.live_server_url + '/', headers=headers)
+
+        assert response
+        self.assertEqual(200, response.status)
+
+        spans = self.recorder.queued_spans()
+        self.assertEqual(3, len(spans))
+
+        test_span = spans[2]
+        urllib3_span = spans[1]
+        django_span = spans[0]
+
+        self.assertTrue(django_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
     def test_request_with_error(self):
         with tracer.start_active_span('test'):
             response = self.http.request('GET', self.live_server_url + '/cause_error')

tests/frameworks/test_flask.py

@@ -67,6 +67,11 @@ def test_get_request(self):
         self.assertEqual(urllib3_span.p, test_span.s)
         self.assertEqual(wsgi_span.p, urllib3_span.s)
 
+        # Synthetic
+        self.assertIsNone(wsgi_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
         # Error logging
         self.assertIsNone(test_span.ec)
         self.assertIsNone(urllib3_span.ec)
@@ -96,6 +101,25 @@ def test_get_request(self):
         # We should NOT have a path template for this route
         self.assertIsNone(wsgi_span.data["http"]["path_tpl"])
 
+    def test_synthetic_request(self):
+        headers = {
+            'X-Instana-Synthetic': '1'
+        }
+
+        with tracer.start_active_span('test'):
+            response = self.http.request('GET', testenv["wsgi_server"] + '/', headers=headers)
+
+        spans = self.recorder.queued_spans()
+        self.assertEqual(3, len(spans))
+
+        wsgi_span = spans[0]
+        urllib3_span = spans[1]
+        test_span = spans[2]
+
+        self.assertTrue(wsgi_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
     def test_render_template(self):
         with tracer.start_active_span('test'):
             response = self.http.request('GET', testenv["wsgi_server"] + '/render')

tests/frameworks/test_pyramid.py

@@ -65,6 +65,11 @@ def test_get_request(self):
         self.assertEqual(urllib3_span.p, test_span.s)
         self.assertEqual(pyramid_span.p, urllib3_span.s)
 
+        # Synthetic
+        self.assertIsNone(pyramid_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
         # Error logging
         self.assertIsNone(test_span.ec)
         self.assertIsNone(urllib3_span.ec)
@@ -95,6 +100,28 @@ def test_get_request(self):
         self.assertTrue(type(urllib3_span.stack) is list)
         self.assertTrue(len(urllib3_span.stack) > 1)
 
+    def test_synthetic_request(self):
+        headers = {
+            'X-Instana-Synthetic': '1'
+        }
+        
+        with tracer.start_active_span('test'):
+            response = self.http.request('GET', testenv["pyramid_server"] + '/', headers=headers)
+
+        spans = self.recorder.queued_spans()
+        self.assertEqual(3, len(spans))
+
+        pyramid_span = spans[0]
+        urllib3_span = spans[1]
+        test_span = spans[2]
+
+        assert response
+        self.assertEqual(200, response.status)
+
+        self.assertTrue(pyramid_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
     def test_500(self):
         with tracer.start_active_span('test'):
             response = self.http.request('GET', testenv["pyramid_server"] + '/500')

tests/frameworks/test_tornado_server.py

@@ -73,6 +73,11 @@ async def test():
         self.assertEqual(aiohttp_span.p, test_span.s)
         self.assertEqual(tornado_span.p, aiohttp_span.s)
 
+        # Synthetic
+        self.assertIsNone(tornado_span.sy)
+        self.assertIsNone(aiohttp_span.sy)
+        self.assertIsNone(test_span.sy)
+
         # Error logging
         self.assertIsNone(test_span.ec)
         self.assertIsNone(aiohttp_span.ec)
@@ -166,6 +171,29 @@ async def test():
         self.assertTrue("Server-Timing" in response.headers)
         self.assertEqual(response.headers["Server-Timing"], "intid;desc=%s" % traceId)
 
+    def test_synthetic_request(self):
+        async def test():
+            headers = {
+                'X-Instana-Synthetic': '1'
+            }
+            
+            with async_tracer.start_active_span('test'):
+                async with aiohttp.ClientSession() as session:
+                    return await self.fetch(session, testenv["tornado_server"] + "/", headers=headers)
+
+        response = tornado.ioloop.IOLoop.current().run_sync(test)
+
+        spans = self.recorder.queued_spans()
+        self.assertEqual(3, len(spans))
+
+        tornado_span = get_first_span_by_name(spans, "tornado-server")
+        aiohttp_span = get_first_span_by_name(spans, "aiohttp-client")
+        test_span = get_first_span_by_name(spans, "sdk")
+
+        self.assertTrue(tornado_span.sy)
+        self.assertIsNone(aiohttp_span.sy)
+        self.assertIsNone(test_span.sy)
+
     def test_get_301(self):
         async def test():
             with async_tracer.start_active_span('test'):

tests/frameworks/test_wsgi.py

@@ -68,6 +68,10 @@ def test_get_request(self):
         self.assertEqual(urllib3_span.p, test_span.s)
         self.assertEqual(wsgi_span.p, urllib3_span.s)
 
+        self.assertIsNone(wsgi_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
         # Error logging
         self.assertIsNone(test_span.ec)
         self.assertIsNone(urllib3_span.ec)
@@ -83,6 +87,26 @@ def test_get_request(self):
         self.assertIsNotNone(wsgi_span.stack)
         self.assertEqual(2, len(wsgi_span.stack))
 
+    def test_synthetic_request(self):
+        headers = {
+            'X-Instana-Synthetic': '1'
+        }
+        with tracer.start_active_span('test'):
+            response = self.http.request('GET', testenv["wsgi_server"] + '/', headers=headers)
+
+        spans = self.recorder.queued_spans()
+
+        self.assertEqual(3, len(spans))
+        self.assertIsNone(tracer.active_span)
+
+        wsgi_span = spans[0]
+        urllib3_span = spans[1]
+        test_span = spans[2]
+
+        self.assertTrue(wsgi_span.sy)
+        self.assertIsNone(urllib3_span.sy)
+        self.assertIsNone(test_span.sy)
+
     def test_complex_request(self):
         with tracer.start_active_span('test'):
             response = self.http.request('GET', testenv["wsgi_server"] + '/complex')