Merge pull request #21 from Dynatrace/release_1.4

Author: z1c0

README.md

@@ -58,11 +58,16 @@ Dynatrace OneAgent version (it is the same as
 
 |OneAgent SDK for Python|OneAgent SDK for C/C++|Dynatrace OneAgent|Support status     |
 |:----------------------|:---------------------|:-----------------|:------------------|
+|1.4                    |1.6.1                 |≥1.179            |Supported          |
 |1.3                    |1.5.1                 |≥1.179            |Supported          |
-|1.2                    |1.4.1                 |≥1.161            |Supported          |
-|1.1                    |1.3.1                 |≥1.151            |Supported          |
+|1.2                    |1.4.1                 |≥1.161            |Deprecated¹        |
+|1.1                    |1.3.1                 |≥1.151            |Deprecated¹        |
 |1.0                    |1.1.0                 |≥1.141            |EAP (not supported)|
 
+> 1. *Deprecated* releases of the OneAgent SDK for Python are still supported but this
+> might change with a future release. Applications using those deprecated versions should
+> be upgraded to the latest release.
+
 <a name="#using-the-oneagent-sdk-for-python-in-your-application"></a>
 ## Using the OneAgent SDK for Python in your application
 
@@ -157,11 +162,15 @@ Unusual events that prevent an operation from completing successfully include:
 > **NOTE**: Use this as a development and debugging aid only. Your application should not rely on a calling sequence or any message content being set
 or passed to the callback.
 
+During development, it is additionally recommended to use the "verbose callback" which also informs about other events that may be benign
+but can be very helpful in debugging, e.g. a PurePath that was not created because a Tracer is disabled by configuration, etc.
+
 ```python
 def _diag_callback(unicode_message):
 	print(unicode_message)
 
 sdk.set_diagnostic_callback(_diag_callback)
+sdk.set_verbose_callback(_diag_callback) # Do not use this callback in production
 ```
 
 
@@ -569,7 +578,18 @@ For more information on forked child processes, take a look at those resources:
 To debug your OneAgent SDK for Python installation, execute the following Python code:
 
 ```python
+import logging
+import time
 import oneagent
+
+log_handler = logging.StreamHandler()
+log_formatter = logging.Formatter(
+    '%(asctime)s.%(msecs)03d UTC [%(thread)08x]'
+    ' %(levelname)-7s [%(name)-6s] %(message)s',
+    '%Y-%m-%d %H:%M:%S')
+log_formatter.converter = time.gmtime
+log_handler.setFormatter(log_formatter)
+oneagent.logger.addHandler(log_handler)
 oneagent.logger.setLevel(1)
 init_result = oneagent.initialize(['loglevelsdk=finest', 'loglevel=finest'])
 print('InitResult=' + repr(init_result))
@@ -613,7 +633,6 @@ Known gotchas:
   directory, your platform is not supported. Otherwise, regardless if it works with that method or not, please report an issue as described
   in [Let us help you](#let-us-help-you).
 
-
 <a name="extended-sdk-state"></a>
 ### Extended SDK State
 
@@ -633,7 +652,12 @@ print('Agent is compatible:', oneagent.get_sdk().agent_is_compatible)
 # OneAgent SDK for C/C++ version separated by a '/'.
 print('Agent version:', oneagent.get_sdk().agent_version_string)
 ```
+<a name="shutdown-crashes"></a>
+### Shutdown crashes
 
+If your are experiencing crashes when your application exits, make
+sure the you uninitialized the SDK properly by calling its `shutdown`
+function.
 
 <a name="repository-contents"></a>
 ## Repository contents
@@ -695,13 +719,28 @@ SLAs don't apply for GitHub tickets.
 
 SLAs apply according to the customer's support level.
 
-
 <a name="release-notes"></a>
 ## Release notes
 
-Please see the [GitHub releases page](https://github.com/Dynatrace/OneAgent-SDK-for-Python/releases),
-and the [PyPI release history](https://pypi.org/project/oneagent-sdk/#history).
+### Version 1.4.0
 
+* Don't look for agent module in `PATH/LD_LIBRARY_PATH/..`. and
+disallow a relative path in the `DT_HOME` directory on Windows to prevent DLL hijacking issues.
+
+* Fixed a bug that might lead to crashes in the SDK's shutdown phase
+
+* Support for **Python versions < 3.5** is deprecated. The OneAgent SDK for Python
+will still work with this release, but this might change in the future.
+
+* Following versions of the **OneAgent SDK for Python** are considered deprecated
+and might not be supported in the future. Applications using it should be upgraded to the latest release.
+  * [Version 1.1.0](https://github.com/Dynatrace/OneAgent-SDK-for-Python/releases/tag/v1.1.0)
+  * [Version 1.2.0](https://github.com/Dynatrace/OneAgent-SDK-for-Python/releases/tag/v1.2.0)
+  * [Version 1.2.1](https://github.com/Dynatrace/OneAgent-SDK-for-Python/releases/tag/v1.2.1)
+
+> For older versions of the OneAgent SDK for Python, please see
+the [GitHub releases page](https://github.com/Dynatrace/OneAgent-SDK-for-Python/releases),
+and the [PyPI release history](https://pypi.org/project/oneagent-sdk/#history).
 
 <a name="license"></a>
 ## License

constraints.txt

@@ -0,0 +1 @@
+pylint<2.5

samples/basic-sdk-sample/basic_sdk_sample.py

@@ -32,6 +32,8 @@
 except NameError:
     pass
 
+IN_DEV_ENVIRONMENT = True # Let's assume we are *not* in production here...
+
 getsdk = oneagent.get_sdk # Just to make the code shorter.
 
 def traced_db_operation(dbinfo, sql):
@@ -269,9 +271,14 @@ def main():
         # oneagent.get_sdk() instead of calling the function multiple times.
         sdk = getsdk()
 
-        # Set the diagnostic callback.
+        # Set the diagnostic callback. Strongly recommended.
         sdk.set_diagnostic_callback(_diag_callback)
 
+        # Set the verbose callback.
+        # Not recommended in production as lots of messages can be emitted.
+        if IN_DEV_ENVIRONMENT:
+            sdk.set_verbose_callback(_diag_callback)
+
         # The agent state is one of the integers in oneagent.sdk.AgentState.
         print('Agent state:', sdk.agent_state)
 

samples/basic-sdk-sample/setup.py

@@ -28,7 +28,7 @@
     name='oneagent-sdk-basic-sample',
     version='0.0', # This sample is not separately versioned
 
-    install_requires=['oneagent-sdk==1.*,>=1.2'],
+    install_requires=['oneagent-sdk==1.*,>=1.4'],
 
     description='OneAgent SDK for Python: Basic sample application',
     long_description=long_description,

samples/fork-sdk-sample/fork_sdk_sample.py

@@ -39,9 +39,17 @@
 def do_some_fancy_stuff(proc_number):
     sdk = getsdk()
 
+    # Initially, the state in the child will be PRE_INITIALIZED (2).
+    print('Agent fork state (child process before SDK call):', sdk.agent_fork_state)
+
     # The agent state in the child process should be ACTIVE (0).
     print('Agent state (child process #{}): {}'.format(proc_number, sdk.agent_state), flush=True)
 
+    # After calling any SDK function but agent_fork_state,
+    # the state in the child will be FULLY_INITIALIZED (3).
+    # In this case the SDK function called was the agent_state property accessed above.
+    print('Agent fork state (child process after SDK call):', sdk.agent_fork_state)
+
     print('Agent found:', sdk.agent_found)
     print('Agent is compatible:', sdk.agent_is_compatible)
     print('Agent version:', sdk.agent_version_string)
@@ -105,6 +113,9 @@ def main():
         # Since we're using the 'forkable' mode the state will be TEMPORARILY_INACTIVE (1) on Linux.
         print('Agent state (parent process):', sdk.agent_state)
 
+        # In the parent, the state will be PARENT_INITIALIZED (1).
+        print('Agent fork state (parent process):', sdk.agent_fork_state)
+
         # The instance attribute 'agent_found' indicates whether an agent could be found or not.
         print('Agent found:', sdk.agent_found)
 

samples/fork-sdk-sample/setup.py

@@ -28,7 +28,7 @@
     name='oneagent-sdk-fork-sample',
     version='0.0', # This sample is not separately versioned
 
-    install_requires=['oneagent-sdk==1.*,>=1.3'],
+    install_requires=['oneagent-sdk==1.*,>=1.4'],
 
     description='OneAgent SDK for Python: Fork sample application',
     long_description=long_description,
@@ -38,7 +38,7 @@
     maintainer_email='[email protected]',
     license='Apache License 2.0',
     entry_points={
-        'console_scripts': ['oneagent-sdk-basic-sample=fork_sdk_sample:main'],
+        'console_scripts': ['oneagent-sdk-fork-sample=fork_sdk_sample:main'],
     },
     classifiers=[
         'Intended Audience :: Developers',

src/oneagent/__init__.py

@@ -84,15 +84,35 @@
 from oneagent._impl.six.moves import range #pylint:disable=import-error
 from oneagent.version import __version__
 
-from .common import SDKError, SDKInitializationError, ErrorCode, _ONESDK_INIT_FLAG_FORKABLE
+from .common import (
+    SDKError, SDKInitializationError, ErrorCode,
+    _ONESDK_INIT_FLAG_FORKABLE, _add_enum_helpers)
 from ._impl.native import nativeagent
 from ._impl.native.nativeagent import try_get_sdk
 from ._impl.native.sdknulliface import SDKNullInterface
 from ._impl.native.sdkdllinfo import WIN32
 
+if hasattr(sys, 'implementation'):
+    def _get_py_edition():
+        return sys.implementation.name # pylint:disable=no-member
+else:
+    import platform
+
+    def _get_py_edition():
+        return platform.python_implementation()
+
 logger = logging.getLogger('py_sdk')
 logger.setLevel(logging.CRITICAL + 1) # Disabled by default
 
+_PROCESS_TECH_PYTHON = 28
+_PROCESS_TECH_ONEAGENT_SDK = 118
+
+def _get_py_version():
+    return '.'.join(map(str, sys.version_info[:3])) + (
+        '' if sys.version_info.releaselevel == "final"
+        else sys.version_info.releaselevel + str(sys.version_info.serial))
+
+@_add_enum_helpers
 class InitResult(namedtuple('InitResult', 'status error')):
     __slots__ = ()
 
@@ -104,6 +124,10 @@ class InitResult(namedtuple('InitResult', 'status error')):
 
     __nonzero__ = __bool__ = lambda self: self.status >= 0
 
+    def __repr__(self):
+        return "InitResult(status={}, error={!r})".format(
+            self._value_name(self.status), self.error) #pylint:disable=no-member
+
 
 _sdk_ref_lk = Lock()
 _sdk_ref_count = 0
@@ -186,8 +210,9 @@ def initialize(sdkopts=(), sdklibname=None, forkable=False):
     * :meth:`oneagent.sdk.SDK.agent_version_string` works as expected.
     * :meth:`oneagent.sdk.SDK.agent_state` will return
         :data:`oneagent.common.AgentState.TEMPORARILY_INACTIVE` - but see the note below.
-    * :meth:`oneagent.sdk.SDK.set_diagnostic_callback` works as expected, the callback will be
-        carried over to forked child processes.
+    * :meth:`oneagent.sdk.SDK.set_diagnostic_callback` and
+      :meth:`oneagent.sdk.SDK.set_verbose_callback` work as expected,
+      the callback will be  carried over to forked child processes.
     * It is recommended you call :func:`shutdown` when the original process will not fork any more
         children that want to use the SDK.
 
@@ -248,8 +273,8 @@ def _try_init_noref(sdkopts=(), sdklibname=None, forkable=False):
 
     try:
         logger.info(
-            'Initializing SDK with options=%s, libname=%s.',
-            sdkopts, sdklibname)
+            'Initializing SDK on Python=%s with options=%s, libname=%s.',
+            (sys.version or '?').replace('\n', '  ').replace('\r', ''), sdkopts, sdklibname)
         sdk = nativeagent.initialize(sdklibname)
 
         have_warning = False
@@ -270,7 +295,11 @@ def _try_init_noref(sdkopts=(), sdklibname=None, forkable=False):
 
         nativeagent.checkresult(sdk, sdk.initialize(flags), 'onesdk_initialize_2')
         _should_shutdown = True
-        logger.debug("initialize successful")
+        logger.debug('initialize successful, adding tech types...')
+        sdk.ex_agent_add_process_technology(_PROCESS_TECH_ONEAGENT_SDK, 'Python', __version__)
+        sdk.ex_agent_add_process_technology(
+            _PROCESS_TECH_PYTHON, _get_py_edition(), _get_py_version())
+        logger.debug('tech type reporting complete')
         return InitResult(
             (InitResult.STATUS_INITIALIZED_WITH_WARNING if have_warning else
              InitResult.STATUS_INITIALIZED),
@@ -280,7 +309,7 @@ def _try_init_noref(sdkopts=(), sdklibname=None, forkable=False):
         #pylint:disable=no-member
         if isinstance(e, SDKError) and e.code == ErrorCode.AGENT_NOT_ACTIVE:
         #pylint:enable=no-member
-            logger.debug("initialized, but agent not active")
+            logger.debug('initialized, but agent not active')
             return InitResult(InitResult.STATUS_INITIALIZED_WITH_WARNING, e)
         logger.exception('Failed initializing agent.')
         sdk = nativeagent.try_get_sdk()

src/oneagent/_impl/native/sdkctypesiface.py

@@ -275,12 +275,16 @@ def __init__(self, libname):
             (),
             result_t)
 
-        # Missing: agent_internl_dispatch(int32, void*); probably useless
+        initfn(
+            'agent_set_warning_callback',
+            (agent_logging_callback_t,),
+            result_t,
+            public=False)
 
         initfn(
-            'agent_set_logging_callback',
+            'agent_set_verbose_callback',
             (agent_logging_callback_t,),
-            None,
+            result_t,
             public=False)
 
         initfn(
@@ -289,6 +293,22 @@ def __init__(self, libname):
             xchar_p,
             public=False)
 
+
+        initfn(
+            'agent_get_fork_state',
+            (),
+            ctypes.c_int32)
+
+        initfn(
+            'ex_api_enable_techtype',
+            (),
+            result_t,
+            public=False)
+        initfn(
+            'ex_agent_add_process_technology_p',
+            (ctypes.c_int32, CCStringPInArg, CCStringPInArg),
+            None).__doc__ = '''(tech_type, tech_edition, tech_version)'''
+
         # Specific nodes
 
         ## Database
@@ -559,6 +579,12 @@ def initialize(self, init_flags=0):
         self._agent_found = found.value != 0
         self._agent_is_compatible = compatible.value != 0
 
+        enable_result = self._ex_api_enable_techtype()
+        if self._agent_is_compatible and enable_result != ErrorCode.SUCCESS:
+            logger.warning(
+                "Tech type reporting API could not be enabled: %d %s",
+                enable_result, self.strerror(enable_result))
+
         return result
 
     def agent_found(self):
@@ -678,24 +704,38 @@ def cb_wrapper(level, msg):
     def agent_get_version_string(self):
         return self._agent_version
 
-    def agent_set_logging_callback(self, callback):
+    def _invoke_agent_log_cb_setter(self, callback, setter, update_stored_cb):
+        result = None
+        store_c_cb = lambda c_cb: setattr(self, "_diag_cb_" + setter.__name__, c_cb)
         if callback is None:
-            self._agent_set_logging_callback(
-                ctypes.cast(None, agent_logging_callback_t))
-            self._diag_cb = None
-            self._py_diag_cb = None
+            result = setter(ctypes.cast(None, agent_logging_callback_t))
+            if result == ErrorCode.SUCCESS:
+                store_c_cb(None)
+                if update_stored_cb:
+                    self._py_diag_cb = None
         else:
-
             @wraps(callback)
             def cb_wrapper(msg):
                 if isinstance(msg, six.binary_type):
                     msg = u8_to_str(msg)
                 return callback(msg)
 
             c_cb = agent_logging_callback_t(cb_wrapper)
-            self._agent_set_logging_callback(c_cb)
-            self._diag_cb = c_cb
-            self._py_diag_cb = cb_wrapper
+            result = setter(c_cb)
+            if result == ErrorCode.SUCCESS:
+                store_c_cb(c_cb)
+                if update_stored_cb:
+                    self._py_diag_cb = cb_wrapper
+        return result
+
+
+    def agent_set_warning_callback(self, callback):
+        return self._invoke_agent_log_cb_setter(
+            callback, self._agent_set_warning_callback, update_stored_cb=True)
+
+    def agent_set_verbose_callback(self, callback):
+        return self._invoke_agent_log_cb_setter(
+            callback, self._agent_set_verbose_callback, update_stored_cb=False)
 
     def __del__(self):
         # __del__ is also called when __init__ fails, so safeguard against that