Merge pull request #532 from adobe-dmeservices/feature/adobe-console-connector

New Feature: Adobe Admin Console connector as Directory Source

Author: adorton-adobe

examples/config files - basic/connector-adobe-console.yml

@@ -0,0 +1,83 @@
+# This is a sample configuration file for the Adobe-Console connector.
+#
+# umapi (user management api) is a network protocol served by Adobe that
+# provides management of users in Adobe-hosted enterprise organizations.
+#
+# This sample file contains all of the settable options for this protocol.
+# All of the settings here can be changed.  It is recommended
+# that you make a copy of this file and edit that to match your configuration.
+# While you are at it, you will likely want to remove a lot of this  commentary,
+# in order to enhance the readability of your file.
+
+# (optional) UMAPI server settings (defaults as shown)
+# The host and endpoint settings specify the Adobe endpoints which
+# host the UMAPI services and those which provide authorization.
+# The timeout and retries settings control how much delay (in seconds)
+# can be tolerated in server responses, and also how many times a request
+# that fails due to server timeout or server throttling will be retried.
+# You will *never* need to alter these settings unless you are provided
+# alternate values by Adobe as part of a support engagement.  It is
+# highly recommended that you leave these values commented out
+# so that the default values are guaranteed to be used.
+server:
+  #host: usermanagement.adobe.io
+  #endpoint: /v2/usermanagement
+  #ims_host: ims-na1.adobelogin.com
+  #ims_endpoint_jwt: /ims/exchange/jwt
+  #timeout: 120
+  #retries: 3
+
+# (required) integration settings
+# You must specify all five of these settings.  Consult the
+# Adobe UMAPI documentation and the Adobe I/O Console to determine
+# the correct settings for your enterprise organization.
+# [NOTE: the priv_key_path setting can be an absolute or relative pathname;
+# if relative, it is interpreted relative to this configuration file.]
+integration:
+  org_id: "Org ID goes here"
+  api_key: "API key goes here"
+  client_secret: "Client secret goes here"
+  tech_acct: "Tech account ID goes here"
+  priv_key_path: "private.key"
+
+  # (optional) As an alternative to priv_key_path, you can place the private key
+  # data directly in this file.  To do this, remove the priv_key_path entry above
+  # and uncomment the following entry.  Replace the sample data with the data
+  # from your private key file (which will be much longer).
+  #priv_key_data: |
+  #   -----BEGIN RSA PRIVATE KEY-----
+  #   MIIf74jfd84oAgEA6brj4uZ2f1Nkf84j843jfjjJGHYJ8756GHHGGz7jLyZWSscH
+  #   CoifurKJY763GHKL98mJGYxWSBvhlWskdjdatagoeshere986fKFUNGd74kdfuEH
+  #   -----END RSA PRIVATE KEY-----
+
+  # (optional) You can store credentials in the operating system credential store
+  # (Windows Credential Manager, Mac Keychain, Linux Freedesktop Secret Service
+  # or KWallet - these will be built into the Linux distribution).
+  # To use this feature, uncomment the following entries and remove the
+  # api_key, client_secret, and priv_key_data above.
+  # The actual credential values are placed in the credential store with the
+  # username as the org_id value, and the key name (perhaps called internet
+  # or network address) as one of the values below.
+  #secure_api_key_key: umapi_api_key
+  #secure_client_secret_key: umapi_client_secret
+  #secure_priv_key_data_key: umapi_private_key_data
+  # Note: the Windows credential store generally can't store data as large as a private
+  # key, so the recommended path for securing your private key on windows is given next.
+
+  # (optional): You can secure your private key data by encrypting it, as with
+  #     openssl pkcs8 -in private.key -topk8 -v2 des3 -out private-encrypted.key
+  # which prompts for a passphrase and creates a passphrase-encrypted file in PKCS#8 format.
+  # Having done this, you can use the setting priv_key_pass to specify the passphrase needed
+  # by User Sync to decrypt the private key file (or private key data), as in:
+  #priv_key_pass: "my passphrase for my private key"
+  # For better security, you should save your passphrase into the secure credential store
+  # on your platform (username = your org ID, service/internet address = "umapi_private_key_passphrase")
+  # and then uncomment this setting:
+  #secure_priv_key_pass_key: umapi_private_key_passphrase
+
+# (optional) identity_type_filter (default value is all)
+# By default, connector will automatically load users from all identity type to be load as directory users.
+# When specify with one of the following value (adobeID, enterpriseID, federatedID)
+# the connector will automatically filter users by the specified identity type.
+identity_type_filter: all
+

examples/config files - basic/user-sync-config.yml

@@ -154,7 +154,12 @@ directory_users:
     # (optional) okta (no default value)
     # okta is a 3rd party federation provider compatible with Adobe Enterprise Federated ID.
     # See https://developer.okta.com/ for Okta developer information.
-    # okta: "connector-okta.ytml"
+    # okta: "connector-okta.yml"
+
+    # (optional) Adobe console
+    # Query users from an Adobe organization to use as an identity source
+    # See user manual for more inforation
+    # adobe_console: connector-adobe-console.yml
 
   # (optional) groups (no default value)
   # The groups setting specifies how groups in the enterprise directory map

user_sync/app.py

@@ -124,7 +124,7 @@ def main():
               help='specify a connector to use; default is LDAP (or CSV if --users file is specified)',
               cls=user_sync.cli.OptionMulti,
               type=list,
-              metavar='ldap|okta|csv [path-to-file.csv]')
+              metavar='ldap|okta|csv|adobe_console [path-to-file.csv]')
 @click.option('--process-groups/--no-process-groups', default=None,
               help='if membership in mapped groups differs between the enterprise directory and Adobe sides, '
                    'the group membership is updated on the Adobe side so that the memberships in mapped '

user_sync/config.py

@@ -127,7 +127,7 @@ def load_invocation_options(self):
         # --connector
         connector_spec = options['connector']
         connector_type = user_sync.helper.normalize_string(connector_spec[0])
-        if connector_type in ["ldap", "okta"]:
+        if connector_type in ["ldap", "okta", "adobe_console"]:
             if len(connector_spec) > 1:
                 raise AssertionException('Must not specify a file (%s) with connector type %s' %
                                          (connector_spec[0], connector_type))
@@ -312,6 +312,7 @@ def get_directory_connector_configs(self):
             connectors_config.get_list('ldap', True)
             connectors_config.get_list('csv', True)
             connectors_config.get_list('okta', True)
+            connectors_config.get_list('adobe_console', True)
         return connectors_config
 
     def get_directory_connector_options(self, connector_name):
@@ -845,7 +846,8 @@ class ConfigFileLoader:
                              }
 
     # like ROOT_CONFIG_PATH_KEYS, but for non-root configuration files
-    SUB_CONFIG_PATH_KEYS = {'/enterprise/priv_key_path': (True, False, None)}
+    SUB_CONFIG_PATH_KEYS = {'/enterprise/priv_key_path': (True, False, None),
+                            '/integration/priv_key_path': (True, False, None)}
 
     @classmethod
     def load_root_config(cls, filename):

user_sync/connector/directory_adobe_console.py

@@ -0,0 +1,238 @@
+# Copyright (c) 2016-2017 Adobe Inc.  All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import six
+import umapi_client
+import user_sync.config
+import user_sync.connector.helper
+import user_sync.helper
+import user_sync.identity_type
+from user_sync.error import AssertionException
+from user_sync.version import __version__ as app_version
+from user_sync.connector.umapi_util import make_auth_dict
+from user_sync.helper import normalize_string
+from user_sync.identity_type import parse_identity_type
+
+
+def connector_metadata():
+    metadata = {
+        'name': AdobeConsoleConnector.name
+    }
+    return metadata
+
+
+def connector_initialize(options):
+    """
+    :type options: dict
+    """
+    state = AdobeConsoleConnector(options)
+    return state
+
+
+def connector_load_users_and_groups(state, groups=None, extended_attributes=None, all_users=True):
+    """
+    :type state: OktaDirectoryConnector
+    :type groups: list(str)
+    :type extended_attributes: list(str)
+    :type all_users: bool
+    :rtype (bool, iterable(dict))
+    """
+
+    return state.load_users_and_groups(groups or [], extended_attributes or [], all_users)
+
+
+class AdobeConsoleConnector(object):
+    name = 'adobe_console'
+
+    def __init__(self, caller_options):
+
+        caller_config = user_sync.config.DictConfig('<%s configuration>' % self.name, caller_options)
+        builder = user_sync.config.OptionsBuilder(caller_config)
+        # Let just ignore this
+        builder.set_string_value('user_identity_type', None)
+        builder.set_string_value('identity_type_filter', 'all')
+        options = builder.get_options()
+
+        if not options['identity_type_filter'] == 'all':
+            try:
+                options['identity_type_filter'] = parse_identity_type(options['identity_type_filter'])
+            except Exception as e:
+                raise AssertionException("Error parsing identity_type_filter option: %s" % e)
+        self.filter_by_identity_type = options['identity_type_filter']
+
+        server_config = caller_config.get_dict_config('server', True)
+        server_builder = user_sync.config.OptionsBuilder(server_config)
+        server_builder.set_string_value('host', 'usermanagement.adobe.io')
+        server_builder.set_string_value('endpoint', '/v2/usermanagement')
+        server_builder.set_string_value('ims_host', 'ims-na1.adobelogin.com')
+        server_builder.set_string_value('ims_endpoint_jwt', '/ims/exchange/jwt')
+        server_builder.set_int_value('timeout', 120)
+        server_builder.set_int_value('retries', 3)
+        options['server'] = server_options = server_builder.get_options()
+
+        enterprise_config = caller_config.get_dict_config('integration')
+        integration_builder = user_sync.config.OptionsBuilder(enterprise_config)
+        integration_builder.require_string_value('org_id')
+        integration_builder.require_string_value('tech_acct')
+        options['integration'] = integration_options = integration_builder.get_options()
+
+        self.logger = logger = user_sync.connector.helper.create_logger(options)
+        logger.debug('%s initialized with options: %s', self.name, options)
+
+        self.options = options
+
+        ims_host = server_options['ims_host']
+        self.org_id = org_id = integration_options['org_id']
+        auth_dict = make_auth_dict(self.name, enterprise_config, org_id, integration_options['tech_acct'], logger)
+
+        # this check must come after we fetch all the settings
+        caller_config.report_unused_values(logger)
+        # open the connection
+        um_endpoint = "https://" + server_options['host'] + server_options['endpoint']
+        logger.debug('%s: creating connection for org %s at endpoint %s', self.name, org_id, um_endpoint)
+
+        try:
+            self.connection = umapi_client.Connection(
+                org_id=org_id,
+                auth_dict=auth_dict,
+                ims_host=ims_host,
+                ims_endpoint_jwt=server_options['ims_endpoint_jwt'],
+                user_management_endpoint=um_endpoint,
+                test_mode=False,
+                user_agent="user-sync/" + app_version,
+                logger=self.logger,
+                timeout_seconds=float(server_options['timeout']),
+                retry_max_attempts=server_options['retries'] + 1,
+            )
+        except Exception as e:
+            raise AssertionException("Connection to org %s at endpoint %s failed: %s" % (org_id, um_endpoint, e))
+        logger.debug('%s: connection established', self.name)
+        self.umapi_users = []
+        self.user_by_usr_key = {}
+
+    def load_users_and_groups(self, groups, extended_attributes, all_users):
+        """
+        :type groups: list(str)
+        :type extended_attributes: list(str)
+        :type all_users: bool
+        :rtype (bool, iterable(dict))
+        """
+
+        if extended_attributes:
+            self.logger.warning("Extended Attributes is not supported")
+
+        # Loading all the groups because UMAPI doesn't support group query. DOH!
+        self.logger.info('Loading groups...')
+        umapi_groups = list(self.iter_umapi_groups())
+        self.logger.info('Loading users...')
+
+        # Loading all umapi users based on ID Type first before doing group filtering
+        filter_by_identity_type = self.filter_by_identity_type
+        self.load_umapi_users(identity_type=filter_by_identity_type)
+
+        grouped_user_records = {}
+        for group in groups:
+            group_users_count = 0
+            if group in umapi_groups:
+                grouped_users = self.iter_group_members(group)
+                for user_key in grouped_users:
+                    if user_key in self.user_by_usr_key:
+                        user = self.user_by_usr_key[user_key]
+                        user['groups'].append(group)
+                        self.user_by_usr_key[user_key] = grouped_user_records[user_key] = user
+                        group_users_count = group_users_count + 1
+                self.logger.debug('Count of users in group "%s": %d', group, group_users_count)
+            else:
+                self.logger.warning("No group found for: %s", group)
+        if all_users:
+            self.logger.debug('Count of users in any groups: %d', len(grouped_user_records))
+            self.logger.debug('Count of users not in any groups: %d',
+                              len(self.user_by_usr_key) - len(grouped_user_records))
+            return six.itervalues(self.user_by_usr_key)
+        else:
+            return six.itervalues(grouped_user_records)
+
+    def convert_user(self, record):
+
+        source_attributes = {}
+        user = user_sync.connector.helper.create_blank_user()
+        user['uid'] = record['username']
+        source_attributes['email'] = user['email'] = email = record['email']
+        user_identity_type = record['type']
+        try:
+            source_attributes['type'] = user['identity_type'] = user_sync.identity_type.parse_identity_type(
+                user_identity_type)
+        except AssertionException as e:
+            self.logger.warning('Skipping user %s: %s', email, e)
+            return None
+
+        source_attributes['username'] = user['username'] = record['username']
+        source_attributes['domain'] = user['domain'] = record['domain']
+
+        if 'firstname' in record:
+            firstname = record['firstname']
+        else:
+            firstname = None
+        source_attributes['firstname'] = user['firstname'] = firstname
+
+        if 'lastname' in record:
+            lastname = record['lastname']
+        else:
+            lastname = None
+        source_attributes['lastname'] = user['lastname'] = lastname
+
+        source_attributes['country'] = user['country'] = record['country']
+
+        user['source_attributes'] = source_attributes.copy()
+        return user
+
+    def iter_umapi_groups(self):
+        try:
+            groups = umapi_client.GroupsQuery(self.connection)
+            for group in groups:
+                yield group['groupName']
+        except umapi_client.UnavailableError as e:
+            raise AssertionException("Error to query groups from Adobe Console: %s" % e)
+
+    def iter_group_members(self, group):
+        umapi_users = self.umapi_users
+        members = filter(lambda u: ('groups' in u and group in u['groups']), umapi_users)
+        for member in members:
+            user_key = self.generate_user_key(member['type'], member['username'], member['domain'])
+            yield (user_key)
+
+    def load_umapi_users(self, identity_type):
+        try:
+            u_query = umapi_client.UsersQuery(self.connection)
+            umapi_users = u_query.all_results()
+
+            if not identity_type == 'all':
+                umapi_users = list(filter(lambda usr: usr['type'] == identity_type, umapi_users))
+
+            self.umapi_users = umapi_users
+            for user in umapi_users:
+                # Generate unique user key because Username/Email is a bad unique identifier
+                user_key = self.generate_user_key(user['type'], user['username'], user['domain'])
+                self.user_by_usr_key[user_key] = self.convert_user(user)
+        except umapi_client.UnavailableError as e:
+            raise AssertionException("Error contacting UMAPI server: %s" % e)
+
+    def generate_user_key(self, identity_type, username, domain):
+        return '%s,%s,%s' % (normalize_string(identity_type), normalize_string(username), normalize_string(domain))