Merge pull request #405 from adorton-adobe/feature/group-sync-updates

Additional group sync updates

Author: adorton-adobe

docs/en/user-manual/advanced_configuration.md

@@ -699,8 +699,6 @@ Possible use cases:
 * ACL groups for [Adobe Experience Manager](https://www.adobe.com/marketing/experience-manager.html)
 * Special-case group, role or profile assignment
 
-Note: This feature only works with the LDAP connector at this time.
-
 ### Additional Group Rules
 
 `additional_groups` is defined in `user-sync-config.yml` in the `groups`
@@ -710,6 +708,9 @@ how corresponding Adobe groups should be named.  Groups that are
 discovered with this feature will be added to a user's list of
 targeted Adobe groups.
 
+**Note:** Additional group mapping will fail if a multiple source groups
+map to the same target group.
+
 ### Additional Group Example
 
 Suppose an Adobe Experience Manager customer would like
@@ -758,6 +759,21 @@ will apply dynamically to any LDAP group that matches the regex
 be included in sync as long as they follow that naming convention -
 no configuration change would be needed.
 
+### Targeting Secondary Orgs
+
+Secondary organizations can be targeted using the additional group
+rules.  Just add the prefix `[org_name]::` to the target group
+pattern.
+
+```yaml
+  additional_groups:
+    - source: "ACL-GRP-(\\d+)"
+      target: "org2::ACL Group \\1"
+ ```
+
+Refer to [Accessing Users in Other Organizations](https://adobe-apiplatform.github.io/user-sync.py/en/user-manual/advanced_configuration.html#accessing-users-in-other-organizations)
+for more information.
+
 ## Automatic Group Creation
 
 The User Sync Tool can be configured to automatically create targeted
@@ -794,6 +810,14 @@ If the Sync Tool is configured to target a misspelled profile name, or
 a profile that doesn't exist, it will automatically create a user group
 with the specified name.
 
+### Targeting Secondary Orgs
+
+Groups targeted to secondary organizations will be automatically
+created on those organizations if `auto_create` is enabled.
+
+Refer to [Accessing Users in Other Organizations](https://adobe-apiplatform.github.io/user-sync.py/en/user-manual/advanced_configuration.html#accessing-users-in-other-organizations)
+for more information.
+
 ---
 
 [Previous Section](usage_scenarios.md)  \| [Next Section](deployment_best_practices.md)

user_sync/config.py

@@ -444,68 +444,73 @@ def get_rule_options(self):
         options.update(self.invocation_options)
 
         # process directory configuration options
-        new_account_type = None
         directory_config = self.main_config.get_dict_config('directory_users', True)
-        if directory_config:
-            # account type
-            new_account_type = directory_config.get_string('user_identity_type', True)
-            new_account_type = user_sync.identity_type.parse_identity_type(new_account_type)
-            if new_account_type:
-                options['new_account_type'] = new_account_type
-            else:
-                self.logger.debug("Using default for new_account_type: %s", options['new_account_type'])
-            # country code
-            default_country_code = directory_config.get_string('default_country_code', True)
-            if default_country_code:
-                options['default_country_code'] = default_country_code
-            additional_groups = directory_config.get_list('additional_groups', True) or []
-            additional_groups = [{'source': re.compile(r['source']), 'target': r['target']} for r in additional_groups]
-            options['additional_groups'] = additional_groups
-            sync_options = directory_config.get_dict_config('group_sync_options', True)
-            if sync_options:
-                options['auto_create'] = sync_options.get_bool('auto_create', True)
-        if not new_account_type:
-            new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE
-            self.logger.debug("Using default for new_account_type: %s", new_account_type)
+        if not directory_config:
+            raise AssertionException("'directory_users' must be specified")
+
+        # account type
+        new_account_type = directory_config.get_string('user_identity_type', True)
+        new_account_type = user_sync.identity_type.parse_identity_type(new_account_type)
+        if new_account_type:
+            options['new_account_type'] = new_account_type
+        else:
+            self.logger.debug("Using default for new_account_type: %s", options['new_account_type'])
+        # country code
+        default_country_code = directory_config.get_string('default_country_code', True)
+        if default_country_code:
+            options['default_country_code'] = default_country_code
+        additional_groups = directory_config.get_list('additional_groups', True) or []
+        try:
+            additional_groups = [{'source': re.compile(r['source']),
+                                  'target': user_sync.rules.AdobeGroup.create(r['target'], index=False)}
+                                 for r in additional_groups]
+        except Exception as e:
+            raise AssertionException("Additional group rule error: {}".format(str(e)))
+        options['additional_groups'] = additional_groups
+        sync_options = directory_config.get_dict_config('group_sync_options', True)
+        if sync_options:
+            options['auto_create'] = sync_options.get_bool('auto_create', True)
 
         # process exclusion configuration options
         adobe_config = self.main_config.get_dict_config('adobe_users', True)
-        if adobe_config:
-            exclude_identity_type_names = adobe_config.get_list('exclude_identity_types', True)
-            if exclude_identity_type_names:
-                exclude_identity_types = []
-                for name in exclude_identity_type_names:
-                    message_format = 'Illegal value in exclude_identity_types: %s'
-                    identity_type = user_sync.identity_type.parse_identity_type(name, message_format)
-                    exclude_identity_types.append(identity_type)
-                options['exclude_identity_types'] = exclude_identity_types
-            exclude_users_regexps = adobe_config.get_list('exclude_users', True)
-            if exclude_users_regexps:
-                exclude_users = []
-                for regexp in exclude_users_regexps:
-                    try:
-                        # add "match begin" and "match end" markers to ensure complete match
-                        # and compile the patterns because we will use them over and over
-                        exclude_users.append(re.compile(r'\A' + regexp + r'\Z', re.UNICODE))
-                    except re.error as e:
-                        validation_message = ('Illegal regular expression (%s) in %s: %s' %
-                                              (regexp, 'exclude_identity_types', e))
-                        raise AssertionException(validation_message)
-                options['exclude_users'] = exclude_users
-            exclude_group_names = adobe_config.get_list('exclude_adobe_groups', True) or []
-            if exclude_group_names:
-                exclude_groups = []
-                for name in exclude_group_names:
-                    group = user_sync.rules.AdobeGroup.create(name)
-                    if not group or group.get_umapi_name() != user_sync.rules.PRIMARY_UMAPI_NAME:
-                        validation_message = 'Illegal value for %s in config file: %s' % ('exclude_groups', name)
-                        if not group:
-                            validation_message += ' (Not a legal group name)'
-                        else:
-                            validation_message += ' (Can only exclude groups in primary organization)'
-                        raise AssertionException(validation_message)
-                    exclude_groups.append(group.get_group_name())
-                options['exclude_groups'] = exclude_groups
+        if not adobe_config:
+            raise AssertionException("'adobe_users' must be specified")
+
+        exclude_identity_type_names = adobe_config.get_list('exclude_identity_types', True)
+        if exclude_identity_type_names:
+            exclude_identity_types = []
+            for name in exclude_identity_type_names:
+                message_format = 'Illegal value in exclude_identity_types: %s'
+                identity_type = user_sync.identity_type.parse_identity_type(name, message_format)
+                exclude_identity_types.append(identity_type)
+            options['exclude_identity_types'] = exclude_identity_types
+        exclude_users_regexps = adobe_config.get_list('exclude_users', True)
+        if exclude_users_regexps:
+            exclude_users = []
+            for regexp in exclude_users_regexps:
+                try:
+                    # add "match begin" and "match end" markers to ensure complete match
+                    # and compile the patterns because we will use them over and over
+                    exclude_users.append(re.compile(r'\A' + regexp + r'\Z', re.UNICODE))
+                except re.error as e:
+                    validation_message = ('Illegal regular expression (%s) in %s: %s' %
+                                          (regexp, 'exclude_identity_types', e))
+                    raise AssertionException(validation_message)
+            options['exclude_users'] = exclude_users
+        exclude_group_names = adobe_config.get_list('exclude_adobe_groups', True) or []
+        if exclude_group_names:
+            exclude_groups = []
+            for name in exclude_group_names:
+                group = user_sync.rules.AdobeGroup.create(name)
+                if not group or group.get_umapi_name() != user_sync.rules.PRIMARY_UMAPI_NAME:
+                    validation_message = 'Illegal value for %s in config file: %s' % ('exclude_groups', name)
+                    if not group:
+                        validation_message += ' (Not a legal group name)'
+                    else:
+                        validation_message += ' (Can only exclude groups in primary organization)'
+                    raise AssertionException(validation_message)
+                exclude_groups.append(group.get_group_name())
+            options['exclude_groups'] = exclude_groups
 
         # get the limits
         limits_config = self.main_config.get_dict_config('limits')

user_sync/connector/directory_ldap.py

@@ -292,17 +292,7 @@ def iter_users(self, users_filter, extended_attributes):
             elif last_attribute_name:
                 self.logger.warning('No country code attribute (%s) for user with dn: %s', last_attribute_name, dn)
 
-            uid_value = LDAPValueFormatter.get_attribute_value(record, six.text_type('uid'))
-            source_attributes['uid'] = uid_value
-
-            user['member_groups'] = []
-            if self.additional_group_filters:
-                member_groups = []
-                for f in self.additional_group_filters:
-                    for g in self.get_member_groups(record):
-                        if f.match(g) and g not in member_groups:
-                            member_groups.append(g)
-                user['member_groups'] = member_groups
+            user['member_groups'] = self.get_member_groups(record) if self.additional_group_filters else []
 
             if extended_attributes is not None:
                 for extended_attribute in extended_attributes:

user_sync/rules.py

@@ -26,6 +26,7 @@
 import user_sync.connector.umapi
 import user_sync.error
 import user_sync.identity_type
+from collections import defaultdict
 from user_sync.helper import normalize_string, CSVAdapter, JobStats
 
 GROUP_NAME_DELIMITER = '::'
@@ -173,10 +174,15 @@ def run(self, directory_groups, directory_connector, umapi_connectors):
             self.read_desired_user_groups(directory_groups, directory_connector)
             load_directory_stats.log_end(logger)
 
+        for umapi_info in self.umapi_info_by_name.values():
+            self.validate_and_log_additional_groups(umapi_info)
+
         umapi_stats = JobStats('Push to UMAPI' if self.push_umapi else 'Sync with UMAPI', divider="-")
         umapi_stats.log_start(logger)
         if directory_connector is not None:
-            if self.options.get('process_groups'):
+            # note: push mode is not supported because if it is, we won't have a list of groups
+            # that exist in the console.  we don't want to attempt to create groups that already exist
+            if self.options.get('process_groups') and not self.push_umapi and self.options.get('auto_create'):
                 self.create_umapi_groups(umapi_connectors)
             self.sync_umapi_users(umapi_connectors)
         if self.will_process_strays:
@@ -185,6 +191,20 @@ def run(self, directory_groups, directory_connector, umapi_connectors):
         umapi_stats.log_end(logger)
         self.log_action_summary(umapi_connectors)
 
+    def validate_and_log_additional_groups(self, umapi_info):
+        """
+        :param umapi_info: UmapiTargetInfo
+        :return:
+        """
+        umapi_name = umapi_info.get_name()
+        for mapped, src_groups in umapi_info.get_additional_group_map().items():
+            if len(src_groups) > 1:
+                raise user_sync.error.AssertionException(
+                    "Additional group resolution conflict: {} map to '{}' on '{}'".format(
+                        src_groups, mapped, umapi_name if umapi_name else 'primary org'))
+            self.logger.info("Mapped additional group '{}' to '{}' on '{}'".format(
+                src_groups[0], mapped, umapi_name if umapi_name else 'primary org'))
+
     def log_action_summary(self, umapi_connectors):
         """
         log number of affected directory and Adobe users,
@@ -394,11 +414,19 @@ def read_desired_user_groups(self, mappings, directory_connector):
             member_groups = directory_user.get('member_groups', [])
             for member_group in member_groups:
                 for group_rule in additional_groups:
-                    if group_rule['source'].match(member_group):
-                        rename_group = group_rule['source'].sub(group_rule['target'], member_group)
-                        umapi_info.add_mapped_group(rename_group)
-                        for umapi_name, umapi_info in six.iteritems(self.umapi_info_by_name):
-                            umapi_info.add_desired_group_for(user_key, rename_group)
+                    source = group_rule['source']
+                    target = group_rule['target']
+                    target_name = target.get_group_name()
+                    umapi_info = self.get_umapi_info(target.get_umapi_name())
+                    if not group_rule['source'].match(member_group):
+                        continue
+                    try:
+                        rename_group = source.sub(target_name, member_group)
+                    except Exception as e:
+                        raise user_sync.error.AssertionException("Additional group resolution error: {}".format(str(e)))
+                    umapi_info.add_mapped_group(rename_group)
+                    umapi_info.add_additional_group(rename_group, member_group)
+                    umapi_info.add_desired_group_for(user_key, rename_group)
 
         self.logger.debug('Total directory users after filtering: %d', len(filtered_directory_user_by_user_key))
         if self.logger.isEnabledFor(logging.DEBUG):
@@ -474,24 +502,32 @@ def create_umapi_groups(self, umapi_connectors):
         in the console, then it will create. Note: Push Mode is not supported
         :type umapi_connectors: UmapiConnectors
         """
-        if not self.push_umapi:
-            umapi_info, umapi_connector = self.get_umapi_info(
-                PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector()
+        for umapi_connector in umapi_connectors.connectors:
+            umapi_name = None if umapi_connector.name.split('.')[-1] == 'primary'\
+                else umapi_connector.name.split('.')[-1]
+            if umapi_name == 'umapi':
+                umapi_name = None
+            if umapi_name not in self.umapi_info_by_name:
+                continue
+            umapi_info = self.umapi_info_by_name[umapi_name]
             mapped_groups = umapi_info.get_non_normalize_mapped_groups()
+
             # pull all user groups from console
             on_adobe_groups = [normalize_string(g['groupName']) for g in umapi_connector.get_groups()]
+
             # verify if group exist and create
-            auto_create = self.options.get('auto_create', None)
-            if auto_create:
-                for mapped_group in mapped_groups:
-                    if normalize_string(mapped_group) not in on_adobe_groups:
-                        self.logger.info("Auto create user-group enabled: Creating %s" % mapped_group)
-                        try:
-                            # create group
-                            res = umapi_connector.create_group(mapped_group)
-                            self.action_summary['adobe_user_groups_created'] += 1
-                        except Exception as e:
-                            self.logger.critical("Unable to create %s user group: %s" % (mapped_group, e))
+            for mapped_group in mapped_groups:
+                if normalize_string(mapped_group) in on_adobe_groups:
+                    continue
+                self.logger.info("Auto create user-group enabled: Creating '{}' on '{}'".format(
+                    mapped_group, umapi_name if umapi_name else 'primary org'))
+                try:
+                    # create group
+                    res = umapi_connector.create_group(mapped_group)
+                    self.action_summary['adobe_user_groups_created'] += 1
+                except Exception as e:
+                    self.logger.critical("Unable to create %s user group: '{}' on '{}' (error: {})".format(
+                        mapped_group, umapi_name if umapi_name else 'primary org', e))
 
     def is_selected_user_key(self, user_key):
         """
@@ -1115,14 +1151,15 @@ def execute_actions(self):
 class AdobeGroup(object):
     index_map = {}
 
-    def __init__(self, group_name, umapi_name):
+    def __init__(self, group_name, umapi_name, index=True):
         """
         :type group_name: str
         :type umapi_name: str
         """
         self.group_name = group_name
         self.umapi_name = umapi_name
-        AdobeGroup.index_map[(group_name, umapi_name)] = self
+        if index:
+            AdobeGroup.index_map[(group_name, umapi_name)] = self
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__
@@ -1166,13 +1203,13 @@ def lookup(cls, qualified_name):
         return cls.index_map.get(cls._parse(qualified_name))
 
     @classmethod
-    def create(cls, qualified_name):
+    def create(cls, qualified_name, index=True):
         group_name, umapi_name = cls._parse(qualified_name)
         existing = cls.index_map.get((group_name, umapi_name))
         if existing:
             return existing
         elif len(group_name) > 0:
-            return cls(group_name, umapi_name)
+            return cls(group_name, umapi_name, index)
         else:
             return None
 
@@ -1196,6 +1233,10 @@ def __init__(self, name):
         self.groups_added_by_user_key = {}
         self.groups_removed_by_user_key = {}
 
+        # keep track of auto-mapped additional groups for conflict tracking.
+        # if feature is disabled, this dict will be empty
+        self.additional_group_map = defaultdict(list)  # type: dict[str, list[str]]
+
     def get_name(self):
         return self.name
 
@@ -1207,6 +1248,13 @@ def add_mapped_group(self, group):
         self.mapped_groups.add(normalized_group_name)
         self.non_normalize_mapped_groups.add(group)
 
+    def add_additional_group(self, rename_group, member_group):
+        if member_group not in self.additional_group_map[rename_group]:
+            self.additional_group_map[normalize_string(rename_group)].append(member_group)
+
+    def get_additional_group_map(self):
+        return self.additional_group_map
+
     def get_mapped_groups(self):
         return self.mapped_groups