Merge pull request #267 from adobe-apiplatform/v2

prepare for 2.2.1rc1 build

Author: adobeDan

Makefile

@@ -17,7 +17,7 @@ endif
 
 pex:
 	pip install --upgrade pip
-	pip install --upgrade wheel pex
+	pip install --upgrade wheel requests pex
 	-$(RM) $(output_dir)
 	pex -v -o $(output_dir)/$(output_filename)$(output_file_extension) -m user_sync.app \
 		-f $(prebuilt_dir) \

RELEASE_NOTES.md

@@ -1,40 +1,27 @@
 # Release Notes for User Sync Tool Version 2.2.1
 
-These notes apply to v2.2 of 2017-08-03.
+These notes apply to v2.2.1rc1 of 2017-08-28.
 
 ## New Features
 
-[#52](https://github.com/adobe-apiplatform/user-sync.py/issues/52): This release runs on both Python 2 and Python 3 (2.7, 3.4, 3.5, and 3.6 to be precise)!
-
-[#182](https://github.com/adobe-apiplatform/user-sync.py/issues/182): At long last, you can select users in nested groups.  The new implementation for determining group members also allows us to avoid fetching the entire directory when the users are only supposed to come from specific groups, as with `--users mapped` ([#129](https://github.com/adobe-apiplatform/user-sync.py/issues/129)).  There is a new LDAP configuration setting `group_member_filter_format` which controls how users are selected for groups (default is "immediate members only", which is backward compatible with prior releases).
-
-[#236](https://github.com/adobe-apiplatform/user-sync.py/issues/236): Directory users can now be pushed directly to Adobe, rather than synchronized with a fetch of Adobe users.  A new command-line argument `--strategy push` (as opposed to the default `--strategy sync`) controls this.
-
-[#234](https://github.com/adobe-apiplatform/user-sync.py/issues/234): There are new UMAPI configuration settings (`timeout` and `retries` in the `server` section) to control the network behavior when talking to the UMAPI server.  The default timeout of 120 seconds and the default retry count of 3 are unchanged.
-
-[#237](https://github.com/adobe-apiplatform/user-sync.py/issues/237): The default encoding for all inputs (config files, CSV files, LDAP attribute values) is now assumed to be `utf8` rather than ASCII.  This is a backward-compatible change that makes it unnecessary (but still allowed) to specify `utf8` explicitly.
+[#266](https://github.com/adobe-apiplatform/user-sync.py/issues/266): Extended attribute values (defined in extensions) can now be multi-valued.  The type of the attribute value in the `source_attributes` dictionary will be:
+* `None` if the attribute has no value;
+* a `str` (or `unicode` in py2) if the attribute has one value;
+* a `list` of `str` (or `unicode` in py2) if the attribute has multiple values.
 
 ## Bug Fixes
 
-[#227](https://github.com/adobe-apiplatform/user-sync.py/issues/227): Fixed crashes due to bad user keys.
-
-[#233](https://github.com/adobe-apiplatform/user-sync.py/issues/233): Exceptions in LDAP connections are handled gracefully, as are keyboard interrupts.
-
-[#235](https://github.com/adobe-apiplatform/user-sync.py/issues/235): Fixed a crash that occurred if an Adobe ID user had no username or domain info.
-
-[#240](https://github.com/adobe-apiplatform/user-sync.py/issues/240): When using the LDAP connector, the domain of each user is now correctly defaulted to the email domain.
-
-[#244](https://github.com/adobe-apiplatform/user-sync.py/issues/244): Build instructions are now provided for all platforms, and the default `Makefile` allows for the use of pre-compiled, platform-specific wheels.
+[#257](https://github.com/adobe-apiplatform/user-sync.py/issues/257): Catch exceptions thrown by umapi-client when creating actions.
 
-[#247](https://github.com/adobe-apiplatform/user-sync.py/issues/247): There is no more use of the `uid` attribute in LDAP directories.
+[#258](https://github.com/adobe-apiplatform/user-sync.py/issues/258): Correctly decrypte private keys in py3.
 
-[#254](https://github.com/adobe-apiplatform/user-sync.py/issues/254): Update windows libraries, reduce use of custom builds.
+[#260](https://github.com/adobe-apiplatform/user-sync.py/issues/260): Make sure the requests library is loaded when using pex on Windows.
 
-[#258](https://github.com/adobe-apiplatform/user-sync.py/issues/258): Correctly decrypt private keys in py3 (byte vs. str type compatibility).
+[#265](https://github.com/adobe-apiplatform/user-sync.py/issues/265): Extended attributes in extensions couldn't be fetched unless they had non-ascii names.
 
 ## Compatibility with Prior Versions
 
-This version is fully backwards-compatible with version 2.1.1.  As mentioned above, there are new configuration settings for filtering group members and controlling network behavior, and there is a new command-line option for controlling the update strategy.  See [the docs](https://adobe-apiplatform.github.io/user-sync.py) for full details about configuration.
+There are no functional changes from prior versions.
 
 ## Known Issues
 

setup.py

@@ -48,7 +48,7 @@
           'pycryptodome',
           'pyldap==2.4.37',
           'PyYAML',
-          'umapi-client>=2.5',
+          'umapi-client>=2.7',
           'psutil',
           'keyring',
           'six'

user_sync/app.py

@@ -276,7 +276,7 @@ def create_config_loader_options(args):
             compiled_expression = re.compile(r'\A' + username_filter_pattern + r'\Z', re.IGNORECASE)
         except Exception as e:
             raise AssertionException("Bad regular expression for --user-filter: %s reason: %s" %
-                                     (username_filter_pattern, e.message))
+                                     (username_filter_pattern, e))
         config_options['username_filter_regex'] = compiled_expression
 
     # --adobe-only-user-action

user_sync/connector/directory_ldap.py

@@ -200,6 +200,7 @@ def iter_users(self, users_filter, extended_attributes):
         user_attribute_names.extend(self.user_username_formatter.get_attribute_names())
         user_attribute_names.extend(self.user_domain_formatter.get_attribute_names())
 
+        extended_attributes = [six.text_type(attr) for attr in extended_attributes]
         extended_attributes = list(set(extended_attributes) - set(user_attribute_names))
         user_attribute_names.extend(extended_attributes)
 
@@ -336,7 +337,7 @@ def __init__(self, string_format):
         else:
             string_format = six.text_type(string_format)    # force unicode so attribute values are unicode
             formatter = string.Formatter()
-            attribute_names = [item[1] for item in formatter.parse(string_format) if item[1]]
+            attribute_names = [six.text_type(item[1]) for item in formatter.parse(string_format) if item[1]]
         self.string_format = string_format
         self.attribute_names = attribute_names
 
@@ -356,7 +357,7 @@ def generate_value(self, record):
         if self.string_format is not None:
             values = {}
             for attribute_name in self.attribute_names:
-                value = self.get_attribute_value(record, attribute_name)
+                value = self.get_attribute_value(record, attribute_name, first_only=True)
                 if value is None:
                     values = None
                     break
@@ -366,17 +367,20 @@ def generate_value(self, record):
         return result, attribute_name
 
     @classmethod
-    def get_attribute_value(cls, attributes, attribute_name):
+    def get_attribute_value(cls, attributes, attribute_name, first_only=False):
         """
         The attribute value type must be decodable (str in py2, bytes in py3)
         :type attributes: dict
         :type attribute_name: unicode
+        :type first_only: bool
         """
-        if attribute_name in attributes:
-            attribute_value = attributes[attribute_name]
-            if len(attribute_value) > 0:
-                try:
-                    return attribute_value[0].decode(cls.encoding)
-                except UnicodeError as e:
-                    raise AssertionException("Encoding error in value of attribute '%s': %s" % (attribute_name, e))
+        attribute_values = attributes.get(attribute_name)
+        if attribute_values:
+            try:
+                if first_only or len(attribute_values) == 1:
+                    return attribute_values[0].decode(cls.encoding)
+                else:
+                    return [val.decode(cls.encoding) for val in attribute_values]
+            except UnicodeError as e:
+                raise AssertionException("Encoding error in value of attribute '%s': %s" % (attribute_name, e))
         return None

user_sync/connector/umapi.py

@@ -299,11 +299,10 @@ def create_action(self, commands):
             identity_type = user_sync.identity_type.FEDERATED_IDENTITY_TYPE
         try:
             umapi_identity_type = umapi_client.IdentityTypes[identity_type]
-        except KeyError:
-            umapi_identity_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE
-
-        action = umapi_client.UserAction(umapi_identity_type, email, username, domain,
-                                         requestID=self.get_next_request_id())
+            action = umapi_client.UserAction(umapi_identity_type, email, username, domain,
+                                             requestID=self.get_next_request_id())
+        except ValueError as e:
+            raise AssertionException("Error creating umapi Action: %s" % e)
         for command in commands.do_list:
             command_name, command_param = command
             command_function = getattr(action, command_name)
@@ -354,7 +353,7 @@ def process_sent_items(self, total_sent, batch_error=None):
         """
         Note items as sent, log any processing errors, and invoke any callbacks
         :param total_sent: number of sent items from queue, must be >= 0
-        :param batch_error: a batch-level error that affected all items, if there was one
+        :param batch_error: exception for a batch-level error that affected all items, if there was one
         :return: 
         """
         # update queue
@@ -383,5 +382,5 @@ def process_sent_items(self, total_sent, batch_error=None):
                 callback({
                     "action": action,
                     "is_success": not batch_error and not errors,
-                    "errors": batch_error or errors
+                    "errors": [batch_error] if batch_error else errors
                 })

user_sync/version.py

@@ -18,4 +18,4 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-__version__ = '2.2.1'
+__version__ = '2.2.1rc1'