fix #237: all encodings default to utf8

* Testing revealed a few edge cases in config file reading and CSV file reading, also fixed.
* Updated the release notes.

Author: adobeDan

RELEASE_NOTES.md

@@ -12,6 +12,8 @@ These notes apply to v2.2rc1 of 2017-06-18.
 
 [#236](https://github.com/adobe-apiplatform/user-sync.py/issues/236): Directory users can now be pushed directly to Adobe, rather than synchronized with a fetch of Adobe users.  A new command-line argument `--strategy push` (as opposed to the default `--strategy sync`) controls this.
 
+[#237](https://github.com/adobe-apiplatform/user-sync.py/issues/237): The default encoding for all inputs (config files, CSV files, LDAP attribute values) is now assumed to be `utf8` rather than ASCII.  This is a backward-compatible change that makes it unnecessary (but still allowed) to specify `utf8` explicitly.
+
 ## Bug Fixes
 
 This release contains bug fixes for:

examples/config files - basic/3 connector-ldap.yml

@@ -83,9 +83,9 @@ group_member_filter_format: "(memberOf={group_dn})"
 # string_encoding specifies the Unicode string encoding used by the directory.
 # All values retrieved from the directory are converted to Unicode before being
 # sent to or compared with values on the Adobe side, to avoid encoding issues.
-# The value must be a Python codec name or alias, such as 'latin1' or 'utf-8.
+# The value must be a Python codec name or alias, such as 'latin1' or 'big5'.
 # See https://docs.python.org/2/library/codecs.html#standard-encodings for details.
-#string_encoding: utf-8
+#string_encoding: utf8
 
 # (optional) user_identity_type_format (no default)
 # user_identity_type_format specifies how to construct a user's desired identity

user_sync/app.py

@@ -102,11 +102,10 @@ def process_args():
                              "users by also including --adobe-only-user-action and one of its arguments",
                         metavar='input_path', dest='stray_list_input_path')
     parser.add_argument('--config-file-encoding',
-                        help="config files are expected to contain only ASCII characters; if you "
-                             "use an extended character set (e.g., to specify group names), then "
-                             "specify the encoding of your configuration files with this argument. "
+                        help="configuration files are expected to be utf8-encoded (which includes ascii); if you "
+                             "use a different character set, then specify it with this argument. "
                              "All encoding names understood by Python are allowed.",
-                        dest='encoding_name', default='ascii')
+                        dest='encoding_name', default='utf8')
     parser.add_argument('--strategy',
                         help="whether to fetch and sync the Adobe directory against the customer directory "
                              "or just to push each customer user to the Adobe side.  Default is to fetch and sync.",

user_sync/config.py

@@ -46,7 +46,7 @@ def __init__(self, caller_options):
         self.options = options = {
             # these are in alphabetical order!  Always add new ones that way!
             'delete_strays': False,
-            'config_file_encoding': 'ascii',
+            'config_file_encoding': 'utf8',
             'directory_connector_module_name': None,
             'directory_connector_overridden_options': None,
             'directory_group_filter': None,
@@ -616,8 +616,8 @@ class ConfigFileLoader:
     Loads config files and does pathname expansion on settings that refer to files or directories
     """
     # config files can contain Unicode characters, so an encoding for them
-    # can be specified as a command line argument.  This defaults to ascii.
-    config_encoding = 'ascii'
+    # can be specified as a command line argument.  This defaults to utf8.
+    config_encoding = 'utf8'
 
     # key_paths in the root configuration file that should have filename values
     # mapped to their value options.  See load_from_yaml for the option meanings.
@@ -724,6 +724,12 @@ def load_from_yaml(cls, filename, path_keys):
                 raise AssertionException("Error parsing configuration file '%s': %s" % (cls.filepath, e))
 
         # process the content of the dict
+        if yml is None:
+            # empty YML files are parsed as None
+            yml = {}
+        elif not isinstance(yml, dict):
+            # malformed YML files produce a non-dictionary
+            raise AssertionException("Configuration file '%s' does not contain settings" % cls.filepath)
         for path_key, options in six.iteritems(path_keys):
             cls.key_path = path_key
             keys = path_key.split('/')

user_sync/connector/directory_csv.py

@@ -60,7 +60,7 @@ def __init__(self, caller_options):
         caller_config = user_sync.config.DictConfig('%s configuration' % self.name, caller_options)
         builder = user_sync.config.OptionsBuilder(caller_config)
         builder.set_string_value('delimiter', None)
-        builder.set_string_value('string_encoding', 'utf-8')
+        builder.set_string_value('string_encoding', 'utf8')
         builder.set_string_value('first_name_column_name', 'firstname')
         builder.set_string_value('last_name_column_name', 'lastname')
         builder.set_string_value('email_column_name', 'email')

user_sync/connector/directory_ldap.py

@@ -323,7 +323,7 @@ def iter_search_result(self, base_dn, scope, filter_string, attributes):
 
 
 class LDAPValueFormatter(object):
-    encoding = 'utf-8'
+    encoding = 'utf8'
 
     def __init__(self, string_format):
         """

user_sync/helper.py

@@ -86,7 +86,7 @@ def guess_delimiter_from_filename(filename):
         return '\t'
 
     @classmethod
-    def read_csv_rows(cls, file_path, recognized_column_names=None, logger=None, encoding=None, delimiter=None):
+    def read_csv_rows(cls, file_path, recognized_column_names=None, logger=None, encoding='utf8', delimiter=None):
         """
         :type file_path: str
         :type recognized_column_names: list(str)
@@ -115,14 +115,14 @@ def read_csv_rows(cls, file_path, recognized_column_names=None, logger=None, enc
                         # in py2, we need to decode both the column names *and* the values
                         newrow = {}
                         for key, val in six.iteritems(row):
-                            newrow[key.decode(encoding, 'strict')] = val.decode(encoding, 'strict')
+                            newrow[key.decode(encoding, 'strict')] = val.decode(encoding, 'strict') if val else None
                         row = newrow
                     yield row
             except UnicodeError as e:
                 raise AssertionException("Encoding error in file '%s': %s" % (file_path, e))
 
     @classmethod
-    def write_csv_rows(cls, file_path, field_names, rows, encoding='ascii', delimiter=None):
+    def write_csv_rows(cls, file_path, field_names, rows, encoding='utf8', delimiter=None):
         """
         :type file_path: str
         :type field_names: list(str)