fix #299: allow specifying command-line values using config file

This required a number of coordinated changes to app startup:

- Argument processing (not parsing) was moved to the config module, including the expansion of the `mapped` users option the actual directory groups.
- Separation was done of  bootstrap options, invocation options, and rule-processing options, so each could be given a single point of definition with app defaults.
- A new, optional `invocation_defaults` section was added to the main configuration file so the default values for all arguments could be specified there.

These changes led to resolution to a number of long-standing issues:

- There are no longer duplicate places where options have to be specified to get them added as rule-processing options and/or command-line arguments.
- The top-level logic doesn't get involved with the interaction of command-line arguments and configuration files, or be sensitive to the timing of configuration loading and argument processing.
- The logging of invocation now shows both the command-line arguments and their actual effect on the invocation options (including the formerly-hidden invocation options that actually override rule processing defaults).
- The names of various internal options are now matched to the names of the command-line arguments that override them, so the code is more readable throughout.

Author: adobeDan

examples/config files - basic/1 user-sync-config.yml

@@ -8,6 +8,65 @@
 # this sample and then customizing it.  Feel free to remove extraneous commentary
 # when you do your customization; doing so can greatly increase legibility.
 
+# The invocation_defaults section controls the default values used
+# for command-line arguments.  (Of course, these cannot be used to
+# change defaults for how the configuration files are read.)  This
+# entire section is optional, as are each of its entries.  But once
+# you standardize on specific argument values for all your runs,
+# putting them here can be an easy way to reliably use them, even
+# when running interactively rather than from a script.
+#
+# For arguments that take multiple arguments, such as --connector
+# and --users, the two-argument case should be specified as a list,
+# either using yaml compact notation:
+#   connector: [csv, users-file.csv]
+# or using yaml multi-line notation:
+#   connector:
+#     - csv
+#     - users-file.csv
+# The one-argument case can either use a string:
+#   connector: ldap
+# or a compact list of one value:
+#   connector: [ldap]
+# or a multi-line list of one value:
+#   connector:
+#     - ldap
+#
+# For arguments that are booleans, yaml syntax allows specifying their
+# default values using either True/False or Yes/No (case insensitive).
+#
+# If you specify filenames as part of your default values, they
+# are treated as if they were specified on the command line; that is,
+# they are interpreted relative to the User Sync current directory
+# (rather than the directory containing the configuration file).
+invocation_defaults:
+  # For argument --adobe-only-user-action, the default is 'preserve'.
+  adobe_only_user_action: preserve
+  # For argument --adobe-only-user-list, the default is empty (no value).
+  adobe_only_user_list:
+  # For argument --connector, the default is 'ldap'.
+  connector: ldap
+  # For argument --process-groups, the default is False (don't process).
+  # If you set this default to True, you can supply the argument
+  # --no-process-groups to override the default.
+  process_groups: No
+  # For argument --strategy, the default is 'sync'.
+  strategy: sync
+  # For argument --test-mode (or -t), the default is False (live run).
+  # if you set this default to True, you can supply the argument
+  # --no-test-mode (or -T) to override the default.
+  test_mode: No
+  # For argument --update-user-info, the default is False (don't update).
+  # If you set this default to True, you can supply the argument
+  # --no-update-user-info to override the default.
+  update_user_info: No
+  # For argument --user-filter, the default is empty (no value).
+  # Because regular expression notation uses special characters,
+  # any default you set should almost certainly be single-quoted.
+  user_filter:
+  # For argument --users, the default is 'all'.
+  users: all
+
 # The adobe_users section controls connection to the Adobe UM API endpoints
 # and also which users on the Adobe side are eligible to be updated.
 adobe_users:

tests/connector/directory_okta_test.py

@@ -161,7 +161,7 @@ def test_found_user_single_group(self, mock_members):
         test_user = tests.helper.create_test_user_uid()
         mock_members.return_value = [test_user]
         directory = self.directory
-        results = directory.load_users_and_groups(groups, [])
+        results = directory.load_users_and_groups(groups, [], False)
         self.assertEqual(len(list(results)), 1)
 
     @mock.patch('user_sync.connector.directory_okta.OktaDirectoryConnector.iter_group_members')
@@ -173,7 +173,7 @@ def test_found_user_multiple_groups(self, mock_members):
             test_users.append([tests.helper.create_test_user_uid()])
         mock_members.side_effect = test_users
         directory = self.directory
-        results = directory.load_users_and_groups(groups, [])
+        results = directory.load_users_and_groups(groups, [], False)
         self.assertEqual(len(list(results)), 2)
 
     @mock.patch('user_sync.connector.directory_okta.OktaDirectoryConnector.iter_group_members')
@@ -187,7 +187,7 @@ def test_found_user_single_group_multiple_user(self, mock_members):
             user_count = user_count + 1
         mock_members.return_value = test_users
         directory = self.directory
-        results = directory.load_users_and_groups(groups, [])
+        results = directory.load_users_and_groups(groups, [], False)
         self.assertEqual(len(list(results)), 5)
 
     @mock.patch('user_sync.connector.directory_okta.OktaDirectoryConnector.iter_group_members')
@@ -204,7 +204,7 @@ def test_found_user_multiple_groups_multiple_user(self, mock_members):
             total_test_users.append(test_users)
         mock_members.side_effect = total_test_users
         directory = self.directory
-        results = directory.load_users_and_groups(groups, [])
+        results = directory.load_users_and_groups(groups, [], False)
         self.assertEqual(len(list(results)), 10)
 
     @mock.patch('user_sync.connector.directory_okta.OktaDirectoryConnector.iter_group_members')
@@ -214,7 +214,7 @@ def test_no_user_single_group(self, mock_members):
         test_users = []
         mock_members.return_value = test_users
         directory = self.directory
-        results = directory.load_users_and_groups(groups, [])
+        results = directory.load_users_and_groups(groups, [], False)
         self.assertEqual(len(list(results)), 0)
 
     @mock.patch('user_sync.connector.directory_okta.OktaDirectoryConnector.iter_group_members')
@@ -224,7 +224,7 @@ def test_no_user_multiple_groups(self, mock_members):
         total_test_users = [[], []]
         mock_members.side_effect = total_test_users
         directory = self.directory
-        results = directory.load_users_and_groups(groups, [])
+        results = directory.load_users_and_groups(groups, [], False)
         self.assertEqual(len(list(results)), 0)
 
 
@@ -236,7 +236,7 @@ def test_same_user_in_multiple_groups(self, mock_members):
         test_user = tests.helper.create_test_user_uid()
         mock_members.side_effect = [[test_user], [copy.deepcopy(test_user)]]
         directory = self.directory
-        result = directory.load_users_and_groups(groups, [])
+        result = directory.load_users_and_groups(groups, [], False)
         result_list = list(result)
         self.assertEqual(result_list[0]['groups'], ['group1','group2'])
 

tests/rules_test.py

@@ -69,7 +69,7 @@ def mock_load_users_and_groups(groups=None, extended_attributes=None, all_users=
             secondary_1_umapi_name: mock_secondary_umapi_connector
         })
 
-        rule_processor = user_sync.rules.RuleProcessor({'manage_groups': True})
+        rule_processor = user_sync.rules.RuleProcessor({'process_groups': True})
         rule_processor.run(directory_groups, mock_directory_connector, umapi_connectors)
 
         rule_options = rule_processor.options
@@ -142,7 +142,7 @@ def mock_load_users_and_groups(groups=None, extended_attributes=None, all_users=
             secondary_1_umapi_name: mock_secondary_umapi_connector
         })
 
-        rule_processor = user_sync.rules.RuleProcessor({'manage_groups': True,
+        rule_processor = user_sync.rules.RuleProcessor({'process_groups': True,
                                                         'strategy': 'push',
                                                         'update_user_info': True,
                                                         })