Break the installation defaults out into their structure.

Add tests that assert the consistency of the options that are accepted
by the generateconfs command line with the internal library function.
Additionally assert that the library routine itself matches the defaults
structure thus making the _structure_ the definitive source of truth.

Doing so highlighted the following missing command line options which
are added as of this commit:
--seafile_secret
--seafile_ccnetid

Author: albu-diku

mig/install/__init__.py

@@ -48,7 +48,7 @@
 # NOTE: moved mig imports into try/except to avoid autopep8 moving to top!
 try:
     from mig.shared.defaults import MIG_BASE, MIG_ENV
-    from mig.shared.install import generate_confs
+    from mig.shared.install import generate_confs, _GENERATE_CONFS_PARAMETERS
 except ImportError:
     print("ERROR: the migrid modules must be in PYTHONPATH")
     sys.exit(1)
@@ -65,7 +65,7 @@ def usage(options):
 ''' % (sys.argv[0], '\n'.join(lines)))
 
 
-def main(argv, _generate_confs=generate_confs, _print=print):
+def _make_parameters():
     str_names = [
         'source',
         'destination',
@@ -248,6 +248,8 @@ def main(argv, _generate_confs=generate_confs, _print=print):
         'openid_port',
         'openid_show_port',
         'openid_session_lifetime',
+        'seafile_secret',
+        'seafile_ccnetid',
         'seafile_seahub_port',
         'seafile_seafhttp_port',
         'seafile_client_port',
@@ -305,7 +307,15 @@ def main(argv, _generate_confs=generate_confs, _print=print):
         'io_account_expire',
         'gdp_email_notify',
     ]
-    names = str_names + int_names + bool_names
+    return (str_names, int_names, bool_names)
+
+
+str_names, int_names, bool_names = _make_parameters()
+_PARAMETERS = str_names + int_names + bool_names
+
+
+def main(argv, _generate_confs=generate_confs, _print=print):
+    names = _PARAMETERS
     settings, options, result = {}, {}, {}
     default_val = 'DEFAULT'
     # Force values to expected type

mig/install/generateconfs.py

@@ -34,6 +34,7 @@
 from past.builtins import basestring
 
 import codecs
+import inspect
 import io
 import sys
 # NOTE: StringIO is only available in python2
@@ -93,6 +94,15 @@ def ensure_native_string(string_or_bytes):
     return textual_output
 
 
+def inspect_args(func):
+    """Wrapper to return the arguments of a function."""
+
+    if PY2:
+        return inspect.getargspec(func).args
+    else:
+        return inspect.getfullargspec(func).args
+
+
 def NativeStringIO(initial_value=''):
     """Mock StringIO pseudo-class to create a StringIO matching the native
     string coding form. That is a BytesIO with utf8 on python 2 and unicode

mig/shared/compat.py

@@ -50,15 +50,16 @@
 import sys
 
 from mig.shared.base import force_native_str, force_utf8
+from mig.shared.compat import ensure_native_string, inspect_args, \
+    SimpleNamespace
 from mig.shared.defaults import default_http_port, default_https_port, \
     auth_openid_mig_db, auth_openid_ext_db, MIG_BASE, STRONG_TLS_CIPHERS, \
     STRONG_TLS_CURVES, STRONG_SSH_HOSTKEYALGOS, STRONG_SSH_KEXALGOS, \
     STRONG_SSH_CIPHERS, STRONG_SSH_MACS, LEGACY_SSH_HOSTKEYALGOS, \
     LEGACY_SSH_KEXALGOS, LEGACY_SSH_CIPHERS, LEGACY_SSH_MACS, \
     FALLBACK_SSH_HOSTKEYALGOS, FALLBACK_SSH_KEXALGOS, FALLBACK_SSH_CIPHERS, \
-    FALLBACK_SSH_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, keyword_any, \
-    keyword_auto
-from mig.shared.compat import ensure_native_string
+    FALLBACK_SSH_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, \
+    keyword_any, keyword_auto
 from mig.shared.fileio import read_file, read_file_lines, write_file, \
     write_file_lines
 from mig.shared.htmlgen import menu_items
@@ -299,16 +300,7 @@ def template_remove(template_file, remove_pattern):
 ]
 
 
-def generate_confs(
-    generateconfs_output_path,
-    # NOTE: make sure command line args with white-space are properly wrapped
-    generateconfs_command=subprocess.list2cmdline(sys.argv),
-    source=keyword_auto,
-    destination=keyword_auto,
-    user=keyword_auto,
-    group=keyword_auto,
-    timezone=keyword_auto,
-    destination_suffix="",
+_DEFAULTS = SimpleNamespace(
     base_fqdn='',
     public_fqdn='',
     public_alias_fqdn='',
@@ -538,6 +530,248 @@ def generate_confs(
     datasafety_link='',
     datasafety_text='',
     wwwserve_max_bytes=-1,
+)
+
+
+def generate_confs(
+    generateconfs_output_path,
+    # NOTE: make sure command line args with white-space are properly wrapped
+    generateconfs_command=subprocess.list2cmdline(sys.argv),
+    source=keyword_auto,
+    destination=keyword_auto,
+    user=keyword_auto,
+    group=keyword_auto,
+    timezone=keyword_auto,
+    destination_suffix="",
+    base_fqdn=_DEFAULTS.base_fqdn,
+    public_fqdn=_DEFAULTS.public_fqdn,
+    public_alias_fqdn=_DEFAULTS.public_alias_fqdn,
+    public_sec_fqdn=_DEFAULTS.public_sec_fqdn,
+    status_alias_fqdn=_DEFAULTS.status_alias_fqdn,
+    mig_cert_fqdn=_DEFAULTS.mig_cert_fqdn,
+    ext_cert_fqdn=_DEFAULTS.ext_cert_fqdn,
+    mig_oid_fqdn=_DEFAULTS.mig_oid_fqdn,
+    ext_oid_fqdn=_DEFAULTS.ext_oid_fqdn,
+    mig_oidc_fqdn=_DEFAULTS.mig_oidc_fqdn,
+    ext_oidc_fqdn=_DEFAULTS.ext_oidc_fqdn,
+    sid_fqdn=_DEFAULTS.sid_fqdn,
+    io_fqdn=_DEFAULTS.io_fqdn,
+    cert_fqdn_extras=_DEFAULTS.cert_fqdn_extras,
+    cloud_fqdn=_DEFAULTS.cloud_fqdn,
+    seafile_fqdn=_DEFAULTS.seafile_fqdn,
+    seafile_base=_DEFAULTS.seafile_base,
+    seafmedia_base=_DEFAULTS.seafmedia_base,
+    seafhttp_base=_DEFAULTS.seafhttp_base,
+    openid_address=_DEFAULTS.openid_address,
+    sftp_address=_DEFAULTS.sftp_address,
+    sftp_subsys_address=_DEFAULTS.sftp_subsys_address,
+    ftps_address=_DEFAULTS.ftps_address,
+    davs_address=_DEFAULTS.davs_address,
+    jupyter_services=_DEFAULTS.jupyter_services,
+    jupyter_services_desc=_DEFAULTS.jupyter_services_desc,
+    jupyter_services_enable_proxy_https=_DEFAULTS.jupyter_services_enable_proxy_https,
+    jupyter_services_proxy_config=_DEFAULTS.jupyter_services_proxy_config,
+    cloud_services=_DEFAULTS.cloud_services,
+    cloud_services_desc=_DEFAULTS.cloud_services_desc,
+    apache_version=_DEFAULTS.apache_version,
+    apache_etc=_DEFAULTS.apache_etc,
+    apache_run=_DEFAULTS.apache_run,
+    apache_lock=_DEFAULTS.apache_lock,
+    apache_log=_DEFAULTS.apache_log,
+    apache_worker_procs=_DEFAULTS.apache_worker_procs,
+    openssh_version=_DEFAULTS.openssh_version,
+    mig_code=_DEFAULTS.mig_code,
+    mig_state=_DEFAULTS.mig_state,
+    mig_certs=_DEFAULTS.mig_certs,
+    auto_add_cert_user=_DEFAULTS.auto_add_cert_user,
+    auto_add_oid_user=_DEFAULTS.auto_add_oid_user,
+    auto_add_oidc_user=_DEFAULTS.auto_add_oidc_user,
+    auto_add_filter_fields=_DEFAULTS.auto_add_filter_fields,
+    auto_add_filter_method=_DEFAULTS.auto_add_filter_method,
+    auto_add_user_permit=_DEFAULTS.auto_add_user_permit,
+    auto_add_user_with_peer=_DEFAULTS.auto_add_user_with_peer,
+    cert_valid_days=_DEFAULTS.cert_valid_days,
+    oid_valid_days=_DEFAULTS.oid_valid_days,
+    oidc_valid_days=_DEFAULTS.oidc_valid_days,
+    generic_valid_days=_DEFAULTS.generic_valid_days,
+    enable_migadmin=_DEFAULTS.enable_migadmin,
+    enable_sftp=_DEFAULTS.enable_sftp,
+    enable_sftp_subsys=_DEFAULTS.enable_sftp_subsys,
+    sftp_subsys_auth_procs=_DEFAULTS.sftp_subsys_auth_procs,
+    enable_davs=_DEFAULTS.enable_davs,
+    enable_ftps=_DEFAULTS.enable_ftps,
+    enable_wsgi=_DEFAULTS.enable_wsgi,
+    wsgi_procs=_DEFAULTS.wsgi_procs,
+    enable_gdp=_DEFAULTS.enable_gdp,
+    enable_jobs=_DEFAULTS.enable_jobs,
+    enable_resources=_DEFAULTS.enable_resources,
+    enable_workflows=_DEFAULTS.enable_workflows,
+    enable_events=_DEFAULTS.enable_events,
+    enable_sharelinks=_DEFAULTS.enable_sharelinks,
+    enable_transfers=_DEFAULTS.enable_transfers,
+    enable_freeze=_DEFAULTS.enable_freeze,
+    enable_sandboxes=_DEFAULTS.enable_sandboxes,
+    enable_vmachines=_DEFAULTS.enable_vmachines,
+    enable_preview=_DEFAULTS.enable_preview,
+    enable_jupyter=_DEFAULTS.enable_jupyter,
+    enable_cloud=_DEFAULTS.enable_cloud,
+    enable_hsts=_DEFAULTS.enable_hsts,
+    enable_vhost_certs=_DEFAULTS.enable_vhost_certs,
+    enable_verify_certs=_DEFAULTS.enable_verify_certs,
+    enable_seafile=_DEFAULTS.enable_seafile,
+    enable_duplicati=_DEFAULTS.enable_duplicati,
+    enable_crontab=_DEFAULTS.enable_crontab,
+    enable_notify=_DEFAULTS.enable_notify,
+    enable_imnotify=_DEFAULTS.enable_imnotify,
+    enable_dev_accounts=_DEFAULTS.enable_dev_accounts,
+    enable_twofactor=_DEFAULTS.enable_twofactor,
+    twofactor_mandatory_protos=_DEFAULTS.twofactor_mandatory_protos,
+    enable_twofactor_strict_address=_DEFAULTS.enable_twofactor_strict_address,
+    twofactor_auth_apps=_DEFAULTS.twofactor_auth_apps,
+    enable_peers=_DEFAULTS.enable_peers,
+    peers_mandatory=_DEFAULTS.peers_mandatory,
+    peers_explicit_fields=_DEFAULTS.peers_explicit_fields,
+    peers_contact_hint=_DEFAULTS.peers_contact_hint,
+    enable_cracklib=_DEFAULTS.enable_cracklib,
+    enable_openid=_DEFAULTS.enable_openid,
+    enable_gravatars=_DEFAULTS.enable_gravatars,
+    enable_sitestatus=_DEFAULTS.enable_sitestatus,
+    enable_quota=_DEFAULTS.enable_quota,
+    prefer_python3=_DEFAULTS.prefer_python3,
+    io_account_expire=_DEFAULTS.io_account_expire,
+    gdp_email_notify=_DEFAULTS.gdp_email_notify,
+    user_interface=_DEFAULTS.user_interface,
+    mig_oid_title=_DEFAULTS.mig_oid_title,
+    mig_oid_provider=_DEFAULTS.mig_oid_provider,
+    ext_oid_title=_DEFAULTS.ext_oid_title,
+    ext_oid_provider=_DEFAULTS.ext_oid_provider,
+    mig_oidc_title=_DEFAULTS.mig_oidc_title,
+    mig_oidc_provider_meta_url=_DEFAULTS.mig_oidc_provider_meta_url,
+    ext_oidc_title=_DEFAULTS.ext_oidc_title,
+    ext_oidc_provider_meta_url=_DEFAULTS.ext_oidc_provider_meta_url,
+    ext_oidc_provider_issuer=_DEFAULTS.ext_oidc_provider_issuer,
+    ext_oidc_provider_authorization_endpoint=_DEFAULTS.ext_oidc_provider_authorization_endpoint,
+    ext_oidc_provider_verify_cert_files=_DEFAULTS.ext_oidc_provider_verify_cert_files,
+    ext_oidc_provider_token_endpoint=_DEFAULTS.ext_oidc_provider_token_endpoint,
+    ext_oidc_provider_token_endpoint_auth=_DEFAULTS.ext_oidc_provider_token_endpoint_auth,
+    ext_oidc_provider_user_info_endpoint=_DEFAULTS.ext_oidc_provider_user_info_endpoint,
+    ext_oidc_scope=_DEFAULTS.ext_oidc_scope,
+    ext_oidc_user_info_token_method=_DEFAULTS.ext_oidc_user_info_token_method,
+    ext_oidc_public_key_files=_DEFAULTS.ext_oidc_public_key_files,
+    ext_oidc_private_key_files=_DEFAULTS.ext_oidc_private_key_files,
+    ext_oidc_response_type=_DEFAULTS.ext_oidc_response_type,
+    ext_oidc_response_mode=_DEFAULTS.ext_oidc_response_mode,
+    ext_oidc_client_id=_DEFAULTS.ext_oidc_client_id,
+    ext_oidc_client_name=_DEFAULTS.ext_oidc_client_name,
+    ext_oidc_pkce_method=_DEFAULTS.ext_oidc_pkce_method,
+    ext_oidc_id_token_encrypted_response_alg=_DEFAULTS.ext_oidc_id_token_encrypted_response_alg,
+    ext_oidc_id_token_encrypted_response_enc=_DEFAULTS.ext_oidc_id_token_encrypted_response_enc,
+    ext_oidc_user_info_signed_response_alg=_DEFAULTS.ext_oidc_user_info_signed_response_alg,
+    ext_oidc_cookie_same_site=_DEFAULTS.ext_oidc_cookie_same_site,
+    ext_oidc_pass_cookies=_DEFAULTS.ext_oidc_pass_cookies,
+    ext_oidc_remote_user_claim=_DEFAULTS.ext_oidc_remote_user_claim,
+    ext_oidc_pass_claim_as=_DEFAULTS.ext_oidc_pass_claim_as,
+    ext_oidc_rewrite_cookie=_DEFAULTS.ext_oidc_rewrite_cookie,
+    dhparams_path=_DEFAULTS.dhparams_path,
+    daemon_keycert=_DEFAULTS.daemon_keycert,
+    daemon_keycert_sha256=_DEFAULTS.daemon_keycert_sha256,
+    daemon_pubkey=_DEFAULTS.daemon_pubkey,
+    daemon_pubkey_from_dns=_DEFAULTS.daemon_pubkey_from_dns,
+    daemon_pubkey_md5=_DEFAULTS.daemon_pubkey_md5,
+    daemon_pubkey_sha256=_DEFAULTS.daemon_pubkey_sha256,
+    daemon_show_address=_DEFAULTS.daemon_show_address,
+    alias_field=_DEFAULTS.alias_field,
+    peers_permit=_DEFAULTS.peers_permit,
+    vgrid_creators=_DEFAULTS.vgrid_creators,
+    vgrid_managers=_DEFAULTS.vgrid_managers,
+    signup_methods=_DEFAULTS.signup_methods,
+    login_methods=_DEFAULTS.login_methods,
+    digest_salt=_DEFAULTS.digest_salt,
+    crypto_salt=_DEFAULTS.crypto_salt,
+    csrf_protection=_DEFAULTS.csrf_protection,
+    password_policy=_DEFAULTS.password_policy,
+    password_legacy_policy=_DEFAULTS.password_legacy_policy,
+    hg_path=_DEFAULTS.hg_path,
+    hgweb_scripts=_DEFAULTS.hgweb_scripts,
+    trac_admin_path=_DEFAULTS.trac_admin_path,
+    trac_ini_path=_DEFAULTS.trac_ini_path,
+    public_port=_DEFAULTS.public_port,
+    public_http_port=_DEFAULTS.public_http_port,
+    public_https_port=_DEFAULTS.public_https_port,
+    mig_cert_port=_DEFAULTS.mig_cert_port,
+    ext_cert_port=_DEFAULTS.ext_cert_port,
+    mig_oid_port=_DEFAULTS.mig_oid_port,
+    ext_oid_port=_DEFAULTS.ext_oid_port,
+    mig_oidc_port=_DEFAULTS.mig_oidc_port,
+    ext_oidc_port=_DEFAULTS.ext_oidc_port,
+    sid_port=_DEFAULTS.sid_port,
+    sftp_port=_DEFAULTS.sftp_port,
+    sftp_subsys_port=_DEFAULTS.sftp_subsys_port,
+    sftp_show_port=_DEFAULTS.sftp_show_port,
+    sftp_max_sessions=_DEFAULTS.sftp_max_sessions,
+    davs_port=_DEFAULTS.davs_port,
+    davs_show_port=_DEFAULTS.davs_show_port,
+    ftps_ctrl_port=_DEFAULTS.ftps_ctrl_port,
+    ftps_ctrl_show_port=_DEFAULTS.ftps_ctrl_show_port,
+    ftps_pasv_ports=_DEFAULTS.ftps_pasv_ports,
+    openid_port=_DEFAULTS.openid_port,
+    openid_show_port=_DEFAULTS.openid_show_port,
+    openid_session_lifetime=_DEFAULTS.openid_session_lifetime,
+    seafile_secret=_DEFAULTS.seafile_secret,
+    seafile_ccnetid=_DEFAULTS.seafile_ccnetid,
+    seafile_seahub_port=_DEFAULTS.seafile_seahub_port,
+    seafile_seafhttp_port=_DEFAULTS.seafile_seafhttp_port,
+    seafile_client_port=_DEFAULTS.seafile_client_port,
+    seafile_quota=_DEFAULTS.seafile_quota,
+    seafile_ro_access=_DEFAULTS.seafile_ro_access,
+    public_use_https=_DEFAULTS.public_use_https,
+    user_clause=_DEFAULTS.user_clause,
+    group_clause=_DEFAULTS.group_clause,
+    listen_clause=_DEFAULTS.listen_clause,
+    serveralias_clause=_DEFAULTS.serveralias_clause,
+    distro=_DEFAULTS.distro,
+    autolaunch_page=_DEFAULTS.autolaunch_page,
+    landing_page=_DEFAULTS.landing_page,
+    skin=_DEFAULTS.skin,
+    title=_DEFAULTS.title,
+    short_title=_DEFAULTS.short_title,
+    extra_userpage_scripts=_DEFAULTS.extra_userpage_scripts,
+    extra_userpage_styles=_DEFAULTS.extra_userpage_styles,
+    external_doc=_DEFAULTS.external_doc,
+    vgrid_label=_DEFAULTS.vgrid_label,
+    secscan_addr=_DEFAULTS.secscan_addr,
+    default_menu=_DEFAULTS.default_menu,
+    user_menu=_DEFAULTS.user_menu,
+    collaboration_links=_DEFAULTS.collaboration_links,
+    default_vgrid_links=_DEFAULTS.default_vgrid_links,
+    advanced_vgrid_links=_DEFAULTS.advanced_vgrid_links,
+    support_email=_DEFAULTS.support_email,
+    admin_email=_DEFAULTS.admin_email,
+    admin_list=_DEFAULTS.admin_list,
+    smtp_server=_DEFAULTS.smtp_server,
+    smtp_sender=_DEFAULTS.smtp_sender,
+    permanent_freeze=_DEFAULTS.permanent_freeze,
+    log_level=_DEFAULTS.log_level,
+    freeze_to_tape=_DEFAULTS.freeze_to_tape,
+    status_system_match=_DEFAULTS.status_system_match,
+    storage_protocols=_DEFAULTS.storage_protocols,
+    duplicati_protocols=_DEFAULTS.duplicati_protocols,
+    imnotify_address=_DEFAULTS.imnotify_address,
+    imnotify_channel=_DEFAULTS.imnotify_channel,
+    imnotify_username=_DEFAULTS.imnotify_username,
+    imnotify_password=_DEFAULTS.imnotify_password,
+    gdp_data_categories=_DEFAULTS.gdp_data_categories,
+    gdp_id_scramble=_DEFAULTS.gdp_id_scramble,
+    gdp_path_scramble=_DEFAULTS.gdp_path_scramble,
+    quota_backend=_DEFAULTS.quota_backend,
+    quota_user_limit=_DEFAULTS.quota_user_limit,
+    quota_vgrid_limit=_DEFAULTS.quota_vgrid_limit,
+    ca_fqdn=_DEFAULTS.ca_fqdn,
+    ca_user=_DEFAULTS.ca_user,
+    ca_smtp=_DEFAULTS.ca_smtp,
+    datasafety_link=_DEFAULTS.datasafety_link,
+    datasafety_text=_DEFAULTS.datasafety_text,
+    wwwserve_max_bytes=_DEFAULTS.wwwserve_max_bytes,
     _getpwnam=pwd.getpwnam,
     _prepare=None,
     _writefiles=None,
@@ -630,6 +864,9 @@ def generate_confs(
     return (options, user_dict)
 
 
+_GENERATE_CONFS_PARAMETERS = set(inspect_args(generate_confs)) - set(_GENERATE_CONFS_NOFORWARD_KEYS)
+
+
 def _generate_confs_prepare(
     options,
     # forwarded arguments

mig/shared/install.py

@@ -35,6 +35,8 @@
 
 from tests.support import MIG_BASE, MigTestCase, testmain, cleanpath
 
+from mig.install.generateconfs import _PARAMETERS
+from mig.shared.install import generate_confs, _GENERATE_CONFS_PARAMETERS
 
 def _import_generateconfs():
     """Internal helper to work around non-package import location"""
@@ -65,6 +67,13 @@ def noop(*args):
     pass
 
 
+class MigInstallGenerateconfs(MigTestCase):
+    def test_consistent_parameters(self):
+        mismatched = _GENERATE_CONFS_PARAMETERS - set(_PARAMETERS)
+        self.assertEqual(len(mismatched), 0,
+                         "defined parameters do not match generate_confs()")
+
+
 class MigInstallGenerateconfs__main(MigTestCase):
     """Unit test helper for the migrid code pointed to in class name"""