Add cosa sign robosignatory for signing OSTrees and images

Add a new `cosa sign` top-level command for general signing operations.
For now, it only supports one subcommand, `robosignatory`, which does
signing by RoboSignatory via fedora-messaging (see [1][2] for details).

Essentially with this, the pipeline flow will be:

```
$ cosa build ostree
$ cosa sign robosignatory --ostree ...
$ cosa buildextend-*
$ cosa buildupload
$ cosa sign robosignatory --images ...
```

[1] coreos/fedora-coreos-tracker#198
[2] coreos/fedora-coreos-tracker#200

Author: jlebon

src/cmd-sign

@@ -0,0 +1,370 @@
+#!/usr/bin/python3
+
+'''
+    Implements signing with RoboSignatory via fedora-messaging. To run this,
+    one needs credentials to the restricted Fedora broker. In a developer
+    workflow, one can also run it (and RoboSignatory) against a local rabbitmq
+    instance. For more details, see:
+
+    https://fedora-messaging.readthedocs.io/en/latest/quick-start.html
+'''
+
+import argparse
+import gi
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import threading
+import uuid
+
+from multiprocessing import Process
+
+import boto3
+
+from fedora_messaging import message
+from fedora_messaging.api import publish, twisted_consume
+from fedora_messaging.config import conf
+
+from twisted.internet import reactor
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from cosalib.meta import GenericBuildMeta as Meta
+from cosalib.builds import Builds
+from cosalib.cmdlib import (
+    get_basearch,
+    sha256sum_file,
+    import_ostree_commit)
+
+gi.require_version('OSTree', '1.0')
+from gi.repository import Gio, OSTree
+
+# these files are part of fedora-messaging
+FEDORA_MESSAGING_PUBLIC_CONF = {
+    'prod': '/etc/fedora-messaging/fedora.toml',
+    'stg': '/etc/fedora-messaging/fedora.stg.toml',
+}
+FEDORA_MESSAGING_TOPIC_PREFIX = {
+    'prod': 'org.fedoraproject.prod.coreos.build.request',
+    'stg': 'org.fedoraproject.stg.coreos.build.request',
+}
+ROBOSIGNATORY_REQUEST_TIMEOUT_SEC = 60 * 10
+
+fedenv = 'prod'
+
+
+def main():
+    args = parse_args()
+    if args.stg:
+        global fedenv
+        fedenv = 'stg'
+    args.func(args)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--build", help="Build ID", default='latest')
+    subparsers = parser.add_subparsers(dest='cmd', title='subcommands')
+    subparsers.required = True
+
+    robosig = subparsers.add_parser('robosignatory', help='sign with '
+                                    'RoboSignatory via fedora-messaging')
+    robosig.add_argument("--s3", metavar='<BUCKET>[/PREFIX]', required=True,
+                         help="bucket and prefix to S3 builds/ dir")
+    group = robosig.add_mutually_exclusive_group(required=True)
+    group.add_argument("--ostree", help="sign commit", action='store_true')
+    group.add_argument("--images", help="sign images", action='store_true')
+    robosig.add_argument("--extra-fedmsg-keys", action='append',
+                         metavar='KEY=VAL', default=[],
+                         help="extra keys to inject into messages")
+    robosig.add_argument("--fedmsg-conf", metavar='CONFIG.TOML',
+                         help="fedora-messaging config file for publishing")
+    robosig.add_argument("--stg", action='store_true',
+                         help="target the stg infra rather than prod")
+    robosig.add_argument("--gpgkeypath", help="path to directory containing "
+                         "public keys to use for signature verification",
+                         default="/etc/pki/rpm-gpg")
+    robosig.set_defaults(func=cmd_robosignatory)
+
+    return parser.parse_args()
+
+
+def cmd_robosignatory(args):
+    builds = Builds()
+    if args.build == 'latest':
+        args.build = builds.get_latest()
+
+    s3 = boto3.client('s3')
+    args.bucket, args.prefix = get_bucket_and_prefix(args.s3)
+
+    args.extra_keys = {}
+    for keyval in args.extra_fedmsg_keys:
+        key, val = keyval.split('=', 1)  # will throw exception if there's no =
+        args.extra_keys[key] = val
+
+    request = 'ostree-sign' if args.ostree else 'artifacts-sign'
+
+    global request_state
+    request_state = {"status": "pending"}
+    cond = threading.Condition()
+    start_consumer_thread(cond, request, {
+        'build_id': args.build,
+        'basearch': get_basearch(),
+        **args.extra_keys
+    })
+
+    # these two are different enough that they deserve separate handlers
+    if args.ostree:
+        robosign_ostree(args, s3, cond)
+    else:
+        assert args.images
+        robosign_images(args, s3, cond)
+
+
+def robosign_ostree(args, s3, cond):
+    build = Meta(build=args.build)
+    builds = Builds()
+    builddir = builds.get_build_dir(args.build)
+    checksum = build['ostree-commit']
+
+    # Copy commit object to a temporary location. A preferred approach here is
+    # to require the pipeline to do a preliminary buildupload and then just
+    # point at the final object location instead. Though we'd want
+    # https://github.com/coreos/coreos-assembler/issues/668 before doing this
+    # so we at least GC on failure. For now, just use a stable path so we
+    # clobber previous runs.
+    build_dir_commit_obj = os.path.join(builddir, 'ostree-commit-object')
+    commit_key = f'{args.prefix}/tmp/ostree-commit-object'
+    commitmeta_key = f'{args.prefix}/tmp/ostree-commitmeta-object'
+    print(f"Uploading s3://{args.bucket}/{commit_key}")
+    s3.upload_file(build_dir_commit_obj, args.bucket, commit_key)
+    s3.delete_object(Bucket=args.bucket, Key=commitmeta_key)
+
+    send_message(args, 'ostree-sign', {
+        'commit_object': f's3://{args.bucket}/{commit_key}',
+        'checksum': f'sha256:{checksum}'
+    })
+
+    validate_response(cond)
+
+    # download back sig and verify it in a throwaway repo
+    print(f"Verifying OSTree signature")
+    with tempfile.TemporaryDirectory(prefix="cosa-sign", dir="tmp") as d:
+        repo = OSTree.Repo.new(Gio.File.new_for_path(d))
+        repo.create(OSTree.RepoMode.ARCHIVE)
+
+        commit_obj_relpath = f'objects/{checksum[:2]}/{checksum[2:]}.commit'
+        commit_obj_path = os.path.join(d, commit_obj_relpath)
+        commitmeta_obj_relpath = f'{commit_obj_relpath}meta'
+        commitmeta_obj_path = os.path.join(d, commitmeta_obj_relpath)
+
+        os.makedirs(os.path.dirname(commit_obj_path), exist_ok=True)
+        shutil.copyfile(build_dir_commit_obj, commit_obj_path)
+        s3.download_file(args.bucket, commitmeta_key, commitmeta_obj_path)
+
+        # this is a bit awkward though the remote API is the only way to point
+        # libostree at armored GPG keys
+        config = repo.copy_config()
+        config.set_string('remote "tmpremote"', 'url', 'https://example.com')
+        config.set_string('remote "tmpremote"', 'gpgkeypath', args.gpgkeypath)
+        config.set_boolean('remote "tmpremote"', 'gpg-verify', True)
+        repo.write_config(config)
+        # XXX: work around ostree_repo_write_config not reloading remotes too
+        repo.reload_config()
+
+        result = repo.verify_commit_for_remote(checksum, 'tmpremote')
+        assert result.count_all() == 1
+        t = result.get(0, [OSTree.GpgSignatureAttr.FINGERPRINT,
+                           OSTree.GpgSignatureAttr.USER_NAME,
+                           OSTree.GpgSignatureAttr.USER_EMAIL,
+                           OSTree.GpgSignatureAttr.VALID])
+        fp = t.get_child_value(0).get_string()
+        name = t.get_child_value(1).get_string()
+        email = t.get_child_value(2).get_string()
+        valid = t.get_child_value(3).get_boolean()
+        msg = (("Valid " if valid else "Invalid ")
+               + f"signature from {name} <{email}> ({fp})")
+        # allow unknown signatures in stg
+        if not valid and fedenv != 'stg':
+            raise Exception(msg)
+        print(msg)
+
+        # ok, it's valid -- add it to the tarfile
+        ostree_image = build['images']['ostree']
+        commit_tarfile = os.path.join(builddir, ostree_image['path'])
+        with tarfile.open(commit_tarfile, 'a:') as t:
+            t.add(commitmeta_obj_path, arcname=commitmeta_obj_relpath)
+        ostree_image['size'] = os.path.getsize(commit_tarfile)
+        ostree_image['sha256'] = sha256sum_file(commit_tarfile)
+        build.write()
+
+    # and finally add it to the tmprepo too so that buildextend-(qemu|metal)
+    # will pull it: we could just nuke the repo to force a re-untar, but it
+    # might nuke a more recent commit if we're not operating on the latest
+    import_ostree_commit('tmp/repo', checksum, commit_tarfile, force=True)
+
+
+def robosign_images(args, s3, cond):
+    build = Meta(build=args.build)
+    builds = Builds()
+    builddir = builds.get_build_dir(args.build)
+
+    # NB: we just handle the current basearch for now
+    full_prefix = f'{args.prefix}/{args.build}/{get_basearch()}'
+
+    # collect all the image paths to sign
+    artifacts = [{
+        'file': f's3://{args.bucket}/{full_prefix}/{img["path"]}',
+        'checksum': f'sha256:{img["sha256"]}'
+    } for img in build['images'].values()]
+
+    send_message(args, 'artifacts-sign', {'artifacts': artifacts})
+    validate_response(cond)
+
+    # download sigs and verify (use /tmp to avoid gpg hitting ENAMETOOLONG)
+    with tempfile.TemporaryDirectory(prefix="cosa-sign-") as d:
+        def gpg(*args):
+            subprocess.check_call(['gpg', '--homedir', d, *args])
+
+        with os.scandir(args.gpgkeypath) as it:
+            keys = [entry.path for entry in it if entry.is_file()]
+            gpg('--quiet', '--import', *keys)
+
+        for img in build['images'].values():
+            sig_s3_key = f'{full_prefix}/{img["path"]}.sig'
+
+            tmp_sig_path = f'tmp/{img["path"]}.sig'
+            s3.download_file(args.bucket, sig_s3_key, tmp_sig_path)
+
+            local_artifact = f'{builddir}/{img["path"]}'
+
+            print(f"Verifying signature for {local_artifact}")
+            try:
+                gpg('--verify', tmp_sig_path, local_artifact)
+            except subprocess.CalledProcessError as e:
+                # allow unknown signatures in stg
+                if fedenv != 'stg':
+                    raise e
+
+            # move into final location
+            os.rename(tmp_sig_path, f'{local_artifact}.sig')
+
+            # and make S3 object public (XXX: fix robosignatory for this?)
+            s3.put_object_acl(Bucket=args.bucket, Key=sig_s3_key,
+                              ACL='public-read')
+
+
+def get_bucket_and_prefix(path):
+    split = path.split("/", 1)
+    if len(split) == 1:
+        return (split[0], "")
+    return split
+
+
+def get_request_topic(request):
+    return f'{FEDORA_MESSAGING_TOPIC_PREFIX[fedenv]}.{request}'
+
+
+def get_request_finished_topic(request):
+    return get_request_topic(request) + '.finished'
+
+
+def send_message(args, request, body):
+    print(f"Sending {request} request for build {args.build}")
+    # This is a bit hacky; we fork to publish the message here so that we can
+    # load the publishing fedora-messaging config. The TL;DR is: we need auth
+    # to publish, but we need to use the public endpoint for consuming so we
+    # can create temporary queues.
+    p = Process(target=send_message_impl, args=(args, request, body))
+    p.start()
+    p.join()
+
+
+def send_message_impl(args, request, body):
+    if args.fedmsg_conf:
+        conf.load_config(args.fedmsg_conf)
+    publish(message.Message(topic=get_request_topic(request), body={
+        'build_id': args.build,
+        'basearch': get_basearch(),
+        **args.extra_keys,
+        **body
+    }))
+
+
+def validate_response(cond):
+    with cond:
+        print("Waiting for response from RoboSignatory")
+        cond.wait_for(lambda: request_state['status'] != 'pending',
+                      timeout=ROBOSIGNATORY_REQUEST_TIMEOUT_SEC)
+        if request_state['status'] == 'pending':
+            raise Exception("Timed out waiting for RoboSignatory")
+        if request_state['status'].lower() == 'failure':
+            # https://pagure.io/robosignatory/pull-request/38
+            if 'failure-message' not in request_state:
+                raise Exception("Signing failed")
+            raise Exception(f"Signing failed: {request_state['failure-message']}")
+        assert request_state['status'].lower() == 'success', str(request_state)
+
+
+def start_consumer_thread(cond, request, filters):
+    registered = threading.Event()
+    t = threading.Thread(target=watch_finished_messages,
+                         args=(cond, registered, request, filters),
+                         daemon=True)
+    t.start()
+    registered.wait()
+    print(f"Successfully started consumer thread")
+
+
+def watch_finished_messages(cond, registered, request, filters):
+    def callback(message):
+        # XXX: For now, we filter like this. In the future, we'll generate a
+        # request_id and just look for that:
+        # https://pagure.io/robosignatory/pull-request/37
+        for (k, v) in filters.items():
+            if k not in message.body or message.body[k] != v:
+                return
+        with cond:
+            global request_state
+            request_state = message.body
+            cond.notify()
+
+    queue = str(uuid.uuid4())
+
+    def registered_cb(consumers):
+        for consumer in consumers:
+            if consumer.queue == queue:
+                registered.set()
+                break
+
+    def error_cb(failure):
+        print(f"Consumer hit failure {failure}")
+        reactor.stop()  # pylint: disable=E1101
+
+    # use the public config for this; see related comment in send_message
+    conf.load_config(FEDORA_MESSAGING_PUBLIC_CONF[fedenv])
+
+    bindings = [{
+        'exchange': 'amq.topic',
+        'queue': queue,
+        'routing_keys': [get_request_finished_topic(request)]
+    }]
+    queues = {
+        queue: {
+            "durable": False,
+            "auto_delete": True,
+            "exclusive": True,
+            "arguments": {}
+        }
+    }
+
+    consumers = twisted_consume(callback, bindings=bindings, queues=queues)
+    consumers.addCallback(registered_cb)
+    consumers.addErrback(error_cb)
+    reactor.run(installSignalHandlers=False)  # pylint: disable=E1101
+
+
+if __name__ == '__main__':
+    sys.exit(main())

src/coreos-assembler

@@ -39,7 +39,7 @@ build_commands="init fetch build run prune clean"
 # commands more likely to be used in a prod pipeline only
 advanced_build_commands="buildprep buildupload oscontainer"
 buildextend_commands="qemu aws azure gcp openstack installer live vmware metal"
-utility_commands="tag compress koji-upload kola aws-replicate"
+utility_commands="tag sign compress koji-upload kola aws-replicate"
 other_commands="shell"
 if [ -z "${cmd}" ]; then
     echo Usage: "coreos-assembler CMD ..."

src/deps.txt

@@ -63,3 +63,6 @@ krb5-libs krb5-workstation koji-utils python3-koji python3-koji-cli-plugins
 
 # LUKS support
 cryptsetup
+
+# For communicating with RoboSignatory for signing requests
+fedora-messaging