Collect all builds in bump-timestamp

Author: vrutkovs

src/bump-timestamp

@@ -5,13 +5,70 @@
 '''
 
 import argparse
+import collections
+import os
+import sys
 
 from cosalib.builds import Builds
+from cosalib.cmdlib import get_timestamp
+
+Build = collections.namedtuple('Build', ['id', 'timestamp', 'basearches'])
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--workdir", default='.', help="Path to workdir")
 args = parser.parse_args()
 
 builds = Builds(args.workdir)
+
+scanned_builds = []
+builds_dir = os.path.join(args.workdir, "builds")
+
+# first, pick up all the builds from the dir itself
+with os.scandir(builds_dir) as it:
+    for entry in it:
+        # ignore non-dirs
+        if not entry.is_dir(follow_symlinks=False):
+            # those are really the only two non-dir things we expect there
+            if entry.name not in ['builds.json', 'latest']:
+                print(f"Ignoring non-directory {entry.path}")
+            continue
+
+        # scan all per-arch builds, pick up the most recent build of those as
+        # the overall "build" timestamp for pruning purposes
+        with os.scandir(entry.path) as basearch_it:
+            multiarch_build = None
+            for basearch_entry in basearch_it:
+                # ignore non-dirs
+                if not basearch_entry.is_dir(follow_symlinks=False):
+                    print(f"Ignoring non-directory {basearch_entry.path}")
+                    continue
+                ts = get_timestamp(basearch_entry)
+                if not ts:
+                    continue
+                if not multiarch_build:
+                    multiarch_build = Build(id=entry.name, timestamp=ts,
+                                            basearches=[basearch_entry.name])
+                else:
+                    multiarch_build.basearches += [basearch_entry.name]
+                    multiarch_build.timestamp = max(
+                        multiarch_build.timestamp, ts)
+            if multiarch_build:
+                scanned_builds.append(multiarch_build)
+
+# just get the trivial case out of the way
+if len(scanned_builds) == 0:
+    print("No builds found!")
+    sys.exit(0)
+
+# sort by timestamp, newest first
+scanned_builds = sorted(scanned_builds,
+                        key=lambda x: x.timestamp,
+                        reverse=True)
+
+builds.raw()['builds'] = []
+for build in reversed(scanned_builds):
+    for basearch in build.basearches:
+        builds.insert_build(build.id, basearch)
+
 builds.bump_timestamp()
 print("Build timestamp was updated")

src/cosalib/cmdlib.py

@@ -23,7 +23,7 @@
 gi.require_version("RpmOstree", "1.0")
 from gi.repository import RpmOstree
 
-from datetime import datetime
+from datetime import datetime, timezone
 
 retry_stop = (stop_after_delay(10) | stop_after_attempt(5))
 retry_boto_exception = (retry_if_exception_type(ConnectionClosedError) |
@@ -204,3 +204,33 @@ def get_basearch():
     except AttributeError:
         get_basearch.saved = RpmOstree.get_basearch()
         return get_basearch.saved
+
+
+def parse_date_string(date_string):
+    """
+    Parses the date strings expected from the build system. Returned
+    datetime instances will be in utc.
+    :param date_string: string to turn into date. Format: %Y-%m-%dT%H:%M:%SZ
+    :type date_string: str
+    :returns: datetime instance from the date string
+    :rtype: datetime.datetime
+    :raises: ValueError, TypeError
+    """
+    dt = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
+    return dt.replace(tzinfo=timezone.utc)
+
+
+def get_timestamp(entry):
+
+    # ignore dirs missing meta.json
+    meta_file = os.path.join(entry.path, 'meta.json')
+    if not os.path.isfile(meta_file):
+        print(f"Ignoring directory {entry.name}")
+        return None
+
+    # collect dirs and timestamps
+    with open(meta_file) as f:
+        j = json.load(f)
+    # Older versions only had ostree-timestamp
+    ts = j.get('coreos-assembler.build-timestamp') or j['ostree-timestamp']
+    return parse_date_string(ts)