Skip to content

Commit ccec60d

Browse files
committed
Add "remote-prune" subcommand
This subcommand cleans up unreferenced builds on remote location. Currently only s3 buckets are implemented
1 parent a8d9031 commit ccec60d

File tree

2 files changed

+56
-1
lines changed

2 files changed

+56
-1
lines changed

src/cmd-remote-prune

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/python3 -u
2+
3+
'''
4+
This script removes unreferenced builds from s3 bucket
5+
'''
6+
7+
import argparse
8+
import sys
9+
10+
from cosalib.builds import Builds
11+
from cosalib.prune import fetch_build_meta, get_unreferenced_s3_builds, delete_build
12+
13+
parser = argparse.ArgumentParser(prog="coreos-assembler remote-prune")
14+
parser.add_argument("--workdir", default='.', help="Path to workdir")
15+
parser.add_argument("--dry-run", help="Don't actually delete anything",
16+
action='store_true')
17+
18+
subparsers = parser.add_subparsers(dest='cmd', title='subcommands')
19+
subparsers.required = True
20+
21+
s3 = subparsers.add_parser('s3', help='Prune s3 buckets')
22+
s3.add_argument("--bucket", help="Bucket name")
23+
s3.add_argument("--prefix", help="Key prefix")
24+
25+
args = parser.parse_args()
26+
27+
builds = Builds(args.workdir)
28+
29+
scanned_builds = []
30+
for build in builds.raw()["builds"]:
31+
for arch in build['arches']:
32+
build = fetch_build_meta(build['id'], arch)
33+
if build:
34+
scanned_builds.append(build)
35+
36+
new_builds = []
37+
builds_to_delete = []
38+
39+
# Find unreferenced builds in the bucket and remove them
40+
buildids = [x['id'] for x in scanned_builds]
41+
unreferenced_s3_builds = get_unreferenced_s3_builds(buildids, args.bucket, args.prefix)
42+
43+
error_during_pruning = False
44+
for unmatched_build_id in unreferenced_s3_builds:
45+
# TODO: fetch arches from s3
46+
build = fetch_build_meta(unmatched_build_id, 'x86_64')
47+
if build and not args.dry_run:
48+
try:
49+
delete_build(build)
50+
except Exception as e:
51+
error_during_pruning = True
52+
print(f"{e}")
53+
54+
if error_during_pruning:
55+
sys.exit(1)

src/coreos-assembler

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ build_commands="init fetch build run prune clean"
3939
# commands more likely to be used in a prod pipeline only
4040
advanced_build_commands="buildprep buildupload oscontainer"
4141
buildextend_commands="qemu aws azure gcp openstack installer live vmware metal"
42-
utility_commands="tag sign compress koji-upload kola aws-replicate"
42+
utility_commands="tag sign compress koji-upload kola aws-replicate remote-prune"
4343
other_commands="shell"
4444
if [ -z "${cmd}" ]; then
4545
echo Usage: "coreos-assembler CMD ..."

0 commit comments

Comments
 (0)