buildupload: scrub skipped missing images from meta.json
Right now, when we use `--artifact` to only upload some artifacts, we
still reference them in the `meta.json`. This is bad though, because
consumers will assume that they're present.
Just scrub them out of the custom `meta.json` we upload if they're not
present in the bucket.
@@ -103,6 +103,7 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix):
# Upload images with special handling for gzipped data.
uploaded = set()
+ scrub = set()
for imgname in build['images']:
img = build['images'][imgname]
bn = img['path']
@@ -121,15 +122,18 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix):
s3_path = f'{prefix}/{nogz}'
set_content_disposition = True
+ s3_target_exists = s3_check_exists(s3_client, bucket, s3_path, args.dry_run)
+
# Mark artifacts that we don't want to upload as already uploaded
- # so they'll get ignored later.
+ # so they'll get ignored later. If they're not in S3, delete them
+ # so that meta.json doesn't reference non-existent objects.
if len(args.artifact) > 0 and imgname not in args.artifact:
print(f"Skipping upload of artifact {bn} upon user request")
uploaded.add(bn)
+ if not s3_target_exists:
+ scrub.add(imgname)
continue
- s3_target_exists = s3_check_exists(s3_client, bucket, s3_path, args.dry_run)
-
if not os.path.exists(path) and not s3_target_exists:
raise Exception(f"{path} not found locally or in the s3 destination!")
@@ -163,8 +167,10 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix):
dry_run=args.dry_run)
# Now upload a modified version of the meta.json which has the fixed
- # filenames without the .gz suffixes. We don't want to modify the local
- # build dir.
+ # filenames without the .gz suffixes and the scrubbed artifacts. We don't
+ # want to modify the local build dir.
+ for imgname in scrub:
+ del build['images'][imgname]
with tempfile.NamedTemporaryFile('w') as f:
json.dump(build, f, indent=4)
f.flush()
