|
| 1 | +# adapted from https://gitlab.haskell.org/bgamari/ghc-utils/-/blob/master/rel-eng/fetch-gitlab-artifacts/fetch_gitlab.py |
| 2 | +import logging |
| 3 | +from pathlib import Path |
| 4 | +import subprocess |
| 5 | +import gitlab |
| 6 | + |
| 7 | +logging.basicConfig(level=logging.INFO) |
| 8 | + |
| 9 | +def strip_prefix(s, prefix): |
| 10 | + if s.startswith(prefix): |
| 11 | + return s[len(prefix):] |
| 12 | + else: |
| 13 | + return None |
| 14 | + |
| 15 | +def fetch_artifacts(release: str, pipeline_id: int, |
| 16 | + dest_dir: Path, gl: gitlab.Gitlab): |
| 17 | + dest_dir.mkdir(exist_ok=True) |
| 18 | + proj = gl.projects.get('haskell/haskell-language-server') |
| 19 | + pipeline = proj.pipelines.get(pipeline_id) |
| 20 | + tmpdir = Path("fetch-gitlab") |
| 21 | + tmpdir.mkdir(exist_ok=True) |
| 22 | + for pipeline_job in pipeline.jobs.list(all=True): |
| 23 | + if len(pipeline_job.artifacts) == 0: |
| 24 | + logging.info(f'job {pipeline_job.name} ({pipeline_job.id}) has no artifacts') |
| 25 | + continue |
| 26 | + |
| 27 | + job = proj.jobs.get(pipeline_job.id) |
| 28 | + platform = strip_prefix(job.name, 'tar-') |
| 29 | + if not platform: |
| 30 | + logging.info(f'Skipping {job.name} (not a tar job)') |
| 31 | + continue |
| 32 | + try: |
| 33 | + destdir = tmpdir / job.name |
| 34 | + zip_name = Path(f"{tmpdir}/{job.name}.zip") |
| 35 | + if not zip_name.exists() or zip_name.stat().st_size == 0: |
| 36 | + logging.info(f'downloading archive {zip_name} for job {job.name} (job {job.id})...') |
| 37 | + with open(zip_name, 'wb') as f: |
| 38 | + job.artifacts(streamed=True, action=f.write) |
| 39 | + |
| 40 | + if zip_name.stat().st_size == 0: |
| 41 | + logging.info(f'artifact archive for job {job.name} (job {job.id}) is empty') |
| 42 | + continue |
| 43 | + |
| 44 | + dest = dest_dir / f'haskell-language-server-{release}-{platform}.tar.xz' |
| 45 | + if dest.exists(): |
| 46 | + logging.info(f'bindist {dest} already exists') |
| 47 | + continue |
| 48 | + |
| 49 | + subprocess.run(['unzip', '-bo', zip_name, '-d', destdir]) |
| 50 | + bindist_files = list(destdir.glob('*/haskell-language-server*.tar.xz')) |
| 51 | + if len(bindist_files) == 0: |
| 52 | + logging.warn(f'Bindist does not exist') |
| 53 | + continue |
| 54 | + |
| 55 | + bindist = bindist_files[0] |
| 56 | + logging.info(f'extracted {job.name} to {dest}') |
| 57 | + bindist.replace(dest) |
| 58 | + except Exception as e: |
| 59 | + logging.error(f'Error fetching job {job.name}: {e}') |
| 60 | + pass |
| 61 | + |
| 62 | +def main(): |
| 63 | + import argparse |
| 64 | + parser = argparse.ArgumentParser() |
| 65 | + parser.add_argument('--pipeline', '-p', required=True, type=int, help="pipeline id") |
| 66 | + parser.add_argument('--release', '-r', required=True, type=str, help="release name") |
| 67 | + parser.add_argument('--output', '-o', type=Path, default=Path.cwd(), help="output directory") |
| 68 | + parser.add_argument('--profile', '-P', default='haskell', |
| 69 | + help='python-gitlab.cfg profile name') |
| 70 | + args = parser.parse_args() |
| 71 | + gl = gitlab.Gitlab.from_config(args.profile) |
| 72 | + fetch_artifacts(args.release, args.pipeline, |
| 73 | + dest_dir=args.output, gl=gl) |
| 74 | + |
| 75 | +if __name__ == '__main__': |
| 76 | + main() |
0 commit comments