WIP reproduce using gitlab-runner

#!/usr/bin/env python3

# Reproduces the build from a specific gitlab pipeline,

# Using `gitlab-runner exec`

import os

import requests

import subprocess

def get_pipeline(project_id, pipeline_id):

    req = requests.get(f'https://gitlab.torproject.org/api/v4/projects/{project_id}/pipelines/{pipeline_id}')

    return req.json()

def get_successful_job(project_id, pipeline_id, name):

    # TODO: handle paginating

    req = requests.get(f'https://gitlab.torproject.org/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs?per_page=100')

    jobs = filter(lambda j: j['status'] == 'success' and j['name'] == name, req.json())

    # TODO: get most recent if multiple?

    return next(jobs)

def checkout_build_repo(url=None, branch=None, depth=None, ref=None, dst_dir=None):

    subprocess.check_call([

        'git',

        'clone',

        f'--depth={depth}',

        f'--branch={branch}',

        url,

        dst_dir,

        ])

    subprocess.check_call([

        'git',

        'checkout',

        ref,

        ],

        cwd=dst_dir)

    # Copy the debian-ci to the root ci. `gitlab-runner exec` doesn't support

    # pointing to a custom yml file afaict.

    subprocess.check_call(['cp', f'{dst_dir}/debian/.debian-ci.yml', f'{dst_dir}/.gitlab-ci.yml'])

    # For some reason gitlab-runner exec says GIT_DEPTH: 1 is invalid.

    subprocess.check_call(['sed', '-i', 's/.*GIT_DEPTH: 1.*//', f'{dst_dir}/.gitlab-ci.yml'])

def run_job(build_dir=None, build_repo_dir=None, branch=None, job_name=None, job_artifacts_path=None):

    build_dir = subprocess.check_output(['realpath', build_dir], text=True).strip()

    env={

        'PATH': os.getenv('PATH'),

        'DOCKER_HOST': os.getenv('DOCKER_HOST'),

        }

    artifacts_dir = f'{build_dir}/artifacts'

    os.makedirs(artifacts_dir, exist_ok=True)

    # FIXME put this somewhere else

    script_path=f'{artifacts_dir}/artifacts_out.sh'

    f = open(script_path, 'w')

    f.write('#!/bin/sh\n')

    f.write(f'cp -r {job_artifacts_path} /artifacts\n')

    f.close()

    subprocess.check_call(['chmod', 'a+rx', script_path])

    # FIXME put this somewhere else

    script_path=f'{artifacts_dir}/artifacts_in.sh'

    f = open(script_path, 'w')

    f.write('#!/bin/sh\n')

    f.write('if [ "`ls /artifacts`" != "" ]\n')

    f.write('then\n')

    f.write('  cp -r /artifacts/* .\n')

    f.write('fi\n')

    f.close()

    subprocess.check_call(['chmod', 'a+rx', script_path])

    subprocess.check_call([

        'gitlab-runner',

        'exec',

        'docker',

        f'--pre-build-script=/artifacts/artifacts_in.sh',

        f'--post-build-script=/artifacts/artifacts_out.sh',

        f'--docker-volumes={artifacts_dir}:/artifacts',

        f'--env=CI_COMMIT_BRANCH={branch}',

        job_name,

        ],

        cwd=build_repo_dir,

        env=env)

if __name__ == '__main__':

    # Project id for https://gitlab.torproject.org/tpo/core/debian/tor.git

    project_id=1218

    # Current most-recent stable build.

    pipeline_id=38207

    pipeline = get_pipeline(project_id, pipeline_id)

    # Where we'll put our build output.

    build_dir=f'reproduce-{pipeline_id}'

    pipeline_src_dir=f'{build_dir}/pipeline_src'

    pipeline_branch='debian-0.4.6'

    if False:

        checkout_build_repo(

                url='https://gitlab.torproject.org/tpo/core/debian/tor.git',

                branch=pipeline_branch,

                # Needs to be at least 2, or something breaks trying to get the

                # parent commit.

                depth=2,

                ref='e6683856224bba07a48a9c6d1149c5de24a3ab85',

                dst_dir=pipeline_src_dir)

    if False:

        run_job(build_dir=build_dir,

                build_repo_dir=pipeline_src_dir,

                branch=pipeline_branch,

                job_name="build_source",

                job_artifacts_path="source-packages")

    # XXX Broken. "no such job name".

    # Is there a way to run just one point in the build_binary matrix?

    run_job(build_dir=build_dir,

            build_repo_dir=pipeline_src_dir,

            branch=pipeline_branch,

            job_name="build_binary: [debian, bullseye, amd64, amd64, -slim]",

            job_artifacts_path="binary-packages")

    #build_job = get_successful_job(project_id, pipeline_id, "build_source")

    #print(build_job)

    #sys.exit(0)