diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69c0c429ef5e0108384ab6c07551cdef0329b8ac..b0b9922696a0b71ab3d3d144faa6485b2b38baad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,10 @@ stages: + - setup - lint variables: IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest include: + - local: '.gitlab/ci/setup.yml' - local: '.gitlab/ci/lint.yml' diff --git a/.gitlab/ci/lint.yml b/.gitlab/ci/lint.yml index 8e08a42093def2c62d463f73cec9f92fdd8016e3..a014320ec4caa76578b148cb8daeb656ee34709a 100644 --- a/.gitlab/ci/lint.yml +++ b/.gitlab/ci/lint.yml @@ -1,8 +1,15 @@ .base: stage: lint + image: $IMAGE_PATH interruptible: true + needs: + - job: setup-env + artifacts: true + - job: create-bundle + artifacts: true variables: MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild" + GIT_STRATEGY: "none" cache: paths: - node_modules @@ -11,12 +18,26 @@ when: 'always' # Share the cache throughout all pipelines running for a given branch key: $CI_COMMIT_REF_SLUG + before_script: + # DEBUG: Are all artifacts here? + - ls -a + - mkdir app && cd app + # Initialize a fresh git repo + - git init + # Add app.bundle as the remote. All operations that communicate with the remote will be local. + - git remote add origin ../app.bundle + # shallow.txt contains the SHA of the base commit of the bundle. + # The bundle is shallow, therefore it's base commit will not have a parent. + # Adding the SHA of the base commit to .git/shallow tells git that it doesn't need + # to crash when it realizes said base commit doesn't have a parent. + - cp ../shallow.txt .git/shallow + # Finally, unpack the bundle. Time it for debugging purposes. + - time git pull origin $BRANCH_NAME eslint: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py eslint + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l eslint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -44,9 +65,8 @@ eslint: stylelint: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py stylelint + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l stylelint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -63,9 +83,8 @@ stylelint: py-black: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py black + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l black rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -83,9 +102,8 @@ py-black: py-ruff: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py ruff + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l ruff rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -103,9 +121,8 @@ py-ruff: yaml: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py yaml + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l yaml rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -119,9 +136,8 @@ yaml: shellcheck: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py shellcheck + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l shellcheck rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -134,10 +150,9 @@ shellcheck: clang-format: extends: .base - image: $IMAGE_PATH script: - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0 - - .gitlab/ci/scripts/run_linters.py clang-format + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l clang-format rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -155,9 +170,8 @@ clang-format: rustfmt: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py rustfmt + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l rustfmt rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -170,9 +184,8 @@ rustfmt: fluent-lint: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py fluent-lint + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l fluent-lint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -186,9 +199,8 @@ fluent-lint: localization: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py l10n + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l l10n rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -204,9 +216,8 @@ localization: mingw-capitalization: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py mingw-capitalization + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mingw-capitalization rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -222,9 +233,8 @@ mingw-capitalization: mscom-init: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py mscom-init + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mscom-init rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -240,9 +250,8 @@ mscom-init: file-whitespace: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py file-whitespace + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l file-whitespace rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -270,9 +279,8 @@ file-whitespace: test-manifest: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -286,9 +294,8 @@ test-manifest: trojan-source: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py trojan-source + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l trojan-source rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: diff --git a/.gitlab/ci/scripts/run_linters.py b/.gitlab/ci/scripts/helpers.py similarity index 60% rename from .gitlab/ci/scripts/run_linters.py rename to .gitlab/ci/scripts/helpers.py index 9049c565b7e4bd4d4e1bfd7fed8e8f25065272f6..8b82d15169ea20239c3e7e722db68b289b02cf85 100755 --- a/.gitlab/ci/scripts/run_linters.py +++ b/.gitlab/ci/scripts/helpers.py @@ -5,7 +5,6 @@ import os import re import shlex import subprocess -import sys def git(command): @@ -15,8 +14,8 @@ def git(command): return result.stdout.strip() -def get_firefox_tag_from_branch_name(branch_name): - """Extracts the Firefox tag associated with a branch name. +def get_firefox_tag(reference): + """Extracts the Firefox tag associated with a branch or tag name. The "firefox tag" is the tag that marks the end of the Mozilla commits and the start of the Tor Project commits. @@ -25,26 +24,26 @@ def get_firefox_tag_from_branch_name(branch_name): this function may return the incorrect reference number. Args: - branch_name: The branch name to extract the tag from. + reference: The branch or tag name to extract the Firefox tag from. Expected format is tor-browser-91.2.0esr-11.0-1, where 91.2.0esr is the Firefox version. Returns: The reference specifier of the matching Firefox tag. - An exception wil be raised if anything goes wrong. + An exception will be raised if anything goes wrong. """ - # Extracts the version number from a branch name. + # Extracts the version number from a branch or tag name. firefox_version = "" - match = re.search(r"(?<=browser-)([^-]+)", branch_name) + match = re.search(r"(?<=browser-)([^-]+)", reference) if match: # TODO: Validate that what we got is actually a valid semver string? firefox_version = match.group(1) else: - raise ValueError(f"Failed to extract version from branch name '{branch_name}'.") + raise ValueError(f"Failed to extract version from reference '{reference}'.") tag = f"FIREFOX_{firefox_version.replace('.', '_')}_" - remote_tags = git("ls-remote --tags") + remote_tags = git("ls-remote --tags origin") # Each line looks like: # 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1 @@ -54,7 +53,7 @@ def get_firefox_tag_from_branch_name(branch_name): return match.group(0).split()[0] else: raise ValueError( - f"Failed to find reference specifier for Firefox tag '{tag}' in branch '{branch_name}'." + f"Failed to find reference specifier for Firefox tag '{tag}' from '{reference}'." ) @@ -74,37 +73,42 @@ def get_list_of_changed_files(): base_reference = "" if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event": - # For merge requests, the base_reference is the common ancestor between the MR and the target branch. + # For merge requests, the base_reference is the common ancestor between the MR and the target branch base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA") else: # When not in merge requests, the base reference is the Firefox tag - base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH")) + base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH")) if not base_reference: raise RuntimeError("No base reference found. There might be more errors above.") # Fetch the tag reference git(f"fetch origin {base_reference} --depth=1 --filter=blob:none") - # Return the list of changed files - return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n") + # Return but filter the issue_templates files because those file names have spaces which can cause issues + return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n") if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history." + parser = argparse.ArgumentParser(description="") + + parser.add_argument( + "--get-firefox-tag", + help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.", + type=str, ) parser.add_argument( - "linters", metavar="L", type=str, nargs="+", help="A list of linters to run." + "--get-changed-files", + help="Get list of changed files." + "When running from a merge request get sthe list of changed files since the merge-base of the current branch." + "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.", + action="store_true", ) + args = parser.parse_args() - changed_files = get_list_of_changed_files() - if changed_files: - command = ["./mach", "lint", "-v"] - for linter in args.linters: - command.extend(["-l", linter]) - command.extend(changed_files) - result = subprocess.run(command, text=True) - sys.exit(result.returncode) + if args.get_firefox_tag: + print(get_firefox_tag(args.get_firefox_tag)) + elif args.get_changed_files: + print("\n".join(get_list_of_changed_files())) else: - print("No files changed, skipping linting.") + print("No valid option provided.") diff --git a/.gitlab/ci/setup.yml b/.gitlab/ci/setup.yml new file mode 100644 index 0000000000000000000000000000000000000000..6e52367744f609dbf321b00b91b39bf5ff2f9b7a --- /dev/null +++ b/.gitlab/ci/setup.yml @@ -0,0 +1,53 @@ +setup-env: + stage: setup + interruptible: true + variables: + GIT_STRATEGY: "none" + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true' + script: + - | + if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then + echo "BRANCH_NAME=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" > build.env + else + echo "BRANCH_NAME=$CI_COMMIT_REF_NAME" > build.env + fi + artifacts: + reports: + dotenv: + - build.env + +create-bundle: + stage: setup + # TODO: Find a better suited image, this one just has git. + image: python + needs: + - job: setup-env + artifacts: true + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true' + variables: + GIT_DEPTH: "1" + interruptible: true + script: + # DEBUG: Check repository status. + - git status + # Switch to a named branch. + - git switch -c $BRANCH_NAME + # Create a git bundle -- this will generate the app.bundle file, + # which can be used as a git remote for offline fetching. + - git bundle create app.bundle --all + # Retain the SHA of the base of this shallow repository. + - cat .git/shallow > shallow.txt + # DEBUG: Check sizes. + - du -sh .git + - du -sh app.bundle + # Since this is the only job we have access to the Gitlab remote, + # let's get a list of changed files to use in the next jobs. + - .gitlab/ci/scripts/helpers.py --get-changed-files > changedfiles.txt + artifacts: + paths: + - app.bundle + - shallow.txt + - changedfiles.txt + expire_in: 1 hour