fixup! Add CI for Base Browser

Bug 42997: Reduce the number of git fetches by fetching during a setup
job and passing that initial fetch around as a git bundle to other jobs
through the Gitlab artifacts. Further reduce the number of fetches by
creating a list of changed files in the setup job and passing that
around as well as an artifact.

.gitlab-ci.yml

 stages:
+  - setup
   - lint
 
 variables:
   IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest
 
 include:
+  - local: '.gitlab/ci/setup.yml'
   - local: '.gitlab/ci/lint.yml'

.gitlab/ci/lint.yml

 .base:
   stage: lint
+  image: $IMAGE_PATH
   interruptible: true
+  needs:
+    - job: setup-env
+      artifacts: true
+    - job: create-bundle
+      artifacts: true
   variables:
     MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"
+    GIT_STRATEGY: "none"
   cache:
     paths:
       - node_modules
@@ -11,12 +18,26 @@
     when: 'always'
     # Share the cache throughout all pipelines running for a given branch
     key: $CI_COMMIT_REF_SLUG
+  before_script:
+    # DEBUG: Are all artifacts here?
+    - ls -a
+    - mkdir app && cd app
+    # Initialize a fresh git repo
+    - git init
+    # Add app.bundle as the remote. All operations that communicate with the remote will be local.
+    - git remote add origin ../app.bundle
+    # shallow.txt contains the SHA of the base commit of the bundle.
+    # The bundle is shallow, therefore it's base commit will not have a parent.
+    # Adding the SHA of the base commit to .git/shallow tells git that it doesn't need
+    # to crash when it realizes said base commit doesn't have a parent.
+    - cp ../shallow.txt .git/shallow
+    # Finally, unpack the bundle. Time it for debugging purposes.
+    - time git pull origin $BRANCH_NAME
 
 eslint:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py eslint
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l eslint
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -44,9 +65,8 @@ eslint:
 
 stylelint:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py stylelint
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l stylelint
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -63,9 +83,8 @@ stylelint:
 
 py-black:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py black
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l black
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -83,9 +102,8 @@ py-black:
 
 py-ruff:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py ruff
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l ruff
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -103,9 +121,8 @@ py-ruff:
 
 yaml:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py yaml
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l yaml
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -119,9 +136,8 @@ yaml:
 
 shellcheck:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py shellcheck
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l shellcheck
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -134,10 +150,9 @@ shellcheck:
 
 clang-format:
   extends: .base
-  image: $IMAGE_PATH
   script:
     - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0
-    - .gitlab/ci/scripts/run_linters.py clang-format
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l clang-format
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -155,9 +170,8 @@ clang-format:
 
 rustfmt:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py rustfmt
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l rustfmt
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -170,9 +184,8 @@ rustfmt:
 
 fluent-lint:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py fluent-lint
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l fluent-lint
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -186,9 +199,8 @@ fluent-lint:
 
 localization:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py l10n
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l l10n
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -204,9 +216,8 @@ localization:
 
 mingw-capitalization:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py mingw-capitalization
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mingw-capitalization
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -222,9 +233,8 @@ mingw-capitalization:
 
 mscom-init:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py mscom-init
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mscom-init
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -240,9 +250,8 @@ mscom-init:
 
 file-whitespace:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py file-whitespace
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l file-whitespace
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -270,9 +279,8 @@ file-whitespace:
 
 test-manifest:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -286,9 +294,8 @@ test-manifest:
 
 trojan-source:
   extends: .base
-  image: $IMAGE_PATH
   script:
-    - .gitlab/ci/scripts/run_linters.py trojan-source
+    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l trojan-source
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:

.gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py

@@ -5,7 +5,6 @@ import os
 import re
 import shlex
 import subprocess
-import sys
 
 
 def git(command):
@@ -15,8 +14,8 @@ def git(command):
     return result.stdout.strip()
 
 
-def get_firefox_tag_from_branch_name(branch_name):
-    """Extracts the Firefox tag associated with a branch name.
+def get_firefox_tag(reference):
+    """Extracts the Firefox tag associated with a branch or tag name.
 
        The "firefox tag" is the tag that marks
        the end of the Mozilla commits and the start of the Tor Project commits.
@@ -25,26 +24,26 @@ def get_firefox_tag_from_branch_name(branch_name):
        this function may return the incorrect reference number.
 
     Args:
-        branch_name: The branch name to extract the tag from.
+        reference: The branch or tag name to extract the Firefox tag from.
         Expected format is tor-browser-91.2.0esr-11.0-1,
         where 91.2.0esr is the Firefox version.
 
     Returns:
         The reference specifier of the matching Firefox tag.
-        An exception wil be raised if anything goes wrong.
+        An exception will be raised if anything goes wrong.
     """
 
-    # Extracts the version number from a branch name.
+    # Extracts the version number from a branch or tag name.
     firefox_version = ""
-    match = re.search(r"(?<=browser-)([^-]+)", branch_name)
+    match = re.search(r"(?<=browser-)([^-]+)", reference)
     if match:
         # TODO: Validate that what we got is actually a valid semver string?
         firefox_version = match.group(1)
     else:
-        raise ValueError(f"Failed to extract version from branch name '{branch_name}'.")
+        raise ValueError(f"Failed to extract version from reference '{reference}'.")
 
     tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"
-    remote_tags = git("ls-remote --tags")
+    remote_tags = git("ls-remote --tags origin")
 
     # Each line looks like:
     # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1
@@ -54,7 +53,7 @@ def get_firefox_tag_from_branch_name(branch_name):
         return match.group(0).split()[0]
     else:
         raise ValueError(
-            f"Failed to find reference specifier for Firefox tag '{tag}' in branch '{branch_name}'."
+            f"Failed to find reference specifier for Firefox tag '{tag}' from '{reference}'."
         )
 
 
@@ -74,37 +73,42 @@ def get_list_of_changed_files():
     base_reference = ""
 
     if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
-        # For merge requests, the base_reference is the common ancestor between the MR and the target branch.
+        # For merge requests, the base_reference is the common ancestor between the MR and the target branch
         base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
     else:
         # When not in merge requests, the base reference is the Firefox tag
-        base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH"))
+        base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))
 
     if not base_reference:
         raise RuntimeError("No base reference found. There might be more errors above.")
 
     # Fetch the tag reference
     git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
-    # Return the list of changed files
-    return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n")
+    # Return but filter the issue_templates files because those file names have spaces which can cause issues
+    return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history."
+    parser = argparse.ArgumentParser(description="")
+
+    parser.add_argument(
+        "--get-firefox-tag",
+        help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",
+        type=str,
     )
     parser.add_argument(
-        "linters", metavar="L", type=str, nargs="+", help="A list of linters to run."
+        "--get-changed-files",
+        help="Get list of changed files."
+        "When running from a merge request get sthe list of changed files since the merge-base of the current branch."
+        "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",
+        action="store_true",
     )
+
     args = parser.parse_args()
 
-    changed_files = get_list_of_changed_files()
-    if changed_files:
-        command = ["./mach", "lint", "-v"]
-        for linter in args.linters:
-            command.extend(["-l", linter])
-        command.extend(changed_files)
-        result = subprocess.run(command, text=True)
-        sys.exit(result.returncode)
+    if args.get_firefox_tag:
+        print(get_firefox_tag(args.get_firefox_tag))
+    elif args.get_changed_files:
+        print("\n".join(get_list_of_changed_files()))
     else:
-        print("No files changed, skipping linting.")
+        print("No valid option provided.")

.gitlab/ci/setup.yml

+setup-env:
+  stage: setup
+  interruptible: true
+  variables:
+    GIT_STRATEGY: "none"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
+  script:
+    - |
+      if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
+        echo "BRANCH_NAME=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" > build.env
+      else
+        echo "BRANCH_NAME=$CI_COMMIT_REF_NAME" > build.env
+      fi
+  artifacts:
+    reports:
+      dotenv:
+        - build.env
+
+create-bundle:
+  stage: setup
+  # TODO: Find a better suited image, this one just has git.
+  image: python
+  needs:
+    - job: setup-env
+      artifacts: true
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
+  variables:
+    GIT_DEPTH: "1"
+  interruptible: true
+  script:
+    # DEBUG: Check repository status.
+    - git status
+    # Switch to a named branch.
+    - git switch -c $BRANCH_NAME
+    # Create a git bundle -- this will generate the app.bundle file,
+    # which can be used as a git remote for offline fetching.
+    - git bundle create app.bundle --all
+    # Retain the SHA of the base of this shallow repository.
+    - cat .git/shallow > shallow.txt
+    # DEBUG: Check sizes.
+    - du -sh .git
+    - du -sh app.bundle
+    # Since this is the only job we have access to the Gitlab remote,
+    # let's get a list of changed files to use in the next jobs.
+    - .gitlab/ci/scripts/helpers.py --get-changed-files > changedfiles.txt
+  artifacts:
+    paths:
+      - app.bundle
+      - shallow.txt
+      - changedfiles.txt
+    expire_in: 1 hour