Add CI for Base Browser

.gitlab-ci.yml

+stages:
+  - lint
+  - update-translations
+
+variables:
+  IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest
+  LOCAL_REPO_PATH: /srv/apps-repos/tor-browser.git
+
+include:
+  - local: '.gitlab/ci/mixins.yml'
+  - local: '.gitlab/ci/jobs/lint/lint.yml'
+  - local: '.gitlab/ci/jobs/update-translations.yml'

.gitlab/ci/docker/base/Dockerfile

+FROM debian:latest
+
+# Base image which includes all* dependencies checked by ./mach configure.
+#
+# * Actually not all dependencies. WASM sandboxed depencies were left out for now.
+# This installs all dependencies checked by `./mach configure --without-wasm-sandboxed-libraries`.
+#
+# # Building and publishing
+#
+# Whenever this file changes, the updated Docker image must be built and published _manually_ to
+# the tor-browser container registry (https://gitlab.torproject.org/tpo/applications/tor-browser/container_registry/185).
+#
+# This image copies a script from the taskcluster/ folder, which requires it
+# to be built from a folder which is a parent of the taskcluster/ folder.
+#
+# To build, run:
+#
+# ```bash
+# docker build \
+#   -f <PATH_TO_DOCKERFILE> \
+#   -t <REGISTRY_URL>/<IMAGE_NAME>:<IMAGE_TAG>
+#   .
+# ```
+#
+# For example, when building from the root of this repository to the main tor-browser repository
+# and assuming image name to be "base" and tag "latest" -- which is the current terminology:
+#
+# ```bash
+# docker build \
+#   -f .gitlab/ci/docker/Dockerfile \
+#   -t containers.torproject.org/tpo/applications/tor-browser/base:latest
+#   .
+# ```
+
+RUN apt-get update && apt-get install -y \
+    clang \
+    curl \
+    git \
+    libasound2-dev \
+    libdbus-glib-1-dev \
+    libgtk-3-dev \
+    libpango1.0-dev \
+    libpulse-dev \
+    libx11-xcb-dev \
+    libxcomposite-dev \
+    libxcursor-dev \
+    libxdamage-dev \
+    libxi-dev \
+    libxrandr-dev \
+    libxtst-dev \
+    m4 \
+    mercurial \
+    nasm \
+    pkg-config \
+    python3 \
+    python3-pip \
+    unzip \
+    wget
+
+COPY taskcluster/docker/recipes/install-node.sh /scripts/install-node.sh
+RUN chmod +x /scripts/install-node.sh
+RUN /scripts/install-node.sh
+
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+RUN $HOME/.cargo/bin/cargo install cbindgen
+
+WORKDIR /app
+
+CMD ["/bin/bash"]

.gitlab/ci/jobs/lint/helpers.py

+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import shlex
+import subprocess
+
+
+def git(command):
+    result = subprocess.run(
+        ["git"] + shlex.split(command), check=True, capture_output=True, text=True
+    )
+    return result.stdout.strip()
+
+
+def get_firefox_tag(reference):
+    """Extracts the Firefox tag associated with a branch or tag name.
+
+       The "firefox tag" is the tag that marks
+       the end of the Mozilla commits and the start of the Tor Project commits.
+
+       Know issue: If ever there is more than one tag per Firefox ESR version,
+       this function may return the incorrect reference number.
+
+    Args:
+        reference: The branch or tag name to extract the Firefox tag from.
+        Expected format is tor-browser-91.2.0esr-11.0-1,
+        where 91.2.0esr is the Firefox version.
+
+    Returns:
+        The reference specifier of the matching Firefox tag.
+        An exception will be raised if anything goes wrong.
+    """
+
+    # Extracts the version number from a branch or tag name.
+    firefox_version = ""
+    match = re.search(r"(?<=browser-)([^-]+)", reference)
+    if match:
+        # TODO: Validate that what we got is actually a valid semver string?
+        firefox_version = match.group(1)
+    else:
+        raise ValueError(f"Failed to extract version from reference '{reference}'.")
+
+    major_version = firefox_version.split(".")[0]
+    minor_patch_version = "_".join(firefox_version.split(".")[1:])
+
+    remote_tags = git("ls-remote --tags origin")
+
+    # Each line looks like:
+    # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1
+    pattern = (
+        rf"(.*)FIREFOX_{re.escape(major_version)}_{re.escape(minor_patch_version)}(.*)$"
+    )
+    match = re.search(pattern, remote_tags, flags=re.MULTILINE)
+    if not match:
+        # Attempt to match with a nightly tag, in case the ESR tag is not found
+        pattern = rf"(.*)FIREFOX_NIGHTLY_{re.escape(major_version)}(.*)$"
+        match = re.search(pattern, remote_tags, flags=re.MULTILINE)
+
+    if match:
+        return match.group(0).split()[0]
+    else:
+        raise ValueError(
+            f"Failed to find reference specifier for Firefox tag of version '{firefox_version}' from '{reference}'."
+        )
+
+
+def get_list_of_changed_files():
+    """Gets a list of files changed in the working directory.
+
+       This function is meant to be run inside the Gitlab CI environment.
+
+       When running in a default branch, get the list of changed files since the last Firefox tag.
+       When running for a new MR commit, get a list of changed files in the current MR.
+
+    Returns:
+        A list of filenames of changed files (excluding deleted files).
+        An exception wil be raised if anything goes wrong.
+    """
+
+    base_reference = ""
+
+    if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
+        # For merge requests, the base_reference is the common ancestor between the MR and the target branch
+        base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
+    else:
+        # When not in merge requests, the base reference is the Firefox tag
+        base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))
+
+    if not base_reference:
+        raise RuntimeError("No base reference found. There might be more errors above.")
+
+    # Fetch the tag reference
+    git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
+    # Return but filter the issue_templates files because those file names have spaces which can cause issues
+    return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="")
+
+    parser.add_argument(
+        "--get-firefox-tag",
+        help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",
+        type=str,
+    )
+    parser.add_argument(
+        "--get-changed-files",
+        help="Get list of changed files."
+        "When running from a merge request get sthe list of changed files since the merge-base of the current branch."
+        "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",
+        action="store_true",
+    )
+
+    args = parser.parse_args()
+
+    if args.get_firefox_tag:
+        print(get_firefox_tag(args.get_firefox_tag))
+    elif args.get_changed_files:
+        print("\n".join(get_list_of_changed_files()))
+    else:
+        print("No valid option provided.")

.gitlab/ci/jobs/lint/lint.yml

+lint-all:
+  extends: .with-local-repo-bash
+  stage: lint
+  image: $IMAGE_PATH
+  interruptible: true
+  variables:
+    MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"
+  cache:
+    paths:
+      - node_modules
+      - .cache/mozbuild
+    # Store the cache regardless on job outcome
+    when: 'always'
+    # Share the cache throughout all pipelines running for a given branch
+    key: $CI_COMMIT_REF_SLUG
+  tags:
+    # Run these jobs in the browser dedicated runners.
+    - firefox
+  script:
+    - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -v
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # Run job whenever a commit is merged to a protected branch
+    - if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')

.gitlab/ci/jobs/update-translations.yml

+.update-translation-base:
+  stage: update-translations
+  rules:
+    - if: ($TRANSLATION_FILES != "" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push")
+      changes:
+        - "**/*.ftl"
+        - "**/*.properties"
+        - "**/*.dtd"
+        - "**/*strings.xml"
+        - "**/update-translations.yml"
+        - "**/l10n/combine/combine.py"
+        - "**/l10n/combine-translation-versions.py"
+    - if: ($TRANSLATION_FILES != "" && $FORCE_UPDATE_TRANSLATIONS == "true")
+  variables:
+    COMBINED_FILES_JSON: "combined-translation-files.json"
+    TRANSLATION_FILES: ''
+
+
+combine-en-US-translations:
+  extends: .update-translation-base
+  needs: []
+  image: python
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  cache:
+    paths:
+      - .cache/pip
+  # Artifact is for translation project job
+  artifacts:
+    paths:
+      - "$COMBINED_FILES_JSON"
+    expire_in: "60 min"
+    reports:
+      dotenv: job_id.env
+  # Don't load artifacts for this job.
+  dependencies: []
+  script:
+    # Save this CI_JOB_ID to the dotenv file to be used in the variables for the
+    # push-en-US-translations job.
+    - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env
+    - pip install compare_locales
+    - python ./tools/base-browser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$COMBINED_FILES_JSON"
+
+push-en-US-translations:
+  extends: .update-translation-base
+  needs:
+    - job: combine-en-US-translations
+  variables:
+    COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${COMBINED_FILES_JSON}"
+  trigger:
+    strategy: depend
+    project: tor-browser-translation-bot/translation
+    branch: tor-browser-ci

.gitlab/ci/mixins.yml

+.with-local-repo-bash:
+  variables:
+      GIT_STRATEGY: "none"
+      FETCH_TIMEOUT: 180 # 3 minutes
+  before_script:
+    - git init
+    - git remote add local "$LOCAL_REPO_PATH"
+    - |
+      # Determine the reference of the target branch in the local repository copy.
+      #
+      # 1. List all references in the local repository
+      # 2. Filter the references to the target branch
+      # 3. Remove tags
+      # 4. Keep a single line, in case there are too many matches
+      # 5. Clean up the output
+      # 6. Remove everything before the last two slashes, because the output is like `refs/heads/...` or `refs/remotes/...`
+      TARGET_BRANCH=$(git ls-remote local | grep ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME} |  grep -v 'refs/tags/' | awk '{print $2}' | tail -1 | sed 's|[^/]*/[^/]*/||')
+      if [ -z "$TARGET_BRANCH" ]; then
+          echo "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."
+          exit 1
+      fi
+    - git fetch --depth 500 local $TARGET_BRANCH
+    - git --no-pager log FETCH_HEAD --oneline -n 5
+    - git remote add origin "$CI_REPOSITORY_URL"
+    - |
+      if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then
+          echo "No branch specified. Stopping the pipeline."
+          exit 1
+      fi
+    - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} with a ${FETCH_TIMEOUT}s timeout."
+    - |
+      fetch_with_timeout() {
+        local remote=$1
+        local branch=$2
+
+        set +e
+        timeout ${FETCH_TIMEOUT} git fetch "$remote" "$branch"
+        local fetch_exit=$?
+        set -e
+
+        if [ "$fetch_exit" -eq 124 ]; then
+          echo "Fetching failed for branch ${remote}/${branch} due to a timeout. Try again later."
+          echo "Gitlab may be experiencing slowness or the local copy of the repository on the CI server may be oudated."
+          return 1
+        fi
+
+        return $fetch_exit
+      }
+
+      if ! fetch_with_timeout origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"; then
+        echo "Fetching failed for branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}."
+        echo "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."
+
+        fetch_with_timeout origin "merge-requests/${CI_MERGE_REQUEST_IID}/head" || exit 1
+      fi
+    - git checkout FETCH_HEAD
+
+.with-local-repo-pwsh:
+  variables:
+      GIT_STRATEGY: "none"
+  before_script:
+    - git init
+    - git remote add local $env:LOCAL_REPO_PATH
+    - |
+      $branchName = $env:CI_COMMIT_BRANCH
+      if ([string]::IsNullOrEmpty($branchName)) {
+          $branchName = $env:CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+      }
+      $TARGET_BRANCH = git ls-remote local | Select-String -Pattern $branchName | Select-String -Pattern -NotMatch 'refs/tags/' | Select-Object -Last 1 | ForEach-Object { $_.ToString().Split()[1] -replace '^[^/]*/[^/]*/', '' }
+      if ([string]::IsNullOrEmpty($TARGET_BRANCH)) {
+          Write-Output "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."
+          exit 1
+      }
+    - git remote add origin $env:CI_REPOSITORY_URL
+    - |
+      $branchName = $env:CI_COMMIT_BRANCH
+      if ([string]::IsNullOrEmpty($branchName)) {
+          $branchName = $env:CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      }
+      if ([string]::IsNullOrEmpty($branchName)) {
+          Write-Output "No branch specified. Stopping the pipeline."
+          exit 1
+      }
+    - Write-Output "Fetching from remote branch $branchName"
+    - |
+      if (! git fetch origin $branchName) {
+        Write-Output "Fetching failed for branch $branchName from $env:CI_REPOSITORY_URL."
+        Write-Output "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."
+        git fetch origin "merge-requests/$env:CI_MERGE_REQUEST_IID/head"
+      }
+    - git checkout FETCH_HEAD