Add CI for Base Browser

stages:

  - lint

  - update-translations

variables:

  IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest

  LOCAL_REPO_PATH: /srv/apps-repos/tor-browser.git

include:

  - local: '.gitlab/ci/mixins.yml'

  - local: '.gitlab/ci/jobs/lint/lint.yml'

  - local: '.gitlab/ci/jobs/update-translations.yml'

#!/usr/bin/env python3

import argparse

import os

import re

import shlex

import subprocess

def git(command):

    result = subprocess.run(

        ["git"] + shlex.split(command), check=True, capture_output=True, text=True

    )

    return result.stdout.strip()

def get_firefox_tag(reference):

    """Extracts the Firefox tag associated with a branch or tag name.

       The "firefox tag" is the tag that marks

       the end of the Mozilla commits and the start of the Tor Project commits.

       Know issue: If ever there is more than one tag per Firefox ESR version,

       this function may return the incorrect reference number.

    Args:

        reference: The branch or tag name to extract the Firefox tag from.

        Expected format is tor-browser-91.2.0esr-11.0-1,

        where 91.2.0esr is the Firefox version.

    Returns:

        The reference specifier of the matching Firefox tag.

        An exception will be raised if anything goes wrong.

    """

    # Extracts the version number from a branch or tag name.

    firefox_version = ""

    match = re.search(r"(?<=browser-)([^-]+)", reference)

    if match:

        # TODO: Validate that what we got is actually a valid semver string?

        firefox_version = match.group(1)

    else:

        raise ValueError(f"Failed to extract version from reference '{reference}'.")

    major_version = firefox_version.split(".")[0]

    minor_patch_version = "_".join(firefox_version.split(".")[1:])

    remote_tags = git("ls-remote --tags origin")

    # Each line looks like:

    # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1

    pattern = (

        rf"(.*)FIREFOX_{re.escape(major_version)}_{re.escape(minor_patch_version)}(.*)$"

    )

    match = re.search(pattern, remote_tags, flags=re.MULTILINE)

    if not match:

        # Attempt to match with a nightly tag, in case the ESR tag is not found

        pattern = rf"(.*)FIREFOX_NIGHTLY_{re.escape(major_version)}(.*)$"

        match = re.search(pattern, remote_tags, flags=re.MULTILINE)

    if match:

        return match.group(0).split()[0]

    else:

        raise ValueError(

            f"Failed to find reference specifier for Firefox tag of version '{firefox_version}' from '{reference}'."

        )

def get_list_of_changed_files():

    """Gets a list of files changed in the working directory.

       This function is meant to be run inside the Gitlab CI environment.

       When running in a default branch, get the list of changed files since the last Firefox tag.

       When running for a new MR commit, get a list of changed files in the current MR.

    Returns:

        A list of filenames of changed files (excluding deleted files).

        An exception wil be raised if anything goes wrong.

    """

    base_reference = ""

    if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":

        # For merge requests, the base_reference is the common ancestor between the MR and the target branch

        base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")

        if not base_reference:

            # Probably because there has been no overall change.

            # See gitlab.com/gitlab-org/gitlab/-/issues/375047#note_2648459916

            return []

    else:

        # When not in merge requests, the base reference is the Firefox tag

        base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))

    if not base_reference:

        raise RuntimeError("No base reference found. There might be more errors above.")

    # Fetch the tag reference

    git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")

    # Return but filter the issue_templates files because those file names have spaces which can cause issues

    return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="")

    parser.add_argument(

        "--get-firefox-tag",

        help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",

        type=str,

    )

    parser.add_argument(

        "--get-changed-files",

        help="Get list of changed files."

        "When running from a merge request get sthe list of changed files since the merge-base of the current branch."

        "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",

        action="store_true",

    )

    args = parser.parse_args()

    if args.get_firefox_tag:

        print(get_firefox_tag(args.get_firefox_tag))

    elif args.get_changed_files:

        # Separate the file names with a 0 byte to be parsed by xargs -0. Also

        # drop the trailing '\n'.

        print("\0".join(get_list_of_changed_files()), end="")

    else:

        print("No valid option provided.")

lint-all:

  extends: .with-local-repo-bash

  stage: lint

  image: $IMAGE_PATH

  interruptible: true

  variables:

    # Has to be the same as defined in `containers/base/Containerfile`

    MOZBUILD_STATE_PATH: "/var/tmp/mozbuild"

  cache:

    paths:

      - node_modules

    # Store the cache regardless on job outcome

    when: 'always'

    # Share the cache throughout all pipelines running for a given branch

    key: $CI_COMMIT_REF_SLUG

  tags:

    # Run these jobs in the browser dedicated runners.

    - firefox

  script:

    - ./mach configure --with-base-browser-version=0.0.0

    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -0 --no-run-if-empty ./mach lint -v

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

    # Run job whenever a commit is merged to a protected branch

    - if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')

.update-translation-base:

  stage: update-translations

  rules:

    - if: ($TRANSLATION_FILES != "" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push")

      changes:

        - "**/*.ftl"

        - "**/*.properties"

        - "**/*.dtd"

        - "**/*strings.xml"

        - "**/update-translations.yml"

        - "**/l10n/combine/combine.py"

        - "**/l10n/combine-translation-versions.py"

    - if: ($TRANSLATION_FILES != "" && $FORCE_UPDATE_TRANSLATIONS == "true")

  variables:

    COMBINED_FILES_JSON: "combined-translation-files.json"

    TRANSLATION_FILES: ''

combine-en-US-translations:

  extends: .update-translation-base

  needs: []

  image: python

  variables:

    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

  cache:

    paths:

      - .cache/pip

  # Artifact is for translation project job

  artifacts:

    paths:

      - "$COMBINED_FILES_JSON"

    expire_in: "60 min"

    reports:

      dotenv: job_id.env

  # Don't load artifacts for this job.

  dependencies: []

  script:

    # Save this CI_JOB_ID to the dotenv file to be used in the variables for the

    # push-en-US-translations job.

    - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env

    - pip install compare_locales

    - python ./tools/base-browser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$COMBINED_FILES_JSON"

push-en-US-translations:

  extends: .update-translation-base

  needs:

    - job: combine-en-US-translations

  variables:

    COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${COMBINED_FILES_JSON}"

  trigger:

    strategy: depend

    project: tor-browser-translation-bot/translation

    branch: tor-browser-ci

.with-local-repo-bash:

  variables:

      GIT_STRATEGY: "none"

      FETCH_TIMEOUT: 180 # 3 minutes

  before_script:

    - git init

    - git remote add local "$LOCAL_REPO_PATH"

    - |

      # Determine the reference of the target branch in the local repository copy.

      #

      # 1. List all references in the local repository

      # 2. Filter the references to the target branch

      # 3. Remove tags

      # 4. Keep a single line, in case there are too many matches

      # 5. Clean up the output

      # 6. Remove everything before the last two slashes, because the output is like `refs/heads/...` or `refs/remotes/...`

      TARGET_BRANCH=$(git ls-remote local | grep ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME} |  grep -v 'refs/tags/' | awk '{print $2}' | tail -1 | sed 's|[^/]*/[^/]*/||')

      if [ -z "$TARGET_BRANCH" ]; then

          echo "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."

          exit 1

      fi

    - git fetch --depth 500 local $TARGET_BRANCH

    - git --no-pager log FETCH_HEAD --oneline -n 5

    - git remote add origin "$CI_REPOSITORY_URL"

    - |

      if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then

          echo "No branch specified. Stopping the pipeline."

          exit 1

      fi

    - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} with a ${FETCH_TIMEOUT}s timeout."

    - |

      fetch_with_timeout() {

        local remote=$1

        local branch=$2

        set +e

        timeout ${FETCH_TIMEOUT} git fetch "$remote" "$branch"

        local fetch_exit=$?

        set -e

        if [ "$fetch_exit" -eq 124 ]; then

          echo "Fetching failed for branch ${remote}/${branch} due to a timeout. Try again later."

          echo "Gitlab may be experiencing slowness or the local copy of the repository on the CI server may be oudated."

          return 1

        fi

        return $fetch_exit

      }

      if ! fetch_with_timeout origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"; then

        echo "Fetching failed for branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}."

        echo "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."

        fetch_with_timeout origin "merge-requests/${CI_MERGE_REQUEST_IID}/head" || exit 1

      fi

    - git checkout FETCH_HEAD

.with-local-repo-pwsh:

  variables:

      GIT_STRATEGY: "none"

  before_script:

    - git init

    - git remote add local $env:LOCAL_REPO_PATH

    - |

      $branchName = $env:CI_COMMIT_BRANCH

      if ([string]::IsNullOrEmpty($branchName)) {

          $branchName = $env:CI_MERGE_REQUEST_TARGET_BRANCH_NAME

      }

      $TARGET_BRANCH = git ls-remote local | Select-String -Pattern $branchName | Select-String -Pattern -NotMatch 'refs/tags/' | Select-Object -Last 1 | ForEach-Object { $_.ToString().Split()[1] -replace '^[^/]*/[^/]*/', '' }

      if ([string]::IsNullOrEmpty($TARGET_BRANCH)) {

          Write-Output "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."

          exit 1

      }

    - git remote add origin $env:CI_REPOSITORY_URL

    - |

      $branchName = $env:CI_COMMIT_BRANCH

      if ([string]::IsNullOrEmpty($branchName)) {

          $branchName = $env:CI_MERGE_REQUEST_SOURCE_BRANCH_NAME

      }

      if ([string]::IsNullOrEmpty($branchName)) {

          Write-Output "No branch specified. Stopping the pipeline."

          exit 1

      }

    - Write-Output "Fetching from remote branch $branchName"

    - |

      if (! git fetch origin $branchName) {

        Write-Output "Fetching failed for branch $branchName from $env:CI_REPOSITORY_URL."

        Write-Output "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."

        git fetch origin "merge-requests/$env:CI_MERGE_REQUEST_IID/head"

      }

    - git checkout FETCH_HEAD