Add CI for Base Browser

stages:

  - setup

  - lint

variables:

  IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest

include:

  - local: '.gitlab/ci/setup.yml'

  - local: '.gitlab/ci/lint.yml'

FROM debian:latest

# Base image which includes all* dependencies checked by ./mach configure.

#

# * Actually not all dependencies. WASM sandboxed depencies were left out for now.

# This installs all dependencies checked by `./mach configure --without-wasm-sandboxed-libraries`.

#

# # Building and publishing

#

# Whenever this file changes, the updated Docker image must be built and published _manually_ to

# the tor-browser container registry (https://gitlab.torproject.org/tpo/applications/tor-browser/container_registry/185).

#

# This image copies a script from the taskcluster/ folder, which requires it

# to be built from a folder which is a parent of the taskcluster/ folder.

#

# To build, run:

#

# ```bash

# docker build \

#   -f <PATH_TO_DOCKERFILE> \

#   -t <REGISTRY_URL>/<IMAGE_NAME>:<IMAGE_TAG>

#   .

# ```

#

# For example, when building from the root of this repository to the main tor-browser repository

# and assuming image name to be "base" and tag "latest" -- which is the current terminology:

#

# ```bash

# docker build \

#   -f .gitlab/ci/docker/Dockerfile \

#   -t containers.torproject.org/tpo/applications/tor-browser/base:latest

#   .

# ```

RUN apt-get update && apt-get install -y \

    clang \

    curl \

    git \

    libasound2-dev \

    libdbus-glib-1-dev \

    libgtk-3-dev \

    libpango1.0-dev \

    libpulse-dev \

    libx11-xcb-dev \

    libxcomposite-dev \

    libxcursor-dev \

    libxdamage-dev \

    libxi-dev \

    libxrandr-dev \

    libxtst-dev \

    m4 \

    mercurial \

    nasm \

    pkg-config \

    python3 \

    python3-pip \

    unzip \

    wget

COPY taskcluster/docker/recipes/install-node.sh /scripts/install-node.sh

RUN chmod +x /scripts/install-node.sh

RUN /scripts/install-node.sh

RUN curl https://sh.rustup.rs -sSf | sh -s -- -y

RUN $HOME/.cargo/bin/cargo install cbindgen

WORKDIR /app

CMD ["/bin/bash"]

.base:

  stage: lint

  image: $IMAGE_PATH

  interruptible: true

  needs:

    - job: setup-env

      artifacts: true

    - job: create-bundle

      artifacts: true

  variables:

    MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"

    GIT_STRATEGY: "none"

  cache:

    paths:

      - node_modules

      - .cache/mozbuild

    # Store the cache regardless on job outcome

    when: 'always'

    # Share the cache throughout all pipelines running for a given branch

    key: $CI_COMMIT_REF_SLUG

  before_script:

    # DEBUG: Are all artifacts here?

    - ls -a

    - mkdir app && cd app

    # Initialize a fresh git repo

    - git init

    # Add app.bundle as the remote. All operations that communicate with the remote will be local.

    - git remote add origin ../app.bundle

    # shallow.txt contains the SHA of the base commit of the bundle.

    # The bundle is shallow, therefore it's base commit will not have a parent.

    # Adding the SHA of the base commit to .git/shallow tells git that it doesn't need

    # to crash when it realizes said base commit doesn't have a parent.

    - cp ../shallow.txt .git/shallow

    # Finally, unpack the bundle. Time it for debugging purposes.

    - time git pull origin $BRANCH_NAME

eslint:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l eslint

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        # Files that are likely audited.

        - '**/*.js'

        - '**/*.jsm'

        - '**/*.json'

        - '**/*.jsx'

        - '**/*.mjs'

        - '**/*.sjs'

        - '**/*.html'

        - '**/*.xhtml'

        - '**/*.xml'

        - 'tools/lint/eslint.yml'

        # Run when eslint policies change.

        - '**/.eslintignore'

        - '**/*eslintrc*'

        # The plugin implementing custom checks.

        - 'tools/lint/eslint/eslint-plugin-mozilla/**'

        - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

stylelint:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l stylelint

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        # Files that are likely audited.

        - '**/*.css'

        - 'tools/lint/styleint.yml'

        # Run when stylelint policies change.

        - '**/.stylelintignore'

        - '**/*stylelintrc*'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

py-black:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l black

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        # The list of extensions should match tools/lint/black.yml

        - '**/*.py'

        - '**/moz.build'

        - '**/*.configure'

        - '**/*.mozbuild'

        - 'pyproject.toml'

        - 'tools/lint/black.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

py-ruff:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l ruff

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.py'

        - '**/*.configure'

        - '**/.ruff.toml'

        - 'pyproject.toml'

        - 'tools/lint/ruff.yml'

        - 'tools/lint/python/ruff.py'

        - 'tools/lint/python/ruff_requirements.txt'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

yaml:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l yaml

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.yml'

        - '**/*.yaml'

        - '**/.ymllint'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

shellcheck:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l shellcheck

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.sh'

        - 'tools/lint/shellcheck.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

clang-format:

  extends: .base

  script:

    - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l clang-format

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.cpp'

        - '**/*.c'

        - '**/*.cc'

        - '**/*.h'

        - '**/*.m'

        - '**/*.mm'

        - 'tools/lint/clang-format.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

rustfmt:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l rustfmt

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.rs'

        - 'tools/lint/rustfmt.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

fluent-lint:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l fluent-lint

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.ftl'

        - 'tools/lint/fluent-lint.yml'

        - 'tools/lint/fluent-lint/exclusions.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

localization:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l l10n

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/locales/en-US/**'

        - '**/l10n.toml'

        - 'third_party/python/compare-locales/**'

        - 'third_party/python/fluent/**'

        - 'tools/lint/l10n.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

mingw-capitalization:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mingw-capitalization

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.cpp'

        - '**/*.cc'

        - '**/*.c'

        - '**/*.h'

        - 'tools/lint/mingw-capitalization.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

mscom-init:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mscom-init

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.cpp'

        - '**/*.cc'

        - '**/*.c'

        - '**/*.h'

        - 'tools/lint/mscom-init.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

file-whitespace:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l file-whitespace

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.c'

        - '**/*.cc'

        - '**/*.cpp'

        - '**/*.css'

        - '**/*.dtd'

        - '**/*.idl'

        - '**/*.ftl'

        - '**/*.h'

        - '**/*.html'

        - '**/*.md'

        - '**/*.properties'

        - '**/*.py'

        - '**/*.rs'

        - '**/*.rst'

        - '**/*.webidl'

        - '**/*.xhtml'

        - 'tools/lint/file-whitespace.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

test-manifest:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.ini'

        - 'python/mozlint/**'

        - 'tools/lint/**'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

trojan-source:

  extends: .base

  script:

    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l trojan-source

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.c'

        - '**/*.cc'

        - '**/*.cpp'

        - '**/*.h'

        - '**/*.py'

        - '**/*.rs'

        - 'tools/lint/trojan-source.yml'

    # Run job whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_REF_PROTECTED == 'true'

#!/usr/bin/env python3

import argparse

import os

import re

import shlex

import subprocess

def git(command):

    result = subprocess.run(

        ["git"] + shlex.split(command), check=True, capture_output=True, text=True

    )

    return result.stdout.strip()

def get_firefox_tag(reference):

    """Extracts the Firefox tag associated with a branch or tag name.

       The "firefox tag" is the tag that marks

       the end of the Mozilla commits and the start of the Tor Project commits.

       Know issue: If ever there is more than one tag per Firefox ESR version,

       this function may return the incorrect reference number.

    Args:

        reference: The branch or tag name to extract the Firefox tag from.

        Expected format is tor-browser-91.2.0esr-11.0-1,

        where 91.2.0esr is the Firefox version.

    Returns:

        The reference specifier of the matching Firefox tag.

        An exception will be raised if anything goes wrong.

    """

    # Extracts the version number from a branch or tag name.

    firefox_version = ""

    match = re.search(r"(?<=browser-)([^-]+)", reference)

    if match:

        # TODO: Validate that what we got is actually a valid semver string?

        firefox_version = match.group(1)

    else:

        raise ValueError(f"Failed to extract version from reference '{reference}'.")

    tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"

    remote_tags = git("ls-remote --tags origin")

    # Each line looks like:

    # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1

    pattern = rf"(.*){re.escape(tag)}(.*)$"

    match = re.search(pattern, remote_tags, flags=re.MULTILINE)

    if match:

        return match.group(0).split()[0]

    else:

        raise ValueError(

            f"Failed to find reference specifier for Firefox tag '{tag}' from '{reference}'."

        )

def get_list_of_changed_files():

    """Gets a list of files changed in the working directory.

       This function is meant to be run inside the Gitlab CI environment.

       When running in a default branch, get the list of changed files since the last Firefox tag.

       When running for a new MR commit, get a list of changed files in the current MR.

    Returns:

        A list of filenames of changed files (excluding deleted files).

        An exception wil be raised if anything goes wrong.

    """

    base_reference = ""

    if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":

        # For merge requests, the base_reference is the common ancestor between the MR and the target branch

        base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")

    else:

        # When not in merge requests, the base reference is the Firefox tag

        base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))

    if not base_reference:

        raise RuntimeError("No base reference found. There might be more errors above.")

    # Fetch the tag reference

    git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")

    # Return but filter the issue_templates files because those file names have spaces which can cause issues

    return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="")

    parser.add_argument(

        "--get-firefox-tag",

        help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",

        type=str,

    )

    parser.add_argument(

        "--get-changed-files",

        help="Get list of changed files."

        "When running from a merge request get sthe list of changed files since the merge-base of the current branch."

        "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",

        action="store_true",

    )

    args = parser.parse_args()

    if args.get_firefox_tag:

        print(get_firefox_tag(args.get_firefox_tag))

    elif args.get_changed_files:

        print("\n".join(get_list_of_changed_files()))

    else:

        print("No valid option provided.")

setup-env:

  stage: setup

  interruptible: true

  variables:

    GIT_STRATEGY: "none"

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'

  script:

    - |

      if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then

        echo "BRANCH_NAME=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" > build.env

      else

        echo "BRANCH_NAME=$CI_COMMIT_REF_NAME" > build.env

      fi

  artifacts:

    reports:

      dotenv:

        - build.env

create-bundle:

  stage: setup

  # TODO: Find a better suited image, this one just has git.

  image: python

  needs:

    - job: setup-env

      artifacts: true

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'

  variables:

    GIT_DEPTH: "1"

  interruptible: true

  script:

    # DEBUG: Check repository status.

    - git status

    # DEBUG: Check branches in repository

    - git branch

    # Force switch to a named branch. We force it in case there is already

    # a branch with the same name from previous runs.

    - git switch -C $BRANCH_NAME

    # Create a git bundle -- this will generate the app.bundle file,

    # which can be used as a git remote for offline fetching.

    - git bundle create app.bundle $BRANCH_NAME

    # Retain the SHA of the base of this shallow repository.

    - cat .git/shallow > shallow.txt

    # DEBUG: Check sizes.

    - du -sh .git

    - du -sh app.bundle

    # Since this is the only job we have access to the Gitlab remote,

    # let's get a list of changed files to use in the next jobs.

    - .gitlab/ci/scripts/helpers.py --get-changed-files > changedfiles.txt

  artifacts:

    paths:

      - app.bundle

      - shallow.txt

      - changedfiles.txt

    expire_in: 1 hour