Skip to content
Snippets Groups Projects
Commit da8c75ef authored by Beatriz Rizental's avatar Beatriz Rizental Committed by Beatriz Rizental
Browse files

Add CI for Base Browser

parent 2723db51
Branches
Tags
1 merge request!1507Rebase Tor Browser onto 136.0a1
stages:
- lint
- update-translations
variables:
IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest
LOCAL_REPO_PATH: /srv/apps-repos/tor-browser.git
include:
- local: '.gitlab/ci/mixins.yml'
- local: '.gitlab/ci/jobs/lint/lint.yml'
- local: '.gitlab/ci/jobs/update-translations.yml'
FROM debian:latest
# Base image which includes all* dependencies checked by ./mach configure.
#
# * Actually not all dependencies. WASM sandboxed depencies were left out for now.
# This installs all dependencies checked by `./mach configure --without-wasm-sandboxed-libraries`.
#
# # Building and publishing
#
# Whenever this file changes, the updated Docker image must be built and published _manually_ to
# the tor-browser container registry (https://gitlab.torproject.org/tpo/applications/tor-browser/container_registry/185).
#
# This image copies a script from the taskcluster/ folder, which requires it
# to be built from a folder which is a parent of the taskcluster/ folder.
#
# To build, run:
#
# ```bash
# docker build \
# -f <PATH_TO_DOCKERFILE> \
# -t <REGISTRY_URL>/<IMAGE_NAME>:<IMAGE_TAG>
# .
# ```
#
# For example, when building from the root of this repository to the main tor-browser repository
# and assuming image name to be "base" and tag "latest" -- which is the current terminology:
#
# ```bash
# docker build \
# -f .gitlab/ci/docker/Dockerfile \
# -t containers.torproject.org/tpo/applications/tor-browser/base:latest
# .
# ```
RUN apt-get update && apt-get install -y \
clang \
curl \
git \
libasound2-dev \
libdbus-glib-1-dev \
libgtk-3-dev \
libpango1.0-dev \
libpulse-dev \
libx11-xcb-dev \
libxcomposite-dev \
libxcursor-dev \
libxdamage-dev \
libxi-dev \
libxrandr-dev \
libxtst-dev \
m4 \
mercurial \
nasm \
pkg-config \
python3 \
python3-pip \
unzip \
wget
COPY taskcluster/docker/recipes/install-node.sh /scripts/install-node.sh
RUN chmod +x /scripts/install-node.sh
RUN /scripts/install-node.sh
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
RUN $HOME/.cargo/bin/cargo install cbindgen
WORKDIR /app
CMD ["/bin/bash"]
#!/usr/bin/env python3
import argparse
import os
import re
import shlex
import subprocess
def git(command):
result = subprocess.run(
["git"] + shlex.split(command), check=True, capture_output=True, text=True
)
return result.stdout.strip()
def get_firefox_tag(reference):
"""Extracts the Firefox tag associated with a branch or tag name.
The "firefox tag" is the tag that marks
the end of the Mozilla commits and the start of the Tor Project commits.
Know issue: If ever there is more than one tag per Firefox ESR version,
this function may return the incorrect reference number.
Args:
reference: The branch or tag name to extract the Firefox tag from.
Expected format is tor-browser-91.2.0esr-11.0-1,
where 91.2.0esr is the Firefox version.
Returns:
The reference specifier of the matching Firefox tag.
An exception will be raised if anything goes wrong.
"""
# Extracts the version number from a branch or tag name.
firefox_version = ""
match = re.search(r"(?<=browser-)([^-]+)", reference)
if match:
# TODO: Validate that what we got is actually a valid semver string?
firefox_version = match.group(1)
else:
raise ValueError(f"Failed to extract version from reference '{reference}'.")
major_version = firefox_version.split(".")[0]
minor_patch_version = "_".join(firefox_version.split(".")[1:])
remote_tags = git("ls-remote --tags origin")
# Each line looks like:
# 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1
pattern = (
rf"(.*)FIREFOX_{re.escape(major_version)}_{re.escape(minor_patch_version)}(.*)$"
)
match = re.search(pattern, remote_tags, flags=re.MULTILINE)
if not match:
# Attempt to match with a nightly tag, in case the ESR tag is not found
pattern = rf"(.*)FIREFOX_NIGHTLY_{re.escape(major_version)}(.*)$"
match = re.search(pattern, remote_tags, flags=re.MULTILINE)
if match:
return match.group(0).split()[0]
else:
raise ValueError(
f"Failed to find reference specifier for Firefox tag of version '{firefox_version}' from '{reference}'."
)
def get_list_of_changed_files():
"""Gets a list of files changed in the working directory.
This function is meant to be run inside the Gitlab CI environment.
When running in a default branch, get the list of changed files since the last Firefox tag.
When running for a new MR commit, get a list of changed files in the current MR.
Returns:
A list of filenames of changed files (excluding deleted files).
An exception wil be raised if anything goes wrong.
"""
base_reference = ""
if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
# For merge requests, the base_reference is the common ancestor between the MR and the target branch
base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
else:
# When not in merge requests, the base reference is the Firefox tag
base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))
if not base_reference:
raise RuntimeError("No base reference found. There might be more errors above.")
# Fetch the tag reference
git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
# Return but filter the issue_templates files because those file names have spaces which can cause issues
return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="")
parser.add_argument(
"--get-firefox-tag",
help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",
type=str,
)
parser.add_argument(
"--get-changed-files",
help="Get list of changed files."
"When running from a merge request get sthe list of changed files since the merge-base of the current branch."
"When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",
action="store_true",
)
args = parser.parse_args()
if args.get_firefox_tag:
print(get_firefox_tag(args.get_firefox_tag))
elif args.get_changed_files:
print("\n".join(get_list_of_changed_files()))
else:
print("No valid option provided.")
.base:
extends: .with-local-repo-bash
stage: lint
image: $IMAGE_PATH
interruptible: true
variables:
MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"
cache:
paths:
- node_modules
- .cache/mozbuild
# Store the cache regardless on job outcome
when: 'always'
# Share the cache throughout all pipelines running for a given branch
key: $CI_COMMIT_REF_SLUG
tags:
# Run these jobs in the browser dedicated runners.
- firefox
eslint:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l eslint
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
# Files that are likely audited.
- '**/*.js'
- '**/*.jsm'
- '**/*.json'
- '**/*.jsx'
- '**/*.mjs'
- '**/*.sjs'
- '**/*.html'
- '**/*.xhtml'
- '**/*.xml'
- 'tools/lint/eslint.yml'
# Run when eslint policies change.
- '**/.eslintignore'
- '**/*eslintrc*'
# The plugin implementing custom checks.
- 'tools/lint/eslint/eslint-plugin-mozilla/**'
- 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
stylelint:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l stylelint
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
# Files that are likely audited.
- '**/*.css'
- 'tools/lint/styleint.yml'
# Run when stylelint policies change.
- '**/.stylelintignore'
- '**/*stylelintrc*'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
py-black:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l black
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
# The list of extensions should match tools/lint/black.yml
- '**/*.py'
- '**/moz.build'
- '**/*.configure'
- '**/*.mozbuild'
- 'pyproject.toml'
- 'tools/lint/black.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
py-ruff:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l ruff
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.py'
- '**/*.configure'
- '**/.ruff.toml'
- 'pyproject.toml'
- 'tools/lint/ruff.yml'
- 'tools/lint/python/ruff.py'
- 'tools/lint/python/ruff_requirements.txt'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
yaml:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l yaml
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.yml'
- '**/*.yaml'
- '**/.ymllint'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
shellcheck:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l shellcheck
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.sh'
- 'tools/lint/shellcheck.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
clang-format:
extends: .base
script:
- ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l clang-format
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.cpp'
- '**/*.c'
- '**/*.cc'
- '**/*.h'
- '**/*.m'
- '**/*.mm'
- 'tools/lint/clang-format.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
rustfmt:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l rustfmt
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.rs'
- 'tools/lint/rustfmt.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
fluent-lint:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l fluent-lint
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.ftl'
- 'tools/lint/fluent-lint.yml'
- 'tools/lint/fluent-lint/exclusions.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
localization:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l l10n
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/locales/en-US/**'
- '**/l10n.toml'
- 'third_party/python/compare-locales/**'
- 'third_party/python/fluent/**'
- 'tools/lint/l10n.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
mingw-capitalization:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mingw-capitalization
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.cpp'
- '**/*.cc'
- '**/*.c'
- '**/*.h'
- 'tools/lint/mingw-capitalization.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
mscom-init:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mscom-init
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.cpp'
- '**/*.cc'
- '**/*.c'
- '**/*.h'
- 'tools/lint/mscom-init.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
file-whitespace:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l file-whitespace
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.c'
- '**/*.cc'
- '**/*.cpp'
- '**/*.css'
- '**/*.dtd'
- '**/*.idl'
- '**/*.ftl'
- '**/*.h'
- '**/*.html'
- '**/*.md'
- '**/*.properties'
- '**/*.py'
- '**/*.rs'
- '**/*.rst'
- '**/*.webidl'
- '**/*.xhtml'
- '**/*.java'
- 'tools/lint/file-whitespace.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
test-manifest:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.ini'
- 'python/mozlint/**'
- 'tools/lint/**'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
trojan-source:
extends: .base
script:
- .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l trojan-source
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
# List copied from: taskcluster/ci/source-test/mozlint.yml
#
- '**/*.c'
- '**/*.cc'
- '**/*.cpp'
- '**/*.h'
- '**/*.py'
- '**/*.rs'
- 'tools/lint/trojan-source.yml'
# Run job whenever a commit is merged to a protected branch
- if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
.update-translation-base:
stage: update-translations
rules:
- if: ($TRANSLATION_FILES != "" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push")
changes:
- "**/*.ftl"
- "**/*.properties"
- "**/*.dtd"
- "**/*strings.xml"
- "**/update-translations.yml"
- "**/l10n/combine/combine.py"
- "**/l10n/combine-translation-versions.py"
- if: ($TRANSLATION_FILES != "" && $FORCE_UPDATE_TRANSLATIONS == "true")
variables:
COMBINED_FILES_JSON: "combined-translation-files.json"
TRANSLATION_FILES: ''
combine-en-US-translations:
extends: .update-translation-base
needs: []
image: python
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
paths:
- .cache/pip
# Artifact is for translation project job
artifacts:
paths:
- "$COMBINED_FILES_JSON"
expire_in: "60 min"
reports:
dotenv: job_id.env
# Don't load artifacts for this job.
dependencies: []
script:
# Save this CI_JOB_ID to the dotenv file to be used in the variables for the
# push-en-US-translations job.
- echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env
- pip install compare_locales
- python ./tools/base-browser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$COMBINED_FILES_JSON"
push-en-US-translations:
extends: .update-translation-base
needs:
- job: combine-en-US-translations
variables:
COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${COMBINED_FILES_JSON}"
trigger:
strategy: depend
project: tor-browser-translation-bot/translation
branch: tor-browser-ci
.with-local-repo-bash:
variables:
GIT_STRATEGY: "none"
before_script:
- git init
- git remote add local "$LOCAL_REPO_PATH"
- |
# Determine the reference of the target branch in the local repository copy.
#
# 1. List all references in the local repository
# 2. Filter the references to the target branch
# 3. Remove tags
# 4. Keep a single line, in case there are too many matches
# 5. Clean up the output
# 6. Remove everything before the last two slashes, because the output is like `refs/heads/...` or `refs/remotes/...`
TARGET_BRANCH=$(git ls-remote local | grep ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME} | grep -v 'refs/tags/' | awk '{print $2}' | tail -1 | sed 's|[^/]*/[^/]*/||')
if [ -z "$TARGET_BRANCH" ]; then
echo "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."
exit 1
fi
- git fetch --depth 500 local $TARGET_BRANCH
- git remote add origin "$CI_REPOSITORY_URL"
- |
if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then
echo "No branch specified. Stopping the pipeline."
exit 1
fi
- echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
- |
if ! git fetch origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"; then
echo -e "\e[31mFetching failed for branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} from $CI_REPOSITORY_URL.\e[0m"
echo "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."
git fetch origin "merge-requests/${CI_MERGE_REQUEST_IID}/head"
fi
- git checkout FETCH_HEAD
.with-local-repo-pwsh:
variables:
GIT_STRATEGY: "none"
before_script:
- git init
- git remote add local $env:LOCAL_REPO_PATH
- |
$branchName = $env:CI_COMMIT_BRANCH
if ([string]::IsNullOrEmpty($branchName)) {
$branchName = $env:CI_MERGE_REQUEST_TARGET_BRANCH_NAME
}
$TARGET_BRANCH = git ls-remote local | Select-String -Pattern $branchName | Select-String -Pattern -NotMatch 'refs/tags/' | Select-Object -Last 1 | ForEach-Object { $_.ToString().Split()[1] -replace '^[^/]*/[^/]*/', '' }
if ([string]::IsNullOrEmpty($TARGET_BRANCH)) {
Write-Output "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."
exit 1
}
- git remote add origin $env:CI_REPOSITORY_URL
- |
$branchName = $env:CI_COMMIT_BRANCH
if ([string]::IsNullOrEmpty($branchName)) {
$branchName = $env:CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
}
if ([string]::IsNullOrEmpty($branchName)) {
Write-Output "No branch specified. Stopping the pipeline."
exit 1
}
- Write-Output "Fetching from remote branch $branchName"
- |
if (! git fetch origin $branchName) {
Write-Output "Fetching failed for branch $branchName from $env:CI_REPOSITORY_URL."
Write-Output "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."
git fetch origin "merge-requests/$env:CI_MERGE_REQUEST_IID/head"
}
- git checkout FETCH_HEAD
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment