Verified Commit 845b9843 authored by Beatriz Rizental's avatar Beatriz Rizental Committed by Pier Angelo Vendrame
Browse files

Add CI for Base Browser

parent d1fb6919

.gitlab-ci.yml

0 → 100644
+5 −0
Original line number Diff line number Diff line 
stages:

  - lint



include:

  - local: '.gitlab/ci/lint.yml'

.gitlab/ci/lint.yml

0 → 100644
+319 −0
Original line number Diff line number Diff line 
variables:

  # This needs to be kept in sync with the max Python version accepted by ./mach

  PYTHON_VERSION: "3.11.7"



.base:

  stage: lint

  interruptible: true

  variables:

    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

  cache:

    paths:

      - node_modules

      - .cache/pip



eslint:

  extends: .base

  image: cimg/python:$PYTHON_VERSION-node

  script:

    - .gitlab/ci/scripts/run_linters.py eslint

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        # Files that are likely audited.

        - '**/*.js'

        - '**/*.jsm'

        - '**/*.json'

        - '**/*.jsx'

        - '**/*.mjs'

        - '**/*.sjs'

        - '**/*.html'

        - '**/*.xhtml'

        - '**/*.xml'

        - 'tools/lint/eslint.yml'

        # Run when eslint policies change.

        - '**/.eslintignore'

        - '**/*eslintrc*'

        # The plugin implementing custom checks.

        - 'tools/lint/eslint/eslint-plugin-mozilla/**'

        - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



stylelint:

  extends: .base

  image: cimg/python:$PYTHON_VERSION-node

  script:

    - .gitlab/ci/scripts/run_linters.py stylelint

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        # Files that are likely audited.

        - '**/*.css'

        - 'tools/lint/styleint.yml'

        # Run when stylelint policies change.

        - '**/.stylelintignore'

        - '**/*stylelintrc*'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



py-black:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py black

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        # The list of extensions should match tools/lint/black.yml

        - '**/*.py'

        - '**/moz.build'

        - '**/*.configure'

        - '**/*.mozbuild'

        - 'pyproject.toml'

        - 'tools/lint/black.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



py-ruff:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py ruff

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.py'

        - '**/*.configure'

        - '**/.ruff.toml'

        - 'pyproject.toml'

        - 'tools/lint/ruff.yml'

        - 'tools/lint/python/ruff.py'

        - 'tools/lint/python/ruff_requirements.txt'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



yaml:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py yaml

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.yml'

        - '**/*.yaml'

        - '**/.ymllint'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



shellcheck:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py shellcheck

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.sh'

        - 'tools/lint/shellcheck.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



clang-format:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py clang-format

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.cpp'

        - '**/*.c'

        - '**/*.cc'

        - '**/*.h'

        - '**/*.m'

        - '**/*.mm'

        - 'tools/lint/clang-format.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



rustfmt:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py rustfmt

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.rs'

        - 'tools/lint/rustfmt.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



fluent-lint:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py fluent-lint

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.ftl'

        - 'tools/lint/fluent-lint.yml'

        - 'tools/lint/fluent-lint/exclusions.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



localization:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py l10n

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/locales/en-US/**'

        - '**/l10n.toml'

        - 'third_party/python/compare-locales/**'

        - 'third_party/python/fluent/**'

        - 'tools/lint/l10n.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



mingw-capitalization:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py mingw-capitalization

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.cpp'

        - '**/*.cc'

        - '**/*.c'

        - '**/*.h'

        - 'tools/lint/mingw-capitalization.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



mscom-init:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py mscom-init

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.cpp'

        - '**/*.cc'

        - '**/*.c'

        - '**/*.h'

        - 'tools/lint/mscom-init.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



file-whitespace:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py file-whitespace

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.c'

        - '**/*.cc'

        - '**/*.cpp'

        - '**/*.css'

        - '**/*.dtd'

        - '**/*.idl'

        - '**/*.ftl'

        - '**/*.h'

        - '**/*.html'

        - '**/*.md'

        - '**/*.properties'

        - '**/*.py'

        - '**/*.rs'

        - '**/*.rst'

        - '**/*.webidl'

        - '**/*.xhtml'

        - 'tools/lint/file-whitespace.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



test-manifest:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.ini'

        - 'python/mozlint/**'

        - 'tools/lint/**'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'



trojan-source:

  extends: .base

  image: cimg/python:$PYTHON_VERSION

  script:

    - .gitlab/ci/scripts/run_linters.py trojan-source

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      changes:

        # List copied from: taskcluster/ci/source-test/mozlint.yml

        #

        - '**/*.c'

        - '**/*.cc'

        - '**/*.cpp'

        - '**/*.h'

        - '**/*.py'

        - '**/*.rs'

        - 'tools/lint/trojan-source.yml'

    # Run job whenever a new tag is created

    # or whenever a commit is merged to a protected branch

    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'

.gitlab/ci/scripts/run_linters.py

0 → 100755
+110 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3



import argparse

import os

import re

import shlex

import subprocess

import sys





def git(command):

    result = subprocess.run(

        ["git"] + shlex.split(command), check=True, capture_output=True, text=True

    )

    return result.stdout.strip()





def get_firefox_tag_from_branch_name(branch_name):

    """Extracts the Firefox tag associated with a branch name.



       The "firefox tag" is the tag that marks

       the end of the Mozilla commits and the start of the Tor Project commits.



       Know issue: If ever there is more than one tag per Firefox ESR version,

       this function may return the incorrect reference number.



    Args:

        branch_name: The branch name to extract the tag from.

        Expected format is tor-browser-91.2.0esr-11.0-1,

        where 91.2.0esr is the Firefox version.



    Returns:

        The reference specifier of the matching Firefox tag.

        An exception wil be raised if anything goes wrong.

    """



    # Extracts the version number from a branch name.

    firefox_version = ""

    match = re.search(r"(?<=browser-)([^-]+)", branch_name)

    if match:

        # TODO: Validate that what we got is actually a valid semver string?

        firefox_version = match.group(1)

    else:

        raise ValueError(f"Failed to extract version from branch name '{branch_name}'.")



    tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"

    remote_tags = git("ls-remote --tags")



    # Each line looks like:

    # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1

    pattern = rf"(.*){re.escape(tag)}(.*)$"

    match = re.search(pattern, remote_tags, flags=re.MULTILINE)

    if match:

        return match.group(0).split()[0]

    else:

        raise ValueError(

            f"Failed to find reference specifier for Firefox tag in branch '{branch_name}'."

        )





def get_list_of_changed_files():

    """Gets a list of files changed in the working directory.



       This function is meant to be run inside the Gitlab CI environment.



       When running in a default branch, get the list of changed files since the last Firefox tag.

       When running for a new MR commit, get a list of changed files in the current MR.



    Returns:

        A list of filenames of changed files (excluding deleted files).

        An exception wil be raised if anything goes wrong.

    """



    base_reference = ""



    if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":

        # For merge requests, the base_reference is the common ancestor between the MR and the target branch.

        base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")

    else:

        # When not in merge requests, the base reference is the Firefox tag

        base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH"))



    if not base_reference:

        raise RuntimeError("No base reference found. There might be more errors above.")



    # Fetch the tag reference

    git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")

    # Return the list of changed files

    return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n")





if __name__ == "__main__":

    parser = argparse.ArgumentParser(

        description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history."

    )

    parser.add_argument(

        "linters", metavar="L", type=str, nargs="+", help="A list of linters to run."

    )

    args = parser.parse_args()



    command = [

        "./mach",

        "lint",

        "-v",

        *(s for l in args.linters for s in ("-l", l)),

        *get_list_of_changed_files(),

    ]

    result = subprocess.run(command, text=True)



    sys.exit(result.returncode)