Unverified Commit 282ad313 authored by Mitchell Hentges's avatar Mitchell Hentges Committed by GitHub
Browse files

Updates Fenix taskcluster tasks to support beta release (#1893) 

* Updates Fenix taskcluster tasks to support beta release

* Throw error if -PversionName isn't set for release builds

* Uses beta secrets for beta

* Improves nightly and beta treeherder symbols
parent db203148

.taskcluster.yml

+78 −62
Original line number Diff line number Diff line
@@ -50,11 +50,6 @@ tasks: 
      else: 1

  in:

    $let:

      track:

        $if: 'trust_level == 3'

        then: 'nightly'

        else: 'staging-nightly'



      # TODO: revisit once bug 1533314 is done to possibly infer better priorities

      tasks_priority: highest
@@ -128,8 +123,6 @@ tasks: 
            owner: ${user}@users.noreply.github.com

            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      in:

        $if: 'tasks_for in ["github-pull-request", "github-push"]'

        then:

        - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'

          then:

            $let:
@@ -175,7 +168,25 @@ tasks: 
                metadata:

                  name: Fenix - Decision task

                  description: Schedules the build and test tasks for Fenix.

        else:

        - $if: 'tasks_for == "github-release"'

          then:

            $mergeDeep:

              - {$eval: 'default_task_definition'}

              - scopes:

                - ${assume_scope_prefix}:release

                payload:

                  command:

                    - >-

                      git fetch ${repository} refs/tags/${head_rev}

                      && git config advice.detachedHead false

                      && git checkout FETCH_HEAD

                      && python automation/taskcluster/decision_task.py beta ${event.release.tag_name}

                extra:

                  treeherder:

                    symbol: beta-D

                metadata:

                  name: Fenix Beta Decision Task

                  description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name}

        - $if: 'tasks_for == "cron"'

          then:

            $mergeDeep:
@@ -185,21 +196,26 @@ tasks: 
                    then: assume:hook-id:project-mobile/fenix-nightly

                    else: assume:hook-id:project-mobile/fenix-nightly-staging

                routes:

                  $if: 'trust_level == 3'

                  then:

                    - notify.email.fenix-eng-notifications@mozilla.com.on-failed

                payload:

                  $let:

                    staging_flag:

                      $if: 'trust_level == 3'

                      then: ''

                      else: '--staging'

                  in:

                    command:

                      - >-

                        git fetch ${repository} ${head_branch}

                        && git config advice.detachedHead false

                        && git checkout FETCH_HEAD

                        && python automation/taskcluster/decision_task.py \

                          release \

                          --nightly \

                          --track ${track}

                        && python automation/taskcluster/decision_task.py nightly ${staging_flag}

                extra:

                  cron: {$json: {$eval: 'cron'}}

                  treeherder:

                      symbol: N

                    symbol: nightly-D

                metadata:

                  name: Fenix Nightly Decision Task

                  description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})

app/build.gradle

+28 −6
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ apply plugin: 'androidx.navigation.safeargs.kotlin' 


apply plugin: 'org.mozilla.appservices'



import com.android.build.gradle.internal.tasks.AppPreBuildTask



appservices {

    defaultConfig {

        megazord = 'fenix'
@@ -25,8 +27,8 @@ android { 
        applicationId "org.mozilla.fenix"

        minSdkVersion Config.minSdkVersion

        targetSdkVersion Config.targetSdkVersion

        versionCode Config.versionCode

        versionName Config.versionName + Config.generateVersionSuffix()

        versionCode 1

        versionName Config.generateDebugVersionName()

        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"

        testInstrumentationRunnerArguments clearPackageData: 'true'

        manifestPlaceholders.isRaptorEnabled = "false"
@@ -57,6 +59,7 @@ android { 
            resValue "bool", "IS_NOT_RELEASED", "false"

        }

        beta releaseTemplate >> {

            applicationIdSuffix ".beta"

            buildConfigField "boolean", "IS_RELEASED", "true"

            resValue "bool", "IS_NOT_RELEASED", "false"

        }
@@ -129,10 +132,11 @@ android.applicationVariants.all { variant -> 


    def buildType = variant.buildType.name

    def versionCode = null

    def isReleased = variant.buildType.buildConfigFields['IS_RELEASED']?.value ?: false



    buildConfigField 'Boolean', 'COLLECTIONS_ENABLED', (buildType != "release").toString()



    if (buildType == "nightly") {

    if (isReleased) {

        versionCode = generatedVersionCode



        // The Google Play Store does not allow multiple APKs for the same app that all have the
@@ -150,7 +154,25 @@ android.applicationVariants.all { variant -> 
        }// else variant.flavorName.contains("Arm")) use generated version code



        variant.outputs.all {

            setVersionCodeOverride(versionCode)

            versionCodeOverride = versionCode

            versionNameOverride = Config.releaseVersionName(project)

        }



        // If this is a release build, validate that "versionName" is set

        tasks.withType(AppPreBuildTask) { prebuildTask ->

            // You can't add a closure to a variant, so we need to look for an early variant-specific type

            // of task (AppPreBuildTask is the first) and filter to make sure we're looking at the task for

            // this variant that we're currently configuring

            if (prebuildTask.variantName != variant.name) {

                return

            }



            // Append to the task so the first thing it does is run our validation

            prebuildTask.doFirst {

                if (!project.hasProperty('versionName')) {

                    throw new RuntimeException("Release builds require the 'versionName' property, e.g.: './gradlew -PversionName=<...> assembleNightly'")

                }

            }

        }

    }
@@ -197,7 +219,7 @@ android.applicationVariants.all { variant -> 


    print("Adjust token: ")



    if (variant.buildType.buildConfigFields['IS_RELEASED']?.value) {

    if (isReleased) {

        try {

            def token = new File("${rootDir}/.adjust_token").text.trim()

            buildConfigField 'String', 'ADJUST_TOKEN', '"' + token + '"'

automation/taskcluster/decision_task.py

+21 −12
Original line number Diff line number Diff line
@@ -9,7 +9,10 @@ Decision task for nightly releases. 
from __future__ import print_function



import argparse

import datetime

import os

import re



import taskcluster



from lib.gradle import get_build_variants, get_geckoview_versions
@@ -94,8 +97,7 @@ def pr_or_push(is_push): 
    return (build_tasks, signing_tasks, other_tasks)





def nightly(track):

    is_staging = track == 'staging-nightly'

def release(track, is_staging, version_name):

    architectures = ['x86', 'arm', 'aarch64']

    apk_paths = ["public/target.{}.apk".format(arch) for arch in architectures]
@@ -104,12 +106,13 @@ def nightly(track): 
    push_tasks = {}



    build_task_id = taskcluster.slugId()

    build_tasks[build_task_id] = BUILDER.craft_assemble_nightly_task(architectures, is_staging)

    build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(architectures, track, is_staging, version_name)



    signing_task_id = taskcluster.slugId()

    signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task(

    signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(

        build_task_id,

        apk_paths=apk_paths,

        track=track,

        is_staging=is_staging,

    )
@@ -117,6 +120,7 @@ def nightly(track): 
    push_tasks[push_task_id] = BUILDER.craft_push_task(

        signing_task_id,

        apks=apk_paths,

        track=track,

        is_staging=is_staging,

    )
@@ -132,15 +136,14 @@ if __name__ == "__main__": 


    subparsers.add_parser('pull-request')

    subparsers.add_parser('push')

    release_parser = subparsers.add_parser('release')



    release_parser.add_argument('--nightly', action="store_true", default=False)

    release_parser.add_argument(

        '--track', action="store", choices=['nightly', 'staging-nightly'], required=True

    )

    nightly_parser = subparsers.add_parser('nightly')

    nightly_parser.add_argument('--staging', action='store_true')



    result = parser.parse_args()

    release_parser = subparsers.add_parser('beta')

    release_parser.add_argument('tag')



    result = parser.parse_args()

    command = result.command

    taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
@@ -148,8 +151,14 @@ if __name__ == "__main__": 
        ordered_groups_of_tasks = pr_or_push(False)

    elif command == 'push':

        ordered_groups_of_tasks = pr_or_push(True)

    elif command == 'release':

        ordered_groups_of_tasks = nightly(result.track)

    elif command == 'nightly':

        formatted_date = datetime.datetime.now().strftime('%y%V')

        ordered_groups_of_tasks = release('nightly', result.staging, '1.0.{}'.format(formatted_date))

    elif command == 'beta':

        semver = re.compile(r'^\d+\.\d+\.\d+-beta\.\d+$')

        if not semver.match(result.tag):

            raise ValueError('Github tag must be in beta semver format, e.g.: "1.0.0-beta.0')

        ordered_groups_of_tasks = release('beta', False, result.tag)

    else:

        raise Exception('Unsupported command "{}"'.format(command))

automation/taskcluster/lib/tasks.py

+42 −36
Original line number Diff line number Diff line
@@ -7,10 +7,9 @@ from __future__ import print_function 
import arrow

import datetime

import json

import os

import taskcluster



from lib.util import convert_camel_case_into_kebab_case, lower_case_first_letter

from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter



DEFAULT_EXPIRES_IN = '1 year'

DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
@@ -41,26 +40,29 @@ class TaskBuilder(object): 
        self.date = arrow.get(date_string)

        self.trust_level = trust_level



    def craft_assemble_nightly_task(self, architectures, is_staging=False):

    def craft_assemble_release_task(self, architectures, track, is_staging, version_name):

        artifacts = {

            'public/target.{}.apk'.format(arch): {

                "type": 'file',

                "path": '/opt/fenix/app/build/outputs/apk/'

                        '{}/nightly/app-{}-nightly-unsigned.apk'.format(arch, arch),

                        '{arch}/{track}/app-{arch}-{track}-unsigned.apk'.format(arch=arch, track=track),

                "expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),

            }

            for arch in architectures

        }



        sentry_secret = '{}project/mobile/fenix/sentry'.format(

            'garbage/staging/' if is_staging else ''

        )

        leanplum_secret = '{}project/mobile/fenix/leanplum'.format(

            'garbage/staging/' if is_staging else ''

        )

        adjust_secret = '{}project/mobile/fenix/adjust'.format(

            'garbage/staging/' if is_staging else ''

        )

        def secret_index(name):

            if is_staging:

                return 'garbage/staging/project/mobile/fenix/{}'.format(name)

            elif track == 'nightly':

                # TODO: Move nightly secrets to "project/mobile/fenix/nightly/..."

                return 'project/mobile/fenix/{}'.format(name)

            else:

                return 'project/mobile/fenix/{}/{}'.format(track, name)



        sentry_secret = secret_index('sentry')

        leanplum_secret = secret_index('leanplum')

        adjust_secret = secret_index('adjust')



        pre_gradle_commands = (

            'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
@@ -69,12 +71,14 @@ class TaskBuilder(object): 
            for secret, key, target_file in (

                (sentry_secret, 'dsn', '.sentry_token'),

                (leanplum_secret, 'production', '.leanplum_token'),

                (adjust_secret, 'Greenfield', '.adjust_token'),

                (adjust_secret, 'adjust', '.adjust_token'),

            )

        )



        capitalized_track = upper_case_first_letter(track)

        gradle_commands = (

            './gradlew --no-daemon -PcrashReports=true -Ptelemetry=true clean test assembleNightly',

            './gradlew --no-daemon -PcrashReports=true -Ptelemetry=true -PversionName={} clean test assemble{}'.format(

                version_name, capitalized_track),

        )



        command = ' && '.join(
@@ -89,8 +93,8 @@ class TaskBuilder(object): 
        ]



        return self._craft_build_ish_task(

            name='Build task',

            description='Build Fenix from source code',

            name='Build {} task'.format(capitalized_track),

            description='Build Fenix {} from source code'.format(capitalized_track),

            command=command,

            scopes=[

                "secrets:get:{}".format(secret) for secret in (sentry_secret, leanplum_secret, adjust_secret)
@@ -102,7 +106,7 @@ class TaskBuilder(object): 
                'machine': {

                    'platform': 'android-all',

                },

                'symbol': 'NA',

                'symbol': '{}-A'.format(track),

                'tier': 1,

            },

        )
@@ -175,7 +179,7 @@ class TaskBuilder(object): 
        return self._craft_clean_gradle_task(

            name='lint',

            description='Running lint for aarch64 release variant',

            gradle_task='lintAarch64Nightly',

            gradle_task='lintDebug',

            treeherder={

                'jobKind': 'test',

                'machine': {
@@ -277,13 +281,13 @@ class TaskBuilder(object): 
        }



        return self._craft_default_task_definition(

            worker_type='mobile-signing-dep-v1' if signing_type == 'dep-signing' else 'mobile-signing-v1',

            worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1',

            provisioner_id='scriptworker-prov-v1',

            dependencies=[assemble_task_id],

            routes=routes,

            scopes=[

                "project:mobile:fenix:releng:signing:format:{}".format(signing_format),

                "project:mobile:fenix:releng:signing:cert:{}".format(signing_type),

                "project:mobile:fenix:releng:signing:cert:{}-signing".format(signing_type),

            ],

            name=name,

            description=description,
@@ -365,7 +369,7 @@ class TaskBuilder(object): 
        return self._craft_signing_task(

            name='sign: {}'.format(variant),

            description='Dep-signing variant {}'.format(variant),

            signing_type="dep-signing",

            signing_type="dep",

            assemble_task_id=assemble_task_id,

            apk_paths=["public/target.apk"],

            routes=routes,
@@ -380,24 +384,26 @@ class TaskBuilder(object): 
            },

        )



    def craft_nightly_signing_task(

        self, build_task_id, apk_paths, is_staging=True,

    def craft_release_signing_task(

        self, build_task_id, apk_paths, track, is_staging=False,

    ):

        index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly'

        capitalized_track = upper_case_first_letter(track)

        index_release = 'staging.{}'.format(track) if is_staging else track



        routes = [

            "index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format(

            "index.project.mobile.fenix.v2.{}.{}.{}.{}.latest".format(

                index_release, self.date.year, self.date.month, self.date.day

            ),

            "index.project.mobile.fenix.{}.nightly.{}.{}.{}.revision.{}".format(

            "index.project.mobile.fenix.v2.{}.{}.{}.{}.revision.{}".format(

                index_release, self.date.year, self.date.month, self.date.day, self.commit

            ),

            "index.project.mobile.fenix.{}.nightly.latest".format(index_release),

            "index.project.mobile.fenix.v2.{}.latest".format(index_release),

        ]



        return self._craft_signing_task(

            name="Signing task",

            description="Sign release builds of Fenix",

            signing_type="dep-signing" if is_staging else "release-signing",

            name="Signing {} task".format(capitalized_track),

            description="Sign {} builds of Fenix".format(capitalized_track),

            signing_type="dep" if is_staging else track,

            assemble_task_id=build_task_id,

            apk_paths=apk_paths,

            routes=routes,
@@ -406,17 +412,17 @@ class TaskBuilder(object): 
                'machine': {

                  'platform': 'android-all',

                },

                'symbol': 'Ns',

                'symbol': '{}-s'.format(track),

                'tier': 1,

            },

        )



    def craft_push_task(

        self, signing_task_id, apks, is_staging=True

        self, signing_task_id, apks, track, is_staging=False

    ):

        payload = {

            "commit": True,

            "google_play_track": 'nightly',

            "google_play_track": track,

            "upstreamArtifacts": [

                {

                    "paths": apks,
@@ -444,7 +450,7 @@ class TaskBuilder(object): 
                'machine': {

                  'platform': 'android-all',

                },

                'symbol': 'gp',

                'symbol': '{}-gp'.format(track),

                'tier': 1,

            },

        )

automation/taskcluster/lib/util.py

+4 −0
Original line number Diff line number Diff line
@@ -9,3 +9,7 @@ def convert_camel_case_into_kebab_case(string): 


def lower_case_first_letter(string):

    return '{}{}'.format(string[0].lower(), string[1:])





def upper_case_first_letter(string):

    return string[0].upper() + string[1:]