Commit 13648fa0 authored by Johan Lorenzo's avatar Johan Lorenzo
Browse files

Bug 1563711 - Remove target.maven.zip r=nalexander 

Differential Revision: https://phabricator.services.mozilla.com/D38175

--HG--
extra : moz-landing-system : lando
parent 7826afea

Makefile.in

+7 −2
Original line number Diff line number Diff line
@@ -159,8 +159,13 @@ endif 


ifdef MOZ_ANDROID_FAT_AAR_ARCHITECTURES

recurse_android-fat-aar-artifact:

	$(call py_action,fat_aar,$(MOZ_ANDROID_FAT_AAR_ARCHITECTURES) --distdir $(abspath $(DIST)/fat-aar))

endif # MOZ_ANDROID_FAT_AAR_ARCHITECTURES

	$(call py_action,fat_aar,\

    $(if $(MOZ_ANDROID_FAT_AAR_ARMEABI_V7A),--armeabi-v7a $(MOZ_ANDROID_FAT_AAR_ARMEABI_V7A)) \

    $(if $(MOZ_ANDROID_FAT_AAR_ARM64_V8A),--arm64-v8a $(MOZ_ANDROID_FAT_AAR_ARM64_V8A)) \

    $(if $(MOZ_ANDROID_FAT_AAR_X86),--x86 $(MOZ_ANDROID_FAT_AAR_X86)) \

    $(if $(MOZ_ANDROID_FAT_AAR_X86_64),--x86-64 $(MOZ_ANDROID_FAT_AAR_X86_64)) \

    --distdir $(abspath $(DIST)/fat-aar))

endif



ifdef MOZ_WIDGET_TOOLKIT

ifdef ENABLE_TESTS

mobile/android/mach_commands.py

+1 −30
Original line number Diff line number Diff line
@@ -9,8 +9,6 @@ import logging 
import os

import json



from zipfile import ZipFile



import mozpack.path as mozpath



from mozbuild.base import (
@@ -401,15 +399,8 @@ class MachCommands(MachCommandBase): 
            self.substs['GRADLE_ANDROID_ARCHIVE_GECKOVIEW_TASKS'] + args,

            verbose=True)



        if ret != 0:

        return ret



        # TODO Bug 1563711 - Remove target.maven.zip

        # The zip archive is passed along in CI to ship geckoview onto a maven repo

        _craft_maven_zip_archive(self.topobjdir)



        return 0



    @SubCommand('android', 'build-geckoview_example',

                """Build geckoview_example """)

    @CommandArgument('args', nargs=argparse.REMAINDER)
@@ -706,26 +697,6 @@ class MachCommands(MachCommandBase): 
        return 0





def _get_maven_archive_abs_and_relative_paths(maven_folder):

    for subdir, _, files in os.walk(maven_folder):

        for file in files:

            full_path = os.path.join(subdir, file)

            relative_path = os.path.relpath(full_path, maven_folder)



            # maven-metadata is intended to be generated on the real maven server

            if 'maven-metadata.xml' not in relative_path:

                yield full_path, relative_path





def _craft_maven_zip_archive(topobjdir):

    geckoview_folder = os.path.join(topobjdir, 'gradle/build/mobile/android/geckoview')

    maven_folder = os.path.join(geckoview_folder, 'maven')



    with ZipFile(os.path.join(geckoview_folder, 'target.maven.zip'), 'w') as target_zip:

        for abs, rel in _get_maven_archive_abs_and_relative_paths(maven_folder):

            target_zip.write(abs, arcname=rel)





@CommandProvider

class AndroidEmulatorCommands(MachCommandBase):

    """

python/mozbuild/mozbuild/action/fat_aar.py

+32 −67
Original line number Diff line number Diff line
@@ -10,8 +10,6 @@ compatibility, and ready inputs to an Android multi-architecture fat AAR build. 
from __future__ import absolute_import, unicode_literals, print_function



import argparse

import buildconfig

import subprocess

import sys



from collections import (
@@ -29,38 +27,9 @@ from mozpack.packager.unpack import UnpackFinder 
import mozpack.path as mozpath





def _download_zips(distdir, architectures):

    # The mapping from Android CPU architecture to TC job is defined here, and the TC index

    # lookup is mediated by python/mozbuild/mozbuild/artifacts.py and

    # python/mozbuild/mozbuild/artifact_builds.py.

    jobs = {

        'arm64-v8a': 'android-aarch64-opt',

        'armeabi-v7a': 'android-api-16-opt',

        'x86': 'android-x86-opt',

        'x86_64': 'android-x86_64-opt',

    }



    for arch in architectures:

        # It's unfortunate that we must couple tightly, but that's the current API for

        # dispatching.  In automation, MOZ_ARTIFACT_TASK* environment variables will ensure

        # that the correct tasks are chosen as install sources.

        subprocess.check_call([sys.executable, mozpath.join(buildconfig.topsrcdir, 'mach'),

                               'artifact', 'install',

                               '--job', jobs[arch],

                               '--distdir', mozpath.join(distdir, 'input', arch),

                               '--no-tests', '--no-process', '--maven-zip'])





def fat_aar(distdir, architectures=[],

            no_download=False, no_process=False, no_compatibility_check=False,

            rewrite_old_archives=False):

    if not no_download:

        _download_zips(distdir, architectures)

    else:

        print('Not downloading architecture-specific artifact Maven zips.')



def fat_aar(distdir, aars_paths, no_process=False, no_compatibility_check=False):

    if no_process:

        print('Not processing architecture-specific artifact Maven zips.')

        print('Not processing architecture-specific artifact Maven AARs.')

        return 0



    # Map {filename: {fingerprint: [arch1, arch2, ...]}}.
@@ -69,7 +38,7 @@ def fat_aar(distdir, architectures=[], 
    # Collect multi-architecture inputs to the fat AAR.

    copier = FileCopier()



    for arch in architectures:

    for arch, aar_path in aars_paths.items():

        # Map old non-architecture-specific path to new architecture-specific path.

        old_rewrite_map = {

            'greprefs.js': '{}/greprefs.js'.format(arch),
@@ -80,16 +49,7 @@ def fat_aar(distdir, architectures=[], 
        arch_prefs = set(old_rewrite_map.values())

        missing_arch_prefs |= set(arch_prefs)



        path = mozpath.join(distdir, 'input', arch, 'target.maven.zip')



        aars = list(JarFinder(path, JarReader(path)).find('**/geckoview-*.aar'))

        if len(aars) != 1:

            raise ValueError('Maven zip "{path}" with more than one candidate AAR found: {aars}'

                             .format(path=path, aars=tuple(sorted(p for p, _ in aars))))



        [aar_path, aar_file] = aars[0]



        jar_finder = JarFinder(aar_file.file.filename, JarReader(fileobj=aar_file.open()))

        jar_finder = JarFinder(aar_path, JarReader(aar_path))

        for path, fileobj in UnpackFinder(jar_finder):

            # Native libraries go straight through.

            if mozpath.match(path, 'jni/**'):
@@ -98,14 +58,6 @@ def fat_aar(distdir, architectures=[], 
            elif path in arch_prefs:

                copier.add(path, fileobj)



            elif rewrite_old_archives and path in old_rewrite_map:

                # Ease testing during transition by allowing old omnijars that don't have

                # architecture-specific files yet.

                new_path = old_rewrite_map[path]

                print('Rewrote old path "{path}" to new path "{new_path}"'.format(

                    path=path, new_path=new_path))

                copier.add(new_path, fileobj)



            elif path in ('classes.jar', 'annotations.zip'):

                # annotations.zip differs due to timestamps, but the contents should not.
@@ -172,36 +124,49 @@ def fat_aar(distdir, architectures=[], 
    if not no_compatibility_check and (missing_arch_prefs or not_allowed):

        return 1



    copier.copy(mozpath.join(distdir, 'output'))

    output_dir = mozpath.join(distdir, 'output')

    copier.copy(output_dir)



    return 0





_ALL_ARCHS = ('armeabi-v7a', 'arm64-v8a', 'x86_64', 'x86')





def main(argv):

    description = '''Fetch and unpack architecture-specific Maven zips, verify cross-architecture

    description = '''Unpack architecture-specific Maven AARs, verify cross-architecture

compatibility, and ready inputs to an Android multi-architecture fat AAR build.'''



    parser = argparse.ArgumentParser(description=description)

    parser.add_argument('--no-download', action='store_true',

                        help='Do not fetch Maven zips.')

    parser.add_argument('--no-process', action='store_true',

                        help='Do not process Maven zips.')

                        help='Do not process Maven AARs.')

    parser.add_argument('--no-compatibility-check', action='store_true',

                        help='Do not fail if Maven zips are not compatible.')

    parser.add_argument('--rewrite-old-archives', action='store_true',

                        help='Rewrite Maven zips containing omnijars that do not contain '

                             'architecture-specific preference files.')

                        help='Do not fail if Maven AARs are not compatible.')

    parser.add_argument('--distdir', required=True)

    parser.add_argument('architectures', nargs='+',

                        choices=('armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'))



    for arch in _ALL_ARCHS:

        command_line_flag = arch.replace('_', '-')

        parser.add_argument('--{}'.format(command_line_flag), dest=arch)



    args = parser.parse_args(argv)



    args_dict = vars(args)



    aars_paths = {

        arch: args_dict.get(arch)

        for arch in _ALL_ARCHS

        if args_dict.get(arch)

    }



    if not aars_paths:

        raise ValueError('You must provide at least one AAR file!')



    return fat_aar(

        args.distdir, architectures=args.architectures,

        no_download=args.no_download, no_process=args.no_process,

        no_compatibility_check=args.no_compatibility_check,

        rewrite_old_archives=args.rewrite_old_archives)

        args.distdir,

        aars_paths,

        no_process=args.no_process,

        no_compatibility_check=args.no_compatibility_check

    )





if __name__ == '__main__':

taskcluster/ci/build-fat-aar/kind.yml

0 → 100644
+122 −0
Original line number Diff line number Diff line 
# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.

---

loader: taskgraph.loader.transform:loader



kind-dependencies:

    - build

    - toolchain



transforms:

    - taskgraph.transforms.build:transforms

    - taskgraph.transforms.build_attrs:transforms

    - taskgraph.transforms.build_lints:transforms

    - taskgraph.transforms.build_fat_aar:transforms

    - taskgraph.transforms.use_toolchains:transforms

    - taskgraph.transforms.job:transforms

    - taskgraph.transforms.task:transforms



job-defaults:

    attributes:

        artifact_map: taskcluster/taskgraph/manifests/fennec_geckoview.yml

    index:

        product: mobile

    fetches: {}   # See build_fat_aar transform

    worker-type: b-linux

    worker:

        docker-image: {in-tree: android-build}

        max-run-time: 7200

        env:

            # Online in order to download the per-architecture AARs.

            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"

            TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"

            MOZ_ANDROID_FAT_AAR_ARCHITECTURES: "armeabi-v7a,arm64-v8a,x86,x86_64"

        artifacts:

            - name: public/build/maven

              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/

              type: directory

            - name: public/build

              path: /builds/worker/artifacts/

              type: directory

    run:

        using: mozharness

        script: "mozharness/scripts/fx_desktop_build.py"

        secrets: true

        mozconfig-variant: null

        tooltool-downloads: internal

        custom-build-variant-cfg: api-16

        # Note: These settings are only honored by nightly (i.e. shipping) builds

        update-channel:

            by-release-type:

                nightly: nightly

                nightly-oak: nightly-oak

                beta:

                    by-shipping-product:

                        devedition: aurora

                        default: beta

                release.*: release

                esr.*: esr

                default: null

    toolchains:

        - android-gradle-dependencies

        - android-ndk-linux

        - android-sdk-linux

        - linux64-clang

        - linux64-rust-android

        - linux64-rust-size

        - linux64-cbindgen

        - linux64-nasm

        - linux64-node



jobs:

    android-geckoview-fat-aar/opt:

        description: "Android GeckoView multi-architecture fat AAR Opt"

        index:

            job-name: android-geckoview-fat-aar-opt

        treeherder:

            platform: android-4-0-geckoview-fat-aar/opt

            symbol: Bgv

        dependencies:

            android-x86-opt: build-android-x86/opt

            android-x86_64-opt: build-android-x86_64/opt

            android-api-16-opt: build-android-api-16/opt

            android-aarch64-opt: build-android-aarch64/opt

        worker:

            env:

                PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-opt

                USE_ARTIFACT: '1'

                MOZ_ARTIFACT_TASK: {task-reference: '<android-api-16-opt>'}

        run:

            actions: [get-secrets, build]

            config: ["builds/releng_base_android_64_builds.py"]

        toolchains:

            - linux64-sccache



    android-geckoview-fat-aar-nightly/opt:

        description: "Android GeckoView multi-architecture fat AAR Nightly"

        attributes:

            nightly: true

            enable-full-crashsymbols: true

            disable-push-apk: true

        shipping-phase: build

        shipping-product: fennec

        index:

            job-name: android-geckoview-fat-aar-nightly

            type: nightly-with-multi-l10n

        treeherder:

            platform: android-4-0-geckoview-fat-aar/opt

            symbol: Ngv

        dependencies:

            android-x86-nightly: build-android-x86-nightly/opt

            android-x86_64-nightly: build-android-x86_64-nightly/opt

            android-api-16-nightly: build-android-api-16-nightly/opt

            android-aarch64-nightly: build-android-aarch64-nightly/opt

        worker:

            env:

                PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-nightly

        run:

            actions: [get-secrets, build, multi-l10n]

            config:

                - builds/releng_base_android_64_builds.py

                - taskcluster_nightly.py

taskcluster/ci/build-signing/kind.yml

+1 −0
Original line number Diff line number Diff line
@@ -12,3 +12,4 @@ transforms: 


kind-dependencies:

    - build

    - build-fat-aar