Commit 25308d30 authored by Mike Shal's avatar Mike Shal
Browse files

Bug 1563403 - Use 3-tier PGO for local MOZ_PGO=1 builds;... 

Bug 1563403 - Use 3-tier PGO for local MOZ_PGO=1 builds; r=firefox-build-system-reviewers,dmajor,chmanchester

Local PGO builds now use 3-tier machinery under the hood. Instead of a
single object directory that gets cleaned in between the instrumented
and profile-use builds, now the instrumented build uses a separate
'${objdir}/instrumented' directory. This makes it easier to handle
within mach since we can drive the two builds with environment variables
and keep all build artifacts separate, without needing to do manual
cleanup in between.

Differential Revision: https://phabricator.services.mozilla.com/D50098

--HG--
extra : moz-landing-system : lando
parent 554a8c64

Makefile.in

+1 −48
Original line number Diff line number Diff line
@@ -115,11 +115,8 @@ install-test-files: 


include $(topsrcdir)/build/moz-automation.mk



# dist and _tests should be purged during cleaning. However, we don't want them

# purged during PGO builds because they contain some auto-generated files.

ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)

# dist and _tests should be purged during cleaning.

GARBAGE_DIRS += dist _tests

endif



# Dummy rule for the cases below where we don't depend on dist/include

recurse_pre-export::
@@ -185,37 +182,6 @@ endif 
default all::

	$(call BUILDSTATUS,TIERS $(TIERS) $(if $(MOZ_AUTOMATION),$(MOZ_AUTOMATION_TIERS)))



# PGO build target.

profiledbuild::

	$(call BUILDSTATUS,TIERS pgo_profile_generate pgo_package pgo_profile pgo_clobber pgo_profile_use)

	$(call BUILDSTATUS,TIER_START pgo_profile_generate)

	$(MAKE) default MOZ_PROFILE_GENERATE=1 MOZ_LTO=

	$(call BUILDSTATUS,TIER_FINISH pgo_profile_generate)

	$(call BUILDSTATUS,TIER_START pgo_package)

	$(MAKE) package

	rm -f jarlog/en-US.log

	$(call BUILDSTATUS,TIER_FINISH pgo_package)

	$(call BUILDSTATUS,TIER_START pgo_profile)

	JARLOG_FILE=jarlog/en-US.log $(PYTHON) $(topsrcdir)/build/pgo/profileserver.py

	$(call BUILDSTATUS,TIER_FINISH pgo_profile)

	$(call BUILDSTATUS,TIER_START pgo_clobber)

	$(MAKE) maybe_clobber_profiledbuild

	$(call BUILDSTATUS,TIER_FINISH pgo_clobber)

	$(call BUILDSTATUS,TIER_START pgo_profile_use)

	$(MAKE) default MOZ_PROFILE_USE=1 MOZ_1TIER_PGO=1

	$(call BUILDSTATUS,TIER_FINISH pgo_profile_use)



# Change default target to PGO build if PGO is enabled.

ifdef MOZ_PGO

ifdef COMPILE_ENVIRONMENT

# If one of these is already set in addition to PGO we are doing a single phase

# of PGO in isolation, so don't override the default target.

ifeq (,$(MOZ_PROFILE_GENERATE)$(MOZ_PROFILE_USE))

OVERRIDE_DEFAULT_GOAL := profiledbuild

endif

endif

endif



include $(topsrcdir)/config/rules.mk



ifdef SCCACHE_VERBOSE_STATS
@@ -281,19 +247,6 @@ update-packaging: 
package-generated-sources:

	$(call py_action,package_generated_sources,'$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)')



# PGO support, but we can't do this test in client.mk

# No point in clobbering if PGO has been explicitly disabled.

ifdef NO_PROFILE_GUIDED_OPTIMIZE

maybe_clobber_profiledbuild:

else

maybe_clobber_profiledbuild: clean

ifneq (,$(findstring clang,$(CC_TYPE)))

	$(LLVM_PROFDATA) merge -o $(DEPTH)/merged.profdata $(DEPTH)/*.profraw

endif

endif # NO_PROFILE_GUIDED_OPTIMIZE



.PHONY: maybe_clobber_profiledbuild



ifdef JS_STANDALONE

# Delegate js-specific rules to js

check-%:

build/docs/pgo.rst

+3 −14
Original line number Diff line number Diff line
@@ -23,17 +23,6 @@ Then:: 


This is roughly equivalent to::



#. Perform a build with *MOZ_PROFILE_GENERATE=1*

#. Performing a run of the instrumented binaries

#. $ make maybe_clobber_profiledbuild

#. Perform a build with *MOZ_PROFILE_USE=1*



Differences between toolchains

==============================



There are some implementation differences depending on the compiler

toolchain being used.



The *maybe_clobber_profiledbuild* step gets its name because of a

difference. On Windows, this step merely moves some *.pgc* files around.

Using GCC or Clang, it is equivalent to a *make clean*.
 
#. Perform a build with *--enable-profile-generate* in $topobjdir/instrumented

#. Perform a run of the instrumented binaries with build/pgo/profileserver.py

#. Perform a build with *--enable-profile-use* in $topobjdir

build/moz.configure/lto-pgo.configure

+3 −1
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ llvm_profdata = check_prog('LLVM_PROFDATA', ['llvm-profdata'], 
                           paths=toolchain_search_path)



js_option('--enable-profile-generate',

          env='MOZ_PROFILE_GENERATE',

          nargs='?',

          choices=('cross',),

          help='Build a PGO instrumented binary')
@@ -25,6 +26,7 @@ set_define('MOZ_PROFILE_GENERATE', 
           depends_if('--enable-profile-generate')(lambda _: True))



js_option('--enable-profile-use',

          env='MOZ_PROFILE_USE',

          nargs='?',

          choices=('cross',),

          help='Use a generated profile during the build')
@@ -51,7 +53,7 @@ def pgo_profile_path(path, pgo_use, profdata, build_env): 
        topobjdir = topobjdir[:-7]



    if not path:

        return os.path.join(topobjdir, 'merged.profdata')

        return os.path.join(topobjdir,  'instrumented', 'merged.profdata')

    if path and not pgo_use:

        die('Pass --enable-profile-use to use --with-pgo-profile-path.')

    if path and not profdata:

build/pgo/profileserver.py

+6 −0
Original line number Diff line number Diff line
@@ -80,6 +80,10 @@ if __name__ == '__main__': 
                       port=PORT,

                       options='primary,privileged')



    old_profraw_files = glob.glob('*.profraw')

    for f in old_profraw_files:

        os.remove(f)



    with TemporaryDirectory() as profilePath:

        # TODO: refactor this into mozprofile

        profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
@@ -151,6 +155,8 @@ if __name__ == '__main__': 
        if jarlog:

            env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)

            print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])

            if os.path.exists(jarlog):

                os.remove(jarlog)



        if 'UPLOAD_PATH' in env:

            process_args['logfile'] = os.path.join(env['UPLOAD_PATH'], 'profile-run-2.log')

python/mozbuild/mozbuild/build_commands.py

+54 −1
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals 


import argparse

import os

import subprocess



from mach.decorators import (

    CommandArgument,
@@ -14,6 +15,9 @@ from mach.decorators import ( 
)



from mozbuild.base import MachCommandBase

from mozbuild.util import ensure_subprocess_env

from mozbuild.mozconfig import MozconfigLoader

import mozpack.path as mozpath



from mozbuild.backend import (

    backends,
@@ -73,6 +77,54 @@ class Build(MachCommandBase): 


        self.log_manager.enable_all_structured_loggers()



        loader = MozconfigLoader(self.topsrcdir)

        mozconfig = loader.read_mozconfig(loader.AUTODETECT)

        doing_pgo = 'MOZ_PGO=1' in mozconfig['configure_args']

        append_env = None



        if doing_pgo:

            if what:

                raise Exception('Cannot specify targets (%s) in MOZ_PGO=1 builds' %

                                what)

            instr = self._spawn(BuildDriver)

            orig_topobjdir = instr._topobjdir

            instr._topobjdir = mozpath.join(instr._topobjdir, 'instrumented')



            append_env = {'MOZ_PROFILE_GENERATE': '1'}

            status = instr.build(

                what=what,

                disable_extra_make_dependencies=disable_extra_make_dependencies,

                jobs=jobs,

                directory=directory,

                verbose=verbose,

                keep_going=keep_going,

                mach_context=self._mach_context,

                append_env=append_env)

            if status != 0:

                return status



            # Packaging the instrumented build is required to get the jarlog

            # data.

            status = instr._run_make(

                directory=".", target='package',

                silent=not verbose, ensure_exit_code=False,

                append_env=append_env)

            if status != 0:

                return status



            pgo_env = os.environ.copy()

            pgo_env['LLVM_PROFDATA'] = instr.config_environment.substs.get('LLVM_PROFDATA')

            pgo_env['JARLOG_FILE'] = mozpath.join(orig_topobjdir, 'jarlog/en-US.log')

            pgo_cmd = [

                instr.virtualenv_manager.python_path,

                mozpath.join(self.topsrcdir, 'build/pgo/profileserver.py'),

            ]

            subprocess.check_call(pgo_cmd, cwd=instr.topobjdir,

                                  env=ensure_subprocess_env(pgo_env))



            # Set the default build to MOZ_PROFILE_USE

            append_env = {'MOZ_PROFILE_USE': '1'}



        driver = self._spawn(BuildDriver)

        return driver.build(

            what=what,
@@ -81,7 +133,8 @@ class Build(MachCommandBase): 
            directory=directory,

            verbose=verbose,

            keep_going=keep_going,

            mach_context=self._mach_context)

            mach_context=self._mach_context,

            append_env=append_env)



    @Command('configure', category='build',

             description='Configure the tree (run configure and config.status).')