From 5854963fb31044f18089159ace37e6c0d377853c Mon Sep 17 00:00:00 2001
From: Steve Fink <sfink@mozilla.com>
Date: Thu, 9 Sep 2021 16:15:11 +0000
Subject: [PATCH] Bug 1726619 - Replace MOZ_LOW_PARALLELISM_BUILD with
automated heuristic parallelism based on memory usage, and use for hazard
builds r=firefox-build-system-reviewers,glandium
Differential Revision: https://phabricator.services.mozilla.com/D123147
---
js/src/devtools/rootAnalysis/mach_commands.py | 1 +
python/mozbuild/mozbuild/base.py | 33 +++++++++++++------
python/mozbuild/mozbuild/build_commands.py | 14 +++++++-
.../mozbuild/mozbuild/controller/building.py | 5 +++
.../ci/build/linux-base-toolchains.yml | 2 --
taskcluster/ci/build/linux.yml | 1 -
6 files changed, 42 insertions(+), 14 deletions(-)
diff --git a/js/src/devtools/rootAnalysis/mach_commands.py b/js/src/devtools/rootAnalysis/mach_commands.py
index 80128f5a0aa45..f329073d4b8f1 100644
--- a/js/src/devtools/rootAnalysis/mach_commands.py
+++ b/js/src/devtools/rootAnalysis/mach_commands.py
@@ -253,6 +253,7 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
buildscript = " ".join(
[
command_context.topsrcdir + "/mach hazards compile",
+ "--job-size=3.0", # Conservatively estimate 3GB/process
"--application=" + application,
"--haz-objdir=" + objdir,
]
diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py
index 2ee479e466dd1..52a461a5bcd0c 100644
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -43,6 +43,11 @@ from .util import (
memoized_property,
)
+try:
+ import psutil
+except Exception:
+ psutil = None
+
def ancestors(path):
"""Emit the parent directories of a path."""
@@ -731,6 +736,7 @@ class MozbuildObject(ProcessExecutionMixin):
print_directory=True,
pass_thru=False,
num_jobs=0,
+ job_size=0,
keep_going=False,
):
"""Invoke make.
@@ -775,17 +781,24 @@ class MozbuildObject(ProcessExecutionMixin):
else:
args.append(flag)
- if num_jobs > 0:
- args.append("-j%d" % num_jobs)
- elif os.environ.get("MOZ_LOW_PARALLELISM_BUILD"):
+ if num_jobs == 0:
+ if job_size == 0:
+ job_size = 2.0 if self.substs.get("CC_TYPE") == "gcc" else 1.0 # GiB
+
cpus = multiprocessing.cpu_count()
- jobs = max(1, int(0.75 * cpus))
- print(
- " Low parallelism requested: using %d jobs for %d cores" % (jobs, cpus)
- )
- args.append("-j%d" % jobs)
- else:
- args.append("-j%d" % multiprocessing.cpu_count())
+ if not psutil or not job_size:
+ num_jobs = cpus
+ else:
+ mem_gb = psutil.virtual_memory().total / 1024 ** 3
+ from_mem = round(mem_gb / job_size)
+ num_jobs = max(1, min(cpus, from_mem))
+ print(
+ " Parallelism determined by memory: using %d jobs for %d cores "
+ "based on %.1f GiB RAM and estimated job size of %.1f GiB"
+ % (num_jobs, cpus, mem_gb, job_size)
+ )
+
+ args.append("-j%d" % num_jobs)
if ignore_errors:
args.append("-k")
diff --git a/python/mozbuild/mozbuild/build_commands.py b/python/mozbuild/mozbuild/build_commands.py
index f7b49e9d11581..3de75bb7c03e8 100644
--- a/python/mozbuild/mozbuild/build_commands.py
+++ b/python/mozbuild/mozbuild/build_commands.py
@@ -40,7 +40,16 @@ class Build(MachCommandBase):
default="0",
metavar="jobs",
type=int,
- help="Number of concurrent jobs to run. Default is the number of CPUs.",
+ help="Number of concurrent jobs to run. Default is based on the number of "
+ "CPUs and the estimated size of the jobs (see --job-size).",
+ )
+ @CommandArgument(
+ "--job-size",
+ default="0",
+ metavar="size",
+ type=float,
+ help="Estimated RAM required, in GiB, for each parallel job. Used to "
+ "compute a default number of concurrent jobs.",
)
@CommandArgument(
"-C",
@@ -65,6 +74,7 @@ class Build(MachCommandBase):
command_context,
what=None,
jobs=0,
+ job_size=0,
directory=None,
verbose=False,
keep_going=False,
@@ -115,6 +125,7 @@ class Build(MachCommandBase):
command_context.metrics,
what=what,
jobs=jobs,
+ job_size=job_size,
directory=directory,
verbose=verbose,
keep_going=keep_going,
@@ -156,6 +167,7 @@ class Build(MachCommandBase):
command_context.metrics,
what=what,
jobs=jobs,
+ job_size=job_size,
directory=directory,
verbose=verbose,
keep_going=keep_going,
diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py
index bf359f209e841..4645eec3678f2 100644
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1064,6 +1064,7 @@ class BuildDriver(MozbuildObject):
metrics,
what=None,
jobs=0,
+ job_size=0,
directory=None,
verbose=False,
keep_going=False,
@@ -1255,6 +1256,7 @@ class BuildDriver(MozbuildObject):
print_directory=False,
ensure_exit_code=False,
num_jobs=jobs,
+ job_size=job_size,
silent=not verbose,
append_env=tgt_env,
keep_going=keep_going,
@@ -1269,6 +1271,7 @@ class BuildDriver(MozbuildObject):
status = self._run_client_mk(
line_handler=output.on_line,
jobs=jobs,
+ job_size=job_size,
verbose=verbose,
keep_going=keep_going,
append_env=append_env,
@@ -1619,6 +1622,7 @@ class BuildDriver(MozbuildObject):
target=None,
line_handler=None,
jobs=0,
+ job_size=0,
verbose=None,
keep_going=False,
append_env=None,
@@ -1701,6 +1705,7 @@ class BuildDriver(MozbuildObject):
line_handler=line_handler,
log=False,
num_jobs=jobs,
+ job_size=job_size,
silent=not verbose,
keep_going=keep_going,
append_env=append_env,
diff --git a/taskcluster/ci/build/linux-base-toolchains.yml b/taskcluster/ci/build/linux-base-toolchains.yml
index d4856e3dc08ab..2eafb126f60d5 100644
--- a/taskcluster/ci/build/linux-base-toolchains.yml
+++ b/taskcluster/ci/build/linux-base-toolchains.yml
@@ -19,7 +19,6 @@ linux64-base-toolchains/opt:
worker:
max-run-time: 7200
env:
- MOZ_LOW_PARALLELISM_BUILD: '1'
PERFHERDER_EXTRA_OPTIONS: base-toolchains
FORCE_GCC: '1'
run:
@@ -59,7 +58,6 @@ linux64-base-toolchains/debug:
worker:
max-run-time: 7200
env:
- MOZ_LOW_PARALLELISM_BUILD: '1'
PERFHERDER_EXTRA_OPTIONS: base-toolchains
FORCE_GCC: '1'
run:
diff --git a/taskcluster/ci/build/linux.yml b/taskcluster/ci/build/linux.yml
index 1287b5997752d..ccc5ca28141dd 100644
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -183,7 +183,6 @@ linux64-gcc/opt:
worker:
max-run-time: 7200
env:
- MOZ_LOW_PARALLELISM_BUILD: '1'
PERFHERDER_EXTRA_OPTIONS: gcc
FORCE_GCC: '1'
run:
--
GitLab