Bug 780329 - Part 5: Add base modules to mozbuild; r=glandium, jhammel

python/mozbuild/mozbuild/base.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import logging
+import multiprocessing
+import os
+import pymake.parser
+import shlex
+import subprocess
+import which
+
+from mozprocess.processhandler import ProcessHandlerMixin
+from pymake.data import Makefile
+from tempfile import TemporaryFile
+
+from mozbuild.config import ConfigProvider
+from mozbuild.config import PositiveIntegerType
+
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if 'SHELL' in os.environ:
+    _current_shell = os.environ['SHELL']
+elif 'MOZILLABUILD' in os.environ:
+    _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+elif 'COMSPEC' in os.environ:
+    _current_shell = os.environ['COMSPEC']
+else:
+    raise Exception('Could not detect environment shell!')
+
+_in_msys = False
+
+if os.environ.get('MSYSTEM', None) == 'MINGW32':
+    _in_msys = True
+
+    if not _current_shell.lower().endswith('.exe'):
+        _current_shell += '.exe'
+
+
+class MozbuildObject(object):
+    """Base class providing basic functionality useful to many modules.
+
+    Modules in this package typically require common functionality such as
+    accessing the current config, getting the location of the source directory,
+    running processes, etc. This classes provides that functionality. Other
+    modules can inherit from this class to obtain this functionality easily.
+    """
+    def __init__(self, topsrcdir, settings, log_manager, topobjdir=None):
+        """Create a new Mozbuild object instance.
+
+        Instances are bound to a source directory, a ConfigSettings instance,
+        and a LogManager instance. The topobjdir may be passed in as well. If
+        it isn't, it will be calculated from the active mozconfig.
+        """
+        self.topsrcdir = topsrcdir
+        self.settings = settings
+        self.config = BuildConfig(settings)
+        self.logger = logging.getLogger(__name__)
+        self.log_manager = log_manager
+
+        self._config_guess_output = None
+        self._make = None
+        self._topobjdir = topobjdir
+
+    @property
+    def topobjdir(self):
+        if self._topobjdir is None:
+            self._load_mozconfig()
+
+        if self._topobjdir is None:
+            self._topobjdir = 'obj-%s' % self._config_guess
+
+        return self._topobjdir
+
+    @property
+    def distdir(self):
+        return os.path.join(self.topobjdir, 'dist')
+
+    @property
+    def bindir(self):
+        return os.path.join(self.topobjdir, 'dist', 'bin')
+
+    @property
+    def statedir(self):
+        return os.path.join(self.topobjdir, '.mozbuild')
+
+    def log(self, level, action, params, format_str):
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def _load_mozconfig(self, path=None):
+        # The mozconfig loader outputs a make file. We parse and load this make
+        # file with pymake and evaluate it in a context similar to client.mk.
+
+        loader = os.path.join(self.topsrcdir, 'build', 'autoconf',
+            'mozconfig2client-mk')
+
+        # os.environ from a library function is somewhat evil. But, mozconfig
+        # files are tightly coupled with the environment by definition. In the
+        # future, perhaps we'll have a more sanitized environment for mozconfig
+        # execution.
+        env = dict(os.environ)
+        if path is not None:
+            env['MOZCONFIG'] = path
+
+        env['CONFIG_GUESS'] = self._config_guess
+
+        output = subprocess.check_output([loader, self.topsrcdir],
+            stderr=subprocess.PIPE, cwd=self.topsrcdir, env=env)
+
+        # The output is make syntax. We parse this in a specialized make
+        # context.
+        statements = pymake.parser.parsestring(output, 'mozconfig')
+
+        makefile = Makefile(workdir=self.topsrcdir, env={
+            'TOPSRCDIR': self.topsrcdir,
+            'CONFIG_GUESS': self._config_guess})
+
+        statements.execute(makefile)
+
+        def get_value(name):
+            exp = makefile.variables.get(name)[2]
+
+            return exp.resolvestr(makefile, makefile.variables)
+
+        for name, flavor, source, value in makefile.variables:
+            # We only care about variables that came from the parsed mozconfig.
+            if source != pymake.data.Variables.SOURCE_MAKEFILE:
+                continue
+
+            # Ignore some pymake built-ins.
+            if name in ('.PYMAKE', 'MAKELEVEL', 'MAKEFLAGS'):
+                continue
+
+            if name == 'MOZ_OBJDIR':
+                self._topobjdir = get_value(name)
+
+            # If we want to extract other variables defined by mozconfig, here
+            # is where we'd do it.
+
+    @property
+    def _config_guess(self):
+        if self._config_guess_output is None:
+            p = os.path.join(self.topsrcdir, 'build', 'autoconf',
+                'config.guess')
+            self._config_guess_output = subprocess.check_output([p],
+                cwd=self.topsrcdir).strip()
+
+        return self._config_guess_output
+
+    def _ensure_objdir_exists(self):
+        if os.path.isdir(self.statedir):
+            return
+
+        os.makedirs(self.statedir)
+
+    def _ensure_state_subdir_exists(self, subdir):
+        path = os.path.join(self.statedir, subdir)
+
+        if os.path.isdir(path):
+            return
+
+        os.makedirs(path)
+
+    def _get_state_filename(self, filename, subdir=None):
+        path = self.statedir
+
+        if subdir:
+            path = os.path.join(path, subdir)
+
+        return os.path.join(path, filename)
+
+    def _get_srcdir_path(self, path):
+        """Convert a relative path in the source directory to a full path."""
+        return os.path.join(self.topsrcdir, path)
+
+    def _get_objdir_path(self, path):
+        """Convert a relative path in the object directory to a full path."""
+        return os.path.join(self.topobjdir, path)
+
+    def _run_make(self, directory=None, filename=None, target=None, log=True,
+            srcdir=False, allow_parallel=True, line_handler=None, env=None,
+            ignore_errors=False):
+        """Invoke make.
+
+        directory -- Relative directory to look for Makefile in.
+        filename -- Explicit makefile to run.
+        target -- Makefile target(s) to make. Can be a string or iterable of
+            strings.
+        srcdir -- If True, invoke make from the source directory tree.
+            Otherwise, make will be invoked from the object directory.
+        """
+        self._ensure_objdir_exists()
+
+        args = [self._make_path]
+
+        if directory:
+            args.extend(['-C', directory])
+
+        if filename:
+            args.extend(['-f', filename])
+
+        if allow_parallel:
+            args.append('-j%d' % self.settings.build.threads)
+
+        if ignore_errors:
+            args.append('-k')
+
+        # Silent mode by default.
+        args.append('-s')
+
+        # Print entering/leaving directory messages. Some consumers look at
+        # these to measure progress. Ideally, we'd do everything with pymake
+        # and use hooks in its API. Unfortunately, it doesn't provide that
+        # feature... yet.
+        args.append('-w')
+
+        if isinstance(target, list):
+            args.extend(target)
+        elif target:
+            args.append(target)
+
+        fn = self._run_command_in_objdir
+
+        if srcdir:
+            fn = self._run_command_in_srcdir
+
+        params = {
+            'args': args,
+            'line_handler': line_handler,
+            'explicit_env': env,
+            'log_level': logging.INFO,
+            'require_unix_environment': True,
+            'ignore_errors': ignore_errors,
+        }
+
+        if log:
+            params['log_name'] = 'make'
+
+        fn(**params)
+
+    @property
+    def _make_path(self):
+        if self._make is None:
+            if self._is_windows():
+                self._make = os.path.join(self.topsrcdir, 'build', 'pymake',
+                    'make.py')
+
+            else:
+                for test in ['gmake', 'make']:
+                    try:
+                        self._make = which.which(test)
+                        break
+                    except which.WhichError:
+                        continue
+
+        if self._make is None:
+            raise Exception('Could not find suitable make binary!')
+
+        return self._make
+
+    def _run_command_in_srcdir(self, **args):
+        self._run_command(cwd=self.topsrcdir, **args)
+
+    def _run_command_in_objdir(self, **args):
+        self._run_command(cwd=self.topobjdir, **args)
+
+    def _run_command(self, args=None, cwd=None, append_env=None,
+        explicit_env=None, log_name=None, log_level=logging.INFO,
+        line_handler=None, require_unix_environment=False,
+        ignore_errors=False):
+        """Runs a single command to completion.
+
+        Takes a list of arguments to run where the first item is the
+        executable. Runs the command in the specified directory and
+        with optional environment variables.
+
+        append_env -- Dict of environment variables to append to the current
+            set of environment variables.
+        explicit_env -- Dict of environment variables to set for the new
+            process. Any existing environment variables will be ignored.
+
+        require_unix_environment if True will ensure the command is executed
+        within a UNIX environment. Basically, if we are on Windows, it will
+        execute the command via an appropriate UNIX-like shell.
+        """
+        assert isinstance(args, list) and len(args)
+
+        if require_unix_environment and _in_msys:
+            # Always munge Windows-style into Unix style for the command.
+            prog = args[0].replace('\\', '/')
+
+            # PyMake removes the C: prefix. But, things seem to work here
+            # without it. Not sure what that's about.
+
+            # We run everything through the msys shell. We need to use
+            # '-c' and pass all the arguments as one argument because that is
+            # how sh works.
+            cline = subprocess.list2cmdline([prog] + args[1:])
+            args = [_current_shell, '-c', cline]
+
+        self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
+
+        def handleLine(line):
+            if line_handler:
+                line_handler(line)
+
+            if not log_name:
+                return
+
+            self.log(log_level, log_name, {'line': line.strip()}, '{line}')
+
+        use_env = {}
+        if explicit_env:
+            use_env = explicit_env
+        else:
+            use_env.update(os.environ)
+
+            if append_env:
+                use_env.update(env)
+
+        p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+            processOutputLine=[handleLine], universal_newlines=True)
+        p.run()
+        p.processOutput()
+        status = p.wait()
+
+        if status != 0 and not ignore_errors:
+            raise Exception('Process executed with non-0 exit code: %s' % args)
+
+    def _is_windows(self):
+        return os.name in ('nt', 'ce')
+
+    def _spawn(self, cls):
+        """Create a new MozbuildObject-derived class instance from ourselves.
+
+        This is used as a convenience method to create other
+        MozbuildObject-derived class instances. It can only be used on
+        classes that have the same constructor arguments as us.
+        """
+
+        return cls(self.topsrcdir, self.settings, self.log_manager,
+            topobjdir=self.topobjdir)
+
+
+class BuildConfig(ConfigProvider):
+    """The configuration for mozbuild."""
+
+    def __init__(self, settings):
+        self.settings = settings
+
+    @classmethod
+    def _register_settings(cls):
+        def register(section, option, type_cls, **kwargs):
+            cls.register_setting(section, option, type_cls, domain='mozbuild',
+                **kwargs)
+
+        register('build', 'threads', PositiveIntegerType,
+            default=multiprocessing.cpu_count())

python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po

+msgid "build.threads.short"
+msgstr "Thread Count"
+
+msgid "build.threads.full"
+msgstr "The number of threads to use when performing CPU intensive tasks. "
+       "This constrols the level of parallelization. The default value is "
+       "the number of cores in your machine."
+

python/mozbuild/mozbuild/test/test_base.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+from tempfile import NamedTemporaryFile
+
+from mozbuild.base import BuildConfig
+from mozbuild.base import MozbuildObject
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+
+curdir = os.path.dirname(__file__)
+topsrcdir = os.path.normpath(os.path.join(curdir, '..', '..', '..', '..'))
+log_manager = LoggingManager()
+
+
+class TestBuildConfig(unittest.TestCase):
+    def test_basic(self):
+        c = ConfigSettings()
+        c.register_provider(BuildConfig)
+
+        c.build.threads = 6
+
+
+class TestMozbuildObject(unittest.TestCase):
+    def get_base(self):
+        settings = ConfigSettings()
+        settings.register_provider(BuildConfig)
+
+        return MozbuildObject(topsrcdir, settings, log_manager)
+
+    def test_mozconfig_parsing(self):
+        with NamedTemporaryFile(mode='wt') as mozconfig:
+            mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir')
+            mozconfig.flush()
+
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            base = self.get_base()
+            base._load_mozconfig()
+
+            self.assertEqual(base.topobjdir, '%s/some-objdir' % topsrcdir)
+
+        del os.environ['MOZCONFIG']
+
+    def test_objdir_config_guess(self):
+        base = self.get_base()
+
+        with NamedTemporaryFile() as mozconfig:
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            self.assertIsNotNone(base.topobjdir)
+            self.assertEqual(len(base.topobjdir.split()), 1)
+
+        del os.environ['MOZCONFIG']