Bug 808357 - Part 2: Implement mozconfig finding and loading in Python; r=glandium

python/mozbuild/mozbuild/mozconfig.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 
 import os
+import re
 import subprocess
 
 import pymake.parser
@@ -13,60 +14,340 @@ from pymake.data import Makefile
 from mach.mixin.process import ProcessExecutionMixin
 
 
+MOZ_MYCONFIG_ERROR = '''
+The MOZ_MYCONFIG environment variable to define the location of mozconfigs
+is deprecated. If you wish to define the mozconfig path via an environment
+variable, use MOZCONFIG instead.
+'''.strip()
+
+MOZCONFIG_LEGACY_PATH = '''
+You currently have a mozconfig at %s. This implicit location is no longer
+supported. Please move it to %s/.mozconfig or set an explicit path
+via the $MOZCONFIG environment variable.
+'''.strip()
+
+
+class MozconfigFindException(Exception):
+    """Raised when a mozconfig location is not defined properly."""
+
+
+class MozconfigLoadException(Exception):
+    """Raised when a mozconfig could not be loaded properly.
+
+    This typically indicates a malformed or misbehaving mozconfig file.
+    """
+
+    def __init__(self, path, message):
+        self.path = path
+        Exception.__init__(self, message)
+
+
 class MozconfigLoader(ProcessExecutionMixin):
     """Handles loading and parsing of mozconfig files."""
 
+    RE_MAKE_VARIABLE = re.compile('''
+        ^\s*                    # Leading whitespace
+        (?P<var>[a-zA-Z_0-9]+)  # Variable name
+        \s* [?:]?= \s*          # Assignment operator surrounded by optional
+                                # spaces
+        (?P<value>.*$)''',      # Everything else (likely the value)
+        re.VERBOSE)
+
+    # Default mozconfig files in the topsrcdir.
+    DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig')
+
+    DEPRECATED_TOPSRCDIR_PATHS = ('mozconfig.sh', 'myconfig.sh')
+    DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh')
+
+    IGNORE_SHELL_VARIABLES = ('_')
+
     def __init__(self, topsrcdir):
         self.topsrcdir = topsrcdir
 
     @property
-    def _client_mk_loader_path(self):
-        return os.path.join(self.topsrcdir, 'build', 'autoconf',
-            'mozconfig2client-mk')
+    def _loader_script(self):
+        our_dir = os.path.abspath(os.path.dirname(__file__))
+
+        return os.path.join(our_dir, 'mozconfig_loader')
+
+    def find_mozconfig(self):
+        """Find the active mozconfig file for the current environment.
+
+        This emulates the logic in mozconfig-find.
+
+        1) If ENV[MOZCONFIG] is set, use that
+        2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it.
+        3) If both exist or if there are legacy locations detected, error out.
+
+        The absolute path to the found mozconfig will be returned on success.
+        None will be returned if no mozconfig could be found. A
+        MozconfigFindException will be raised if there is a bad state,
+        including conditions from #3 above.
+        """
+        # Check for legacy methods first.
+
+        if 'MOZ_MYCONFIG' in os.environ:
+            raise MozconfigFindException(MOZ_MYCONFIG_ERROR)
+
+        env_path = os.environ.get('MOZCONFIG', None)
+        if env_path is not None:
+            if not os.path.exists(env_path):
+                raise MozconfigFindException(
+                    'MOZCONFIG environment variable refers to a path that '
+                    'does not exist: ' + env_path)
+
+            if not os.path.isfile(env_path):
+                raise MozconfigFindException(
+                    'MOZCONFIG environment variable refers to a '
+                    'non-file: ' + env_path)
+
+        srcdir_paths = [os.path.join(self.topsrcdir, p) for p in
+            self.DEFAULT_TOPSRCDIR_PATHS]
+        existing = [p for p in srcdir_paths if os.path.isfile(p)]
+
+        if env_path is None and len(existing) > 1:
+            raise MozconfigFindException('Multiple default mozconfig files '
+                'present. Remove all but one. ' + ', '.join(existing))
+
+        path = None
+
+        if env_path is not None:
+            path = env_path
+        elif len(existing):
+            assert len(existing) == 1
+            path = existing[0]
 
-    def read_mozconfig(self, path=None):
-        env = dict(os.environ)
         if path is not None:
-            # Python < 2.7.3 doesn't like unicode strings in env keys.
-            env[b'MOZCONFIG'] = path
+            return os.path.abspath(path)
 
-        args = self._normalize_command([self._client_mk_loader_path,
-            self.topsrcdir], True)
+        deprecated_paths = [os.path.join(self.topsrcdir, s) for s in
+            self.DEPRECATED_TOPSRCDIR_PATHS]
 
-        output = subprocess.check_output(args, stderr=subprocess.PIPE,
-            cwd=self.topsrcdir, env=env)
+        home = os.environ.get('HOME', None)
+        if home is not None:
+            deprecated_paths.extend([os.path.join(home, s) for s in
+            self.DEPRECATED_HOME_PATHS])
+
+        for path in deprecated_paths:
+            if os.path.exists(path):
+                raise MozconfigFindException(
+                    MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir))
 
-        # The output is make syntax. We parse this in a specialized make
-        # context.
-        statements = pymake.parser.parsestring(output, 'mozconfig')
+        return None
 
-        makefile = Makefile(workdir=self.topsrcdir, env={
-            'TOPSRCDIR': self.topsrcdir})
+    def read_mozconfig(self, path=None):
+        """Read the contents of a mozconfig into a data structure.
 
-        statements.execute(makefile)
+        This takes the path to a mozconfig to load. If it is not defined, we
+        will try to find a mozconfig from the environment using
+        find_mozconfig().
+
+        mozconfig files are shell scripts. So, we can't just parse them.
+        Instead, we run the shell script in a wrapper which allows us to record
+        state from execution. Thus, the output from a mozconfig is a friendly
+        static data structure.
+        """
+        if path is None:
+            path = self.find_mozconfig()
 
         result = {
+            'path': path,
             'topobjdir': None,
+            'configure_args': None,
+            'make_flags': None,
+            'make_extra': None,
+            'env': None,
+        }
+
+        if path is None:
+            return result
+
+        path = path.replace(os.sep, '/')
+
+        result['configure_args'] = []
+        result['make_extra'] = []
+        result['make_flags'] = []
+
+        env = dict(os.environ)
+
+        args = self._normalize_command([self._loader_script, path], True)
+
+        output = subprocess.check_output(args, stderr=subprocess.PIPE,
+            cwd=self.topsrcdir, env=env)
+
+        parsed = self._parse_loader_output(output)
+
+        all_variables = set(parsed['vars_before'].keys())
+        all_variables |= set(parsed['vars_after'].keys())
+
+        changed = {
+            'added': {},
+            'removed': {},
+            'modified': {},
+            'unmodified': {},
         }
 
-        def get_value(name):
-            exp = makefile.variables.get(name)[2]
+        for key in all_variables:
+            if key in self.IGNORE_SHELL_VARIABLES:
+                continue
+
+            if key not in parsed['vars_before']:
+                changed['added'][key] = parsed['vars_after'][key]
+                continue
+
+            if key not in parsed['vars_after']:
+                changed['removed'][key] = parsed['vars_before'][key]
+                continue
+
+            if parsed['vars_before'][key] != parsed['vars_after'][key]:
+                changed['modified'][key] = (
+                    parsed['vars_before'][key], parsed['vars_after'][key])
+                continue
+
+            changed['unmodified'][key] = parsed['vars_after'][key]
+
+        result['env'] = changed
 
-            return exp.resolvestr(makefile, makefile.variables)
+        result['configure_args'] = \
+            [o.replace('@TOPSRCDIR@', self.topsrcdir) for o in parsed['ac']]
 
-        for name, flavor, source, value in makefile.variables:
-            # We only care about variables that came from the parsed mozconfig.
-            if source != pymake.data.Variables.SOURCE_MAKEFILE:
+        mk = [o.replace('@TOPSRCDIR@', self.topsrcdir) for o in parsed['mk']]
+
+        for o in mk:
+            match = self.RE_MAKE_VARIABLE.match(o)
+
+            if match is None:
+                result['make_extra'].append(o)
                 continue
 
-            # Ignore some pymake built-ins.
-            if name in ('.PYMAKE', 'MAKELEVEL', 'MAKEFLAGS'):
+            name, value = match.group('var'), match.group('value')
+
+            if name == 'MOZ_MAKE_FLAGS':
+                result['make_flags'] = value
                 continue
 
             if name == 'MOZ_OBJDIR':
-                result['topobjdir'] = get_value(name)
+                result['topobjdir'] = value
+                continue
 
-            # If we want to extract other variables defined by mozconfig, here
-            # is where we'd do it.
+            result['make_extra'].append(o)
 
         return result
+
+    def _parse_loader_output(self, output):
+        mk_options = []
+        ac_options = []
+        before_source = {}
+        after_source = {}
+
+        current = None
+        current_type = None
+        in_variable = None
+
+        for line in output.split('\n'):
+            if not len(line):
+                continue
+
+            if line.startswith('------BEGIN_'):
+                assert current_type is None
+                assert current is None
+                assert not in_variable
+                current_type = line[len('------BEGIN_'):]
+                current = []
+                continue
+
+            if line.startswith('------END_'):
+                assert not in_variable
+                section = line[len('------END_'):]
+                assert current_type == section
+
+                if current_type == 'AC_OPTION':
+                    ac_options.append('\n'.join(current))
+                elif current_type == 'MK_OPTION':
+                    mk_options.append('\n'.join(current))
+
+                current = None
+                current_type = None
+                continue
+
+            assert current_type is not None
+
+            if current_type in ('BEFORE_SOURCE', 'AFTER_SOURCE'):
+                # mozconfigs are sourced using the Bourne shell (or at least
+                # in Bourne shell mode). This means |set| simply lists
+                # variables from the current shell (not functions). (Note that
+                # if Bash is installed in /bin/sh it acts like regular Bourne
+                # and doesn't print functions.) So, lines should have the
+                # form:
+                #
+                #  key='value'
+                #  key=value
+                #
+                # The only complication is multi-line variables. Those have the
+                # form:
+                #
+                #  key='first
+                #  second'
+
+                # TODO Bug 818377 Properly handle multi-line variables of form:
+                # $ foo="a='b'
+                # c='d'"
+                # $ set
+                # foo='a='"'"'b'"'"'
+                # c='"'"'d'"'"
+
+                name = in_variable
+                value = None
+
+                if in_variable:
+                    # Reached the end of a multi-line variable.
+                    if line.endswith("'") and not line.endswith("\\'"):
+                        current.append(line[:-1])
+                        value = '\n'.join(current)
+                        in_variable = None
+                    else:
+                        current.append(line)
+                        continue
+                else:
+                    equal_pos = line.find('=')
+
+                    if equal_pos < 1:
+                        # TODO log warning
+                        continue
+
+                    name = line[0:equal_pos]
+                    has_quote = line[equal_pos + 1] == "'"
+
+                    if has_quote:
+                        value = line[equal_pos + 2:]
+                    else:
+                        value = line[equal_pos + 1:]
+
+                    # Lines with a quote not ending in a quote are multi-line.
+                    if has_quote and not value.endswith("'"):
+                        in_variable = name
+                        current.append(value)
+                        continue
+                    else:
+                        value = value[:-1] if has_quote else value
+
+                assert name is not None
+
+                if current_type == 'BEFORE_SOURCE':
+                    before_source[name] = value
+                else:
+                    after_source[name] = value
+
+                current = []
+
+                continue
+
+            current.append(line)
+
+        return {
+            'mk': mk_options,
+            'ac': ac_options,
+            'vars_before': before_source,
+            'vars_after': after_source,
+        }
+

python/mozbuild/mozbuild/mozconfig_loader

+#!/bin/sh
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script provides an execution environment for mozconfig scripts.
+# This script is not meant to be called by users. Instead, some
+# higher-level driver invokes it and parses the machine-tailored output.
+
+set -e
+
+ac_add_options() {
+  local opt
+  for opt in $*; do
+    echo "------BEGIN_AC_OPTION"
+    echo $opt
+    echo "------END_AC_OPTION"
+  done
+}
+
+ac_add_app_options() {
+  echo "ac_add_app_options is not supported."
+  exit 1
+}
+
+mk_add_options() {
+  local opt
+  for opt in $*; do
+    echo "------BEGIN_MK_OPTION"
+    echo $opt
+    echo "------END_MK_OPTION"
+  done
+}
+
+echo "------BEGIN_BEFORE_SOURCE"
+set
+echo "------END_BEFORE_SOURCE"
+
+. $1
+
+echo "------BEGIN_AFTER_SOURCE"
+set
+echo "------END_AFTER_SOURCE"
+

python/mozbuild/mozbuild/test/test_mozconfig.py

@@ -7,26 +7,270 @@ from __future__ import unicode_literals
 import os
 import unittest
 
-from tempfile import NamedTemporaryFile
+from shutil import rmtree
 
-from mozbuild.mozconfig import MozconfigLoader
+from tempfile import (
+    gettempdir,
+    mkdtemp,
+    NamedTemporaryFile,
+)
 
-
-curdir = os.path.dirname(__file__)
-topsrcdir = os.path.normpath(os.path.join(curdir, '..', '..', '..', '..'))
+from mozbuild.mozconfig import (
+    MozconfigFindException,
+    MozconfigLoader,
+)
 
 
 class TestMozconfigLoader(unittest.TestCase):
+    def setUp(self):
+        self._old_env = dict(os.environ)
+        self._temp_dirs = set()
+
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self._old_env)
+
+        for d in self._temp_dirs:
+            rmtree(d)
+
     def get_loader(self):
-        return MozconfigLoader(topsrcdir)
+        return MozconfigLoader(self.get_temp_dir())
+
+    def get_temp_dir(self):
+        d = mkdtemp()
+        self._temp_dirs.add(d)
+
+        return d
+
+    def test_find_legacy_env(self):
+        """Ensure legacy mozconfig path definitions result in error."""
+
+        os.environ[b'MOZ_MYCONFIG'] = '/foo'
+
+        with self.assertRaises(MozconfigFindException) as e:
+            self.get_loader().find_mozconfig()
+
+        self.assertTrue(e.exception.message.startswith('The MOZ_MYCONFIG'))
+
+    def test_find_path_not_exist(self):
+        """Ensure missing files are detected."""
+        os.environ[b'MOZCONFIG'] = '/foo/bar/does/not/exist'
+
+        with self.assertRaises(MozconfigFindException) as e:
+            self.get_loader().find_mozconfig()
+
+        self.assertIn('path that does not exist', e.exception.message)
+        self.assertTrue(e.exception.message.endswith('/foo/bar/does/not/exist'))
+
+    def test_find_path_not_file(self):
+        """Ensure non-file paths are detected."""
+
+        os.environ[b'MOZCONFIG'] = gettempdir()
+
+        with self.assertRaises(MozconfigFindException) as e:
+            self.get_loader().find_mozconfig()
+
+        self.assertIn('refers to a non-file', e.exception.message)
+        self.assertTrue(e.exception.message.endswith(gettempdir()))
+
+    def test_find_default_files(self):
+        """Ensure default paths are used when present."""
+        for p in MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS:
+            d = self.get_temp_dir()
+            path = os.path.join(d, p)
+
+            with open(path, 'w'):
+                pass
+
+            self.assertEqual(MozconfigLoader(d).find_mozconfig(), path)
+
+    def test_find_multiple_defaults(self):
+        """Ensure we error when multiple default files are present."""
+        self.assertGreater(len(MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS), 1)
+
+        d = self.get_temp_dir()
+        for p in MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS:
+            with open(os.path.join(d, p), 'w'):
+                pass
+
+        with self.assertRaises(MozconfigFindException) as e:
+            MozconfigLoader(d).find_mozconfig()
+
+        self.assertIn('Multiple default mozconfig files present',
+            e.exception.message)
+
+    def test_find_deprecated_path_srcdir(self):
+        """Ensure we error when deprecated path locations are present."""
+        for p in MozconfigLoader.DEPRECATED_TOPSRCDIR_PATHS:
+            d = self.get_temp_dir()
+            with open(os.path.join(d, p), 'w'):
+                pass
+
+            with self.assertRaises(MozconfigFindException) as e:
+                MozconfigLoader(d).find_mozconfig()
+
+            self.assertIn('This implicit location is no longer',
+                e.exception.message)
+            self.assertIn(d, e.exception.message)
+
+    def test_find_deprecated_home_paths(self):
+        """Ensure we error when deprecated home directory paths are present."""
+
+        for p in MozconfigLoader.DEPRECATED_HOME_PATHS:
+            home = self.get_temp_dir()
+            os.environ[b'HOME'] = home
+            path = os.path.join(home, p)
+
+            with open(path, 'w'):
+                pass
+
+            with self.assertRaises(MozconfigFindException) as e:
+                self.get_loader().find_mozconfig()
+
+            self.assertIn('This implicit location is no longer',
+                e.exception.message)
+            self.assertIn(path, e.exception.message)
+
+    def test_read_no_mozconfig(self):
+        # This is basically to ensure changes to defaults incur a test failure.
+        result = self.get_loader().read_mozconfig()
+
+        self.assertEqual(result, {
+            'path': None,
+            'topobjdir': None,
+            'configure_args': None,
+            'make_flags': None,
+            'make_extra': None,
+            'env': None,
+        })
+
+    def test_read_empty_mozconfig(self):
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+
+            self.assertEqual(result['path'], mozconfig.name)
+            self.assertIsNone(result['topobjdir'])
+            self.assertEqual(result['configure_args'], [])
+            self.assertEqual(result['make_flags'], [])
+            self.assertEqual(result['make_extra'], [])
 
-    def test_mozconfig_reading_moz_objdir(self):
-        with NamedTemporaryFile(mode='wt') as mozconfig:
+            for f in ('added', 'removed', 'modified'):
+                self.assertEqual(len(result['env'][f]), 0)
+
+            self.assertGreater(len(result['env']['unmodified']), 0)
+
+    def test_read_capture_ac_options(self):
+        """Ensures ac_add_options calls are captured."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('ac_add_options --enable-debug\n')
+            mozconfig.write('ac_add_options --disable-tests --enable-foo\n')
+            mozconfig.write('ac_add_options --foo="bar"\n')
+            mozconfig.flush()
+
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+            self.assertEqual(result['configure_args'], [
+                '--enable-debug', '--disable-tests', '--enable-foo',
+                '--foo=bar'])
+
+    def test_read_ac_options_substitution(self):
+        """Ensure ac_add_options values are substituted."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('ac_add_options --foo=@TOPSRCDIR@\n')
+            mozconfig.flush()
+
+            loader = self.get_loader()
+            result = loader.read_mozconfig(mozconfig.name)
+            self.assertEqual(result['configure_args'], [
+                '--foo=%s' % loader.topsrcdir])
+
+    def test_read_capture_mk_options(self):
+        """Ensures mk_add_options calls are captured."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('mk_add_options MOZ_OBJDIR=/foo/bar\n')
+            mozconfig.write('mk_add_options MOZ_MAKE_FLAGS=-j8\n')
+            mozconfig.write('mk_add_options FOO=BAR\n')
+            mozconfig.write('mk_add_options BIZ=1\n')
+            mozconfig.flush()
+
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+            self.assertEqual(result['topobjdir'], '/foo/bar')
+            self.assertEqual(result['make_flags'], '-j8')
+            self.assertEqual(result['make_extra'], ['FOO=BAR', 'BIZ=1'])
+
+    def test_read_moz_objdir_substitution(self):
+        """Ensure @TOPSRCDIR@ substitution is recognized in MOZ_OBJDIR."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
             mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir')
             mozconfig.flush()
 
             loader = self.get_loader()
             result = loader.read_mozconfig(mozconfig.name)
 
-            self.assertEqual(result['topobjdir'], '%s/some-objdir' % topsrcdir)
+            self.assertEqual(result['topobjdir'], '%s/some-objdir' %
+                loader.topsrcdir)
+
+    def test_read_new_variables(self):
+        """New variables declared in mozconfig file are detected."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('CC=/usr/local/bin/clang\n')
+            mozconfig.write('CXX=/usr/local/bin/clang++\n')
+            mozconfig.flush()
+
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+
+            self.assertEqual(result['env']['added'], {
+                'CC': '/usr/local/bin/clang',
+                'CXX': '/usr/local/bin/clang++'})
+
+    def test_read_exported_variables(self):
+        """Exported variables are caught as new variables."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('export MY_EXPORTED=woot\n')
+            mozconfig.flush()
+
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+
+            self.assertEqual(result['env']['added'], {
+                'MY_EXPORTED': 'woot'})
+
+    def test_read_modify_variables(self):
+        """Variables modified by mozconfig are detected."""
+        os.environ[b'CC'] = b'/usr/bin/gcc'
+
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('CC=/usr/local/bin/clang\n')
+            mozconfig.flush()
+
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+
+            self.assertEqual(result['env']['modified'], {
+                'CC': ('/usr/bin/gcc', '/usr/local/bin/clang')
+            })
+
+    def test_read_removed_variables(self):
+        """Variables unset by the mozconfig are detected."""
+        os.environ[b'CC'] = b'/usr/bin/clang'
+
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('unset CC\n')
+            mozconfig.flush()
+
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+
+            self.assertEqual(result['env']['removed'], {
+                'CC': '/usr/bin/clang'})
+
+    def test_read_multiline_variables(self):
+        """Ensure multi-line variables are captured properly."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('multi="foo\nbar"\n')
+            mozconfig.write('single=1\n')
+            mozconfig.flush()
+
+            result = self.get_loader().read_mozconfig(mozconfig.name)
+
+            self.assertEqual(result['env']['added'], {
+                'multi': 'foo\nbar',
+                'single': '1'
+            })