diff --git a/build/moz.configure/util.configure b/build/moz.configure/util.configure index 1dec409767c79c..7ad720e5f5963c 100644 --- a/build/moz.configure/util.configure +++ b/build/moz.configure/util.configure @@ -540,6 +540,46 @@ def project_flag(env=None, set_as_define=False, **kwargs): set_define(env, option_implementation) +# A template providing a shorthand for setting a variable. The created +# option will only be settable from a confvars.sh file. +# If required, the set_as_define argument will additionally cause the variable +# to be set using set_define. +# Similarly, set_as_config can be set to False if the variable should not be +# passed to set_config. +@template +def confvar( + env=None, + set_as_config=True, + set_as_define=False, + allow_implied=False, + **kwargs, +): + if not env: + configure_error("A project_flag must be passed a variable name to set.") + + if kwargs.get("nargs", 0) not in (0, 1): + configure_error("A project_flag must be passed nargs={0,1}.") + + origins = ("confvars",) + if allow_implied: + origins += ("implied",) + opt = option(env=env, possible_origins=origins, **kwargs) + + @depends(opt.option) + def option_implementation(value): + if value: + if len(value) == 1: + return value[0] + elif len(value): + return value + return bool(value) + + if set_as_config: + set_config(env, option_implementation) + if set_as_define: + set_define(env, option_implementation) + + @template @imports(_from="mozbuild.configure.constants", _import="RaiseErrorOnUse") def obsolete_config(name, *, replacement): diff --git a/moz.configure b/moz.configure index 1931ad941e30d5..1c81efed86c37d 100755 --- a/moz.configure +++ b/moz.configure @@ -440,6 +440,29 @@ pack_relative_relocs_flags = dependable(False) include(include_project_configure) +@depends(build_environment, build_project, "--help") +@checking("if project-specific confvars.sh exists") +# This gives access to the sandbox. Don't copy this blindly. +@imports("__sandbox__") +@imports(_from="mozbuild.configure", _import="confvars") +@imports("os.path") +def load_confvars(build_env, build_project, help): + confvars_path = os.path.join(build_env.topsrcdir, build_project, "confvars.sh") + if os.path.exists(confvars_path): + helper = __sandbox__._helper + # parse confvars + try: + keyvals = confvars.parse(confvars_path) + except confvars.ConfVarsSyntaxError as e: + die(str(e)) + for key, value in keyvals.items(): + # FIXME: remove test once we no longer load confvars from old-configure. + if key in __sandbox__._options: + # ~= imply_option, but with an accurate origin + helper.add(f"{key}={value}", origin="confvars", args=helper._args) + return confvars_path + + # Final flags validation and gathering # ------------------------------------------------- diff --git a/python/mozbuild/mozbuild/configure/confvars.py b/python/mozbuild/mozbuild/configure/confvars.py new file mode 100644 index 00000000000000..8401b91bdd3400 --- /dev/null +++ b/python/mozbuild/mozbuild/configure/confvars.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozbuild.shellutil + + +class ConfVarsSyntaxError(SyntaxError): + def __init__(self, msg, file, lineno, colnum, line): + super().__init__(msg, (file, lineno, colnum, line)) + + +def parse(path): + with open(path) as confvars: + keyvals = {} + for lineno, rawline in enumerate(confvars, start=1): + line = rawline.rstrip() + # Empty line / comment. + line_no_leading_blank = line.lstrip() + if not line_no_leading_blank or line_no_leading_blank.startswith("#"): + continue + + head, sym, tail = line.partition("=") + if sym != "=" or "#" in head: + raise ConfVarsSyntaxError( + "Expecting key=value format", path, lineno, 1, line + ) + key = head.strip() + + # Verify there's no unexpected spaces. + if key != head: + colno = 1 + line.index(key) + raise ConfVarsSyntaxError( + f"Expecting no spaces around '{key}'", path, lineno, colno, line + ) + if tail.lstrip() != tail: + colno = 1 + line.index(tail) + raise ConfVarsSyntaxError( + f"Expecting no spaces between '=' and '{tail.lstrip()}'", + path, + lineno, + colno, + line, + ) + + # Verify we don't have duplicate keys. + if key in keyvals: + raise ConfVarsSyntaxError( + f"Invalid redefinition for '{key}'", + path, + lineno, + 1 + line.index(key), + line, + ) + + # Parse value. + try: + values = mozbuild.shellutil.split(tail) + except mozbuild.shellutil.MetaCharacterException as e: + raise ConfVarsSyntaxError( + f"Unquoted, non-escaped special character '{e.char}'", + path, + lineno, + 1 + line.index(e.char), + line, + ) + except Exception as e: + raise ConfVarsSyntaxError( + e.args[0].replace(" in command", ""), + path, + lineno, + 1 + line.index("="), + line, + ) + value = values[0] if values else "" + + # Finally, commit the key<> value pair \o/. + keyvals[key] = value + return keyvals diff --git a/python/mozbuild/mozbuild/repackaging/msix.py b/python/mozbuild/mozbuild/repackaging/msix.py index 711d1e189c6d33..f0db2613b65475 100644 --- a/python/mozbuild/mozbuild/repackaging/msix.py +++ b/python/mozbuild/mozbuild/repackaging/msix.py @@ -33,6 +33,7 @@ from mozpack.packager.unpack import UnpackFinder from six.moves import shlex_quote +from mozbuild.configure import confvars from mozbuild.dirutils import ensureParentDir from mozbuild.repackaging.application_ini import get_application_ini_values @@ -208,29 +209,23 @@ def get_appconstants_sys_mjs_values(finder, *args): def get_branding(use_official, topsrcdir, build_app, finder, log=None): """Figure out which branding directory to use.""" - conf_vars = mozpath.join(topsrcdir, build_app, "confvars.sh") + confvars_path = mozpath.join(topsrcdir, build_app, "confvars.sh") + confvars_content = confvars.parse(confvars_path) + for key, value in confvars_content.items(): + log( + logging.INFO, + "msix", + {"key": key, "conf_vars": confvars_path, "value": value}, + "Read '{key}' from {conf_vars}: {value}", + ) def conf_vars_value(key): - lines = [line.strip() for line in open(conf_vars).readlines()] - for line in lines: - if line and line[0] == "#": - continue - if key not in line: - continue - _, _, value = line.partition("=") - if not value: - continue - log( - logging.INFO, - "msix", - {"key": key, "conf_vars": conf_vars, "value": value}, - "Read '{key}' from {conf_vars}: {value}", - ) - return value + if key in confvars_content: + return confvars_content[key] log( logging.ERROR, "msix", - {"key": key, "conf_vars": conf_vars}, + {"key": key, "conf_vars": confvars_content}, "Unable to find '{key}' in {conf_vars}!", ) diff --git a/python/mozbuild/mozbuild/test/configure/data/confvars/confvars.sh b/python/mozbuild/mozbuild/test/configure/data/confvars/confvars.sh new file mode 100644 index 00000000000000..3d2b837ce3abb6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/confvars/confvars.sh @@ -0,0 +1,4 @@ +# line comment +CONFVAR=" a b c" +OTHER_CONFVAR=d # trailing comment + diff --git a/python/mozbuild/mozbuild/test/configure/data/confvars/moz.configure b/python/mozbuild/mozbuild/test/configure/data/confvars/moz.configure new file mode 100644 index 00000000000000..4e3dac8aadec5c --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/confvars/moz.configure @@ -0,0 +1,10 @@ +confvar( + "CONFVAR", + nargs=1, + help="Confvar", +) +confvar( + "OTHER_CONFVAR", + nargs=1, + help="Other confvar", +) diff --git a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py index c10adb3b476d6a..d8185f5b82800a 100644 --- a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py @@ -193,5 +193,20 @@ def check_nsis_version(version): self.assertEqual(check_nsis_version("v3.1"), "3.1") +class TestConfVars(BaseConfigureTest): + def test_loading(self): + sandbox = self.get_sandbox( + paths={}, + config={}, + args=[ + "--enable-project=python/mozbuild/mozbuild/test/configure/data/confvars" + ], + ) + self.assertEqual( + list(sandbox._helper), + ["CONFVAR= a b c", "OTHER_CONFVAR=d"], + ) + + if __name__ == "__main__": main() diff --git a/python/mozbuild/mozbuild/test/python.toml b/python/mozbuild/mozbuild/test/python.toml index e3471b47f09ccb..54ae72c130ecfb 100644 --- a/python/mozbuild/mozbuild/test/python.toml +++ b/python/mozbuild/mozbuild/test/python.toml @@ -79,6 +79,8 @@ subsuite = "mozbuild" ["test_base.py"] +["test_confvars.py"] + ["test_containers.py"] ["test_dotproperties.py"] diff --git a/python/mozbuild/mozbuild/test/test_confvars.py b/python/mozbuild/mozbuild/test/test_confvars.py new file mode 100644 index 00000000000000..401e44e66647fc --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_confvars.py @@ -0,0 +1,139 @@ +import os +import re +import unittest +from tempfile import NamedTemporaryFile + +import mozunit + +from mozbuild.configure.confvars import ConfVarsSyntaxError, parse + + +def TemporaryConfVars(): + return NamedTemporaryFile("wt", delete=False) + + +class TestContext(unittest.TestCase): + + def loads(self, *lines): + with NamedTemporaryFile("wt", delete=False) as ntf: + ntf.writelines(lines) + try: + confvars = parse(ntf.name) + finally: + os.remove(ntf.name) + return confvars + + def test_parse_empty_file(self): + confvars = self.loads("# comment\n") + self.assertEqual(confvars, {}) + + def test_parse_simple_assignment(self): + confvars = self.loads("a=b\n") + self.assertEqual(confvars, {"a": "b"}) + + def test_parse_simple_assignment_with_equal_in_value(self): + confvars = self.loads("a='='\n", "b==") + self.assertEqual(confvars, {"a": "=", "b": "="}) + + def test_parse_simple_assignment_with_sharp_in_value(self): + confvars = self.loads("a='#'\n") + self.assertEqual(confvars, {"a": "#"}) + + def test_parse_simple_assignment_with_trailing_spaces(self): + confvars = self.loads("a1=1\t\n", "\n", "a2=2\n", "a3=3 \n", "a4=4") + + self.assertEqual( + confvars, + { + "a1": "1", + "a2": "2", + "a3": "3", + "a4": "4", + }, + ) + + def test_parse_trailing_comment(self): + confvars = self.loads("a=b#comment\n") + self.assertEqual(confvars, {"a": "b"}) + + def test_parse_invalid_assign_in_trailing_comment(self): + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads("a#=comment\n") + self.assertTrue( + re.match("Expecting key=value format \\(.*, line 1\\)", str(cm.exception)) + ) + + def test_parse_quoted_assignment(self): + confvars = self.loads("a='b'\n" "b=' c'\n" 'c=" \'c"\n') + self.assertEqual(confvars, {"a": "b", "b": " c", "c": " 'c"}) + + def test_parse_invalid_assignment(self): + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads("a#comment\n") + self.assertTrue( + re.match("Expecting key=value format \\(.*, line 1\\)", str(cm.exception)) + ) + + def test_parse_empty_value(self): + confvars = self.loads("a=\n") + self.assertEqual(confvars, {"a": ""}) + + def test_parse_invalid_value(self): + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads("#comment\na='er\n") + self.assertTrue( + re.match( + "Unterminated quoted string \\(.*, line 2\\)", + str(cm.exception), + ) + ) + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads("a= er\n") + self.assertTrue( + re.match( + "Expecting no spaces between '=' and 'er' \\(.*, line 1\\)", + str(cm.exception), + ) + ) + + def test_parse_invalid_char(self): + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads("a=$\n") + self.assertTrue( + re.match( + "Unquoted, non-escaped special character '\\$' \\(.*, line 1\\)", + str(cm.exception), + ) + ) + + def test_parse_invalid_key(self): + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads(" a=1\n") + self.assertTrue( + re.match( + "Expecting no spaces around 'a' \\(.*, line 1\\)", + str(cm.exception), + ) + ) + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads("a =1\n") + self.assertTrue( + re.match( + "Expecting no spaces around 'a' \\(.*, line 1\\)", + str(cm.exception), + ) + ) + + def test_parse_redundant_key(self): + with self.assertRaises(ConfVarsSyntaxError) as cm: + self.loads("a=1\na=2\n") + self.assertTrue( + re.match( + "Invalid redefinition for 'a' \\(.*, line 2\\)", + str(cm.exception), + ) + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/toolkit/moz.configure b/toolkit/moz.configure index 5eba7cacc21685..f29d7a0316c1c4 100644 --- a/toolkit/moz.configure +++ b/toolkit/moz.configure @@ -124,7 +124,7 @@ def all_configure_options(): # defaults. if ( value is not None - and value.origin not in ("default", "implied") + and value.origin not in ("default", "implied", "confvars") and value != option.default ): result.append(