# HG changeset patch # User Chris Manchester # Date 1533928049 25200 # Node ID f3a2d3db15e629812f0860ebdbadc5d6050c0307 # Parent fa573f0f70cc7bb787f81db571a8153f77f20787 Bug 1474028 - Add a way to exclude libraries from the default build. r=ted MozReview-Commit-ID: MVfplx9lN2 diff --git a/build/gecko_templates.mozbuild b/build/gecko_templates.mozbuild --- a/build/gecko_templates.mozbuild +++ b/build/gecko_templates.mozbuild @@ -93,33 +93,33 @@ def GeckoCppUnitTests(names, **kwargs): CppUnitTests(names) kwargs.setdefault('mozglue', 'program') GeckoBinary(**kwargs) @template -def GeckoSharedLibrary(name, **kwargs): +def GeckoSharedLibrary(name, output_category=None, **kwargs): '''Template for shared libraries related to Gecko. `name` identifies the library base name. See the documentation for `GeckoBinary` for other possible arguments. ''' - SharedLibrary(name) + SharedLibrary(name, output_category) kwargs.setdefault('mozglue', 'library') GeckoBinary(**kwargs) @template -def GeckoFramework(name, **kwargs): +def GeckoFramework(name, output_category=None, **kwargs): '''Template for OSX frameworks related to Gecko. `name` identifies the library base name. See the documentation for `GeckoBinary` for other possible arguments. ''' - Framework(name) + Framework(name, output_category) kwargs.setdefault('mozglue', 'library') GeckoBinary(**kwargs) diff --git a/build/templates.mozbuild b/build/templates.mozbuild --- a/build/templates.mozbuild +++ b/build/templates.mozbuild @@ -53,45 +53,51 @@ def Library(name): '''Template for libraries.''' LIBRARY_NAME = name @template def AllowCompilerWarnings(): COMPILE_FLAGS['WARNINGS_AS_ERRORS'] = [] @template -def RustLibrary(name, features=None, target_dir=None): +def RustLibrary(name, features=None, target_dir=None, output_category=None): '''Template for Rust libraries.''' Library(name) IS_RUST_LIBRARY = True # Some Rust build scripts compile C/C++ sources, don't error on warnings for them. AllowCompilerWarnings() if features: RUST_LIBRARY_FEATURES = features if target_dir: RUST_LIBRARY_TARGET_DIR = target_dir + if output_category: + RUST_LIBRARY_OUTPUT_CATEGORY = output_category + @template -def SharedLibrary(name): +def SharedLibrary(name, output_category=None): '''Template for shared libraries.''' Library(name) FORCE_SHARED_LIB = True + if output_category: + SHARED_LIBRARY_OUTPUT_CATEGORY = output_category + Binary() @template -def Framework(name): +def Framework(name, output_category=None): '''Template for OSX Frameworks.''' - SharedLibrary(name) + SharedLibrary(name, output_category) IS_FRAMEWORK = True @template def HostProgram(name): '''Template for build tools executables.''' HOST_PROGRAM = name diff --git a/config/makefiles/target_binaries.mk b/config/makefiles/target_binaries.mk --- a/config/makefiles/target_binaries.mk +++ b/config/makefiles/target_binaries.mk @@ -13,17 +13,19 @@ PROGRAMS_DEST ?= $(FINAL_TARGET) PROGRAMS_TARGET := target INSTALL_TARGETS += PROGRAMS endif ifdef SHARED_LIBRARY SHARED_LIBRARY_FILES = $(SHARED_LIBRARY) SHARED_LIBRARY_DEST ?= $(FINAL_TARGET) +ifndef SHARED_LIBRARY_TARGET SHARED_LIBRARY_TARGET = target +endif INSTALL_TARGETS += SHARED_LIBRARY endif # SHARED_LIBRARY ifneq (,$(strip $(HOST_SIMPLE_PROGRAMS))) HOST_PROGRAMS_EXECUTABLES = $(HOST_SIMPLE_PROGRAMS) $(HOST_RUST_PROGRAMS) HOST_PROGRAMS_DEST ?= $(DIST)/host/bin HOST_PROGRAMS_TARGET = host INSTALL_TARGETS += HOST_PROGRAMS diff --git a/config/recurse.mk b/config/recurse.mk --- a/config/recurse.mk +++ b/config/recurse.mk @@ -36,17 +36,17 @@ include root.mk # Special rule that does install-manifests (cf. Makefile.in) + compile binaries:: +$(MAKE) recurse_compile # Carefully avoid $(eval) type of rule generation, which makes pymake slower # than necessary. # Get current tier and corresponding subtiers from the data in root.mk. -CURRENT_TIER := $(filter $(foreach tier,$(TIERS),recurse_$(tier) $(tier)-deps),$(MAKECMDGOALS)) +CURRENT_TIER := $(filter $(foreach tier,$(TIERS) $(non_default_tiers),recurse_$(tier) $(tier)-deps),$(MAKECMDGOALS)) ifneq (,$(filter-out 0 1,$(words $(CURRENT_TIER)))) $(error $(CURRENT_TIER) not supported on the same make command line) endif CURRENT_TIER := $(subst recurse_,,$(CURRENT_TIER:-deps=)) # The rules here are doing directory traversal, so we don't want further # recursion to happen when running make -C subdir $tier. But some make files # further call make -C something else, and sometimes expect recursion to diff --git a/config/rules.mk b/config/rules.mk --- a/config/rules.mk +++ b/config/rules.mk @@ -437,17 +437,17 @@ GLOBAL_DEPS += Makefile $(addprefix $(DE ############################################## ifdef COMPILE_ENVIRONMENT OBJ_TARGETS = $(OBJS) $(PROGOBJS) $(HOST_OBJS) $(HOST_PROGOBJS) compile:: host target host:: $(HOST_LIBRARY) $(HOST_PROGRAM) $(HOST_SIMPLE_PROGRAMS) $(HOST_RUST_PROGRAMS) $(HOST_RUST_LIBRARY_FILE) $(HOST_SHARED_LIBRARY) -target:: $(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(RUST_LIBRARY_FILE) $(RUST_PROGRAMS) +target:: $(filter-out $(MOZBUILD_NON_DEFAULT_TARGETS),$(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(RUST_LIBRARY_FILE) $(RUST_PROGRAMS)) ifndef LIBRARY ifdef OBJS target:: $(OBJS) endif endif syms:: diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -805,37 +805,73 @@ class RecursiveMakeBackend(CommonBackend # they all end up rooting to nodes from the above category. But the # way make works[1] is such that there can be benefits listing them # as direct dependencies of the top recursion target, to somehow # prioritize them. # 1. See bug 1262241 comment 5. compile_roots = [t for t, deps in self._compile_graph.iteritems() if not deps or t not in all_compile_deps] - rule = root_deps_mk.create_rule(['recurse_compile']) - rule.add_dependencies(compile_roots) - for target, deps in sorted(self._compile_graph.items()): - if deps: - rule = root_deps_mk.create_rule([target]) - rule.add_dependencies(deps) + def add_category_rules(category, roots, graph): + rule = root_deps_mk.create_rule(['recurse_%s' % category]) + rule.add_dependencies(roots) + for target, deps in sorted(graph.items()): + if deps: + rule = root_deps_mk.create_rule([target]) + rule.add_dependencies(deps) + + non_default_roots = defaultdict(list) + non_default_graphs = defaultdict(lambda: OrderedDefaultDict(set)) + + for root in compile_roots: + # If this is a non-default target, separate the root from the + # rest of the compile graph. + target_name = mozpath.basename(root) + + if target_name not in ('target', 'host'): + non_default_roots[target_name].append(root) + non_default_graphs[target_name][root] = self._compile_graph[root] + del self._compile_graph[root] + + for root in chain(*non_default_roots.values()): + compile_roots.remove(root) + dirname = mozpath.dirname(root) + # If a directory only contains non-default compile targets, we don't + # attempt to dump symbols there. + if (dirname in self._no_skip['syms'] and + '%s/target' % dirname not in self._compile_graph): + self._no_skip['syms'].remove(dirname) + + add_category_rules('compile', compile_roots, self._compile_graph) + for category, graph in non_default_graphs.iteritems(): + add_category_rules(category, non_default_roots[category], graph) root_mk = Makefile() # Fill root.mk with the convenience variables. for tier, filter in filters: all_dirs = self._traversal.traverse('', filter) root_mk.add_statement('%s_dirs := %s' % (tier, ' '.join(all_dirs))) # Need a list of compile targets because we can't use pattern rules: # https://savannah.gnu.org/bugs/index.php?42833 root_mk.add_statement('compile_targets := %s' % ' '.join(sorted( set(self._compile_graph.keys()) | all_compile_deps))) root_mk.add_statement('syms_targets := %s' % ' '.join(sorted( set('%s/syms' % d for d in self._no_skip['syms'])))) + root_mk.add_statement('non_default_tiers := %s' % ' '.join(sorted( + non_default_roots.keys()))) + + for category, graphs in non_default_graphs.iteritems(): + category_dirs = [mozpath.dirname(target) + for target in graphs.keys()] + root_mk.add_statement('%s_dirs := %s' % (category, + ' '.join(category_dirs))) + root_mk.add_statement('include root-deps.mk') with self._write_file( mozpath.join(self.environment.topobjdir, 'root.mk')) as root: root_mk.dump(root, removal_guard=False) with self._write_file( mozpath.join(self.environment.topobjdir, 'root-deps.mk')) as root_deps: @@ -1275,27 +1311,39 @@ class RecursiveMakeBackend(CommonBackend for flag in per_source_flag.flags: backend_file.write('%s_FLAGS += %s\n' % (mozpath.basename(per_source_flag.file_name), flag)) def _process_computed_flags(self, computed_flags, backend_file): for var, flags in computed_flags.get_flags(): backend_file.write('COMPUTED_%s += %s\n' % (var, ' '.join(make_quote(shell_quote(f)) for f in flags))) + def _process_non_default_target(self, libdef, target_name, backend_file): + backend_file.write("%s:: %s\n" % (libdef.output_category, target_name)) + backend_file.write('MOZBUILD_NON_DEFAULT_TARGETS += %s\n' % target_name) + def _process_shared_library(self, libdef, backend_file): backend_file.write_once('LIBRARY_NAME := %s\n' % libdef.basename) backend_file.write('FORCE_SHARED_LIB := 1\n') backend_file.write('IMPORT_LIBRARY := %s\n' % libdef.import_name) backend_file.write('SHARED_LIBRARY := %s\n' % libdef.lib_name) if libdef.soname: backend_file.write('DSO_SONAME := %s\n' % libdef.soname) if libdef.symbols_file: backend_file.write('SYMBOLS_FILE := %s\n' % libdef.symbols_file) if not libdef.cxx_link: backend_file.write('LIB_IS_C_ONLY := 1\n') + if libdef.output_category: + self._process_non_default_target(libdef, libdef.lib_name, + backend_file) + # Override the install rule target for this library. This is hacky, + # but can go away as soon as we start building libraries in their + # final location (bug 1459764). + backend_file.write('SHARED_LIBRARY_TARGET := %s\n' % + libdef.output_category) def _process_static_library(self, libdef, backend_file): backend_file.write_once('LIBRARY_NAME := %s\n' % libdef.basename) backend_file.write('FORCE_STATIC_LIB := 1\n') backend_file.write('REAL_LIBRARY := %s\n' % libdef.lib_name) if libdef.no_expand_lib: backend_file.write('NO_EXPAND_LIBS := 1\n') @@ -1306,26 +1354,31 @@ class RecursiveMakeBackend(CommonBackend # possible invocations of Cargo with this CARGO_TARGET_DIR. Otherwise, # Cargo's dependency calculations don't work as we expect and we wind # up recompiling lots of things. target_dir = mozpath.join(backend_file.objdir, libdef.target_dir) target_dir = mozpath.normpath(target_dir) backend_file.write('CARGO_TARGET_DIR := %s\n' % target_dir) if libdef.features: backend_file.write('%s := %s\n' % (libdef.FEATURES_VAR, ' '.join(libdef.features))) + if libdef.output_category: + self._process_non_default_target(libdef, libdef.import_name, backend_file) def _process_host_library(self, libdef, backend_file): backend_file.write('HOST_LIBRARY_NAME = %s\n' % libdef.basename) def _process_host_shared_library(self, libdef, backend_file): backend_file.write('HOST_SHARED_LIBRARY = %s\n' % libdef.lib_name) def _build_target_for_obj(self, obj): + target_name = obj.KIND + if hasattr(obj, 'output_category') and obj.output_category: + target_name = obj.output_category return '%s/%s' % (mozpath.relpath(obj.objdir, - self.environment.topobjdir), obj.KIND) + self.environment.topobjdir), target_name) def _process_linked_libraries(self, obj, backend_file): def pretty_relpath(lib, name): return os.path.normpath(mozpath.join(mozpath.relpath(lib.objdir, obj.objdir), name)) topobjdir = mozpath.normsep(obj.topobjdir) # This will create the node even if there aren't any linked libraries. diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -1558,16 +1558,28 @@ VARIABLES = { 'SHARED_LIBRARY_NAME': (unicode, unicode, """The name of the static library generated for a directory, if it needs to differ from the library code name. Implies FORCE_SHARED_LIB. """), + 'SHARED_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode, + """The output category for this context's shared library. If set this will + correspond to the build command that will build this shared library, and + the library will not be built as part of the default build. + """), + + 'RUST_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode, + """The output category for this context's rust library. If set this will + correspond to the build command that will build this rust library, and + the library will not be built as part of the default build. + """), + 'IS_FRAMEWORK': (bool, bool, """Whether the library to build should be built as a framework on OSX. This implies the name of the library won't be prefixed nor suffixed. Implies FORCE_SHARED_LIB. """), 'STATIC_LIBRARY_NAME': (unicode, unicode, diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -668,16 +668,17 @@ class RustLibrary(StaticLibrary): """Context derived container object for a static library""" __slots__ = ( 'cargo_file', 'crate_type', 'dependencies', 'deps_path', 'features', 'target_dir', + 'output_category', ) TARGET_SUBST_VAR = 'RUST_TARGET' FEATURES_VAR = 'RUST_LIBRARY_FEATURES' LIB_FILE_VAR = 'RUST_LIBRARY_FILE' def __init__(self, context, basename, cargo_file, crate_type, dependencies, features, target_dir, **args): StaticLibrary.__init__(self, context, basename, **args) @@ -689,16 +690,17 @@ class RustLibrary(StaticLibrary): # many other things in the build system depend on that. assert self.crate_type == 'staticlib' self.lib_name = '%s%s%s' % (context.config.rust_lib_prefix, basename.replace('-', '_'), context.config.rust_lib_suffix) self.dependencies = dependencies self.features = features self.target_dir = target_dir + self.output_category = context.get('RUST_LIBRARY_OUTPUT_CATEGORY') # Skip setting properties below which depend on cargo # when we don't have a compile environment. The required # config keys won't be available, but the instance variables # that we don't set should never be accessed by the actual # build in that case. if not context.config.substs.get('COMPILE_ENVIRONMENT'): return build_dir = mozpath.join(target_dir, @@ -708,16 +710,17 @@ class RustLibrary(StaticLibrary): class SharedLibrary(Library): """Context derived container object for a shared library""" __slots__ = ( 'soname', 'variant', 'symbols_file', + 'output_category', ) DICT_ATTRS = { 'basename', 'import_name', 'install_target', 'lib_name', 'relobjdir', @@ -728,16 +731,17 @@ class SharedLibrary(Library): MAX_VARIANT = 2 def __init__(self, context, basename, real_name=None, soname=None, variant=None, symbols_file=False): assert(variant in range(1, self.MAX_VARIANT) or variant is None) Library.__init__(self, context, basename, real_name) self.variant = variant self.lib_name = real_name or basename + self.output_category = context.get('SHARED_LIBRARY_OUTPUT_CATEGORY') assert self.lib_name if variant == self.FRAMEWORK: self.import_name = self.lib_name else: self.import_name = '%s%s%s' % ( context.config.import_prefix, self.lib_name, diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -596,29 +596,22 @@ class GTestCommands(MachCommandBase): try: config = self.config_environment except Exception: print("Please run |./mach build| before |./mach gtest|.") return 1 active_backend = config.substs.get('BUILD_BACKENDS', [None])[0] if 'Tup' in active_backend: - gtest_build_path = mozpath.join(self.topobjdir, '') + gtest_build_target = mozpath.join(self.topobjdir, '') else: - # This path happens build the necessary parts of the tree in the - # Make backend due to the odd nature of partial tree builds. - gtest_build_path = mozpath.relpath(mozpath.join(self.topobjdir, - 'toolkit', 'library', - 'gtest', 'rust'), - self.topsrcdir) - - os.environ[b'LINK_GTEST_DURING_COMPILE'] = b'1' + gtest_build_target = 'recurse_gtest' + res = self._mach_context.commands.dispatch('build', self._mach_context, - what=[gtest_build_path]) - del os.environ[b'LINK_GTEST_DURING_COMPILE'] + what=[gtest_build_target]) if res: print("Could not build xul-gtest") return res if self.substs.get('MOZ_WIDGET_TOOLKIT') == 'cocoa': self._run_make(directory='browser/app', target='repackage', ensure_exit_code=True) diff --git a/toolkit/library/gtest/Makefile.in b/toolkit/library/gtest/Makefile.in --- a/toolkit/library/gtest/Makefile.in +++ b/toolkit/library/gtest/Makefile.in @@ -1,28 +1,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -# Enforce that the clean/distclean rules removes everything that needs -# to be removed from this directory. -ifneq (,$(filter clean distclean,$(MAKECMDGOALS))) -LINK_GTEST_DURING_COMPILE = 1 -endif - -# Don't link the gtest xul during MOZ_PROFILE_GENERATE, it doesn't get -# used during profiling anyway. -ifdef MOZ_PROFILE_GENERATE -LINK_GTEST_DURING_COMPILE = -endif - -ifndef LINK_GTEST_DURING_COMPILE -# Force to not include backend.mk unless LINK_GTEST_DURING_COMPILE is set. -# Not including backend.mk makes traversing this directory do nothing. -STANDALONE_MAKEFILE = 1 - -else - include $(topsrcdir)/toolkit/library/libxul.mk include $(topsrcdir)/config/config.mk - -endif diff --git a/toolkit/library/gtest/moz.build b/toolkit/library/gtest/moz.build --- a/toolkit/library/gtest/moz.build +++ b/toolkit/library/gtest/moz.build @@ -27,13 +27,14 @@ if CONFIG['OS_ARCH'] == 'Linux' and CONF GENERATED_FILES['symverscript'].inputs = ['../symverscript.in'] GENERATED_FILES['symverscript'].flags = [ 'xul%s' % CONFIG['MOZILLA_SYMBOLVERSION'] ] SYMBOLS_FILE = '!symverscript' # This needs to come after static:xul to avoid things like libfallible coming # before StaticXULComponentStart. -Libxul('xul-gtest-real') +Libxul('xul-gtest-real', + output_category=None if CONFIG['LINK_GTEST_DURING_COMPILE'] else 'gtest') DIRS += [ 'static', ] diff --git a/toolkit/library/gtest/rust/Makefile.in b/toolkit/library/gtest/rust/Makefile.in deleted file mode 100644 --- a/toolkit/library/gtest/rust/Makefile.in +++ /dev/null @@ -1,29 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -# This file looks quite similar to toolkit/library/gtest/Makefile.in. -# We only want to build gkrust-gtest when we are building libxul-gtest. - -# Enforce that the clean/distclean rules removes everything that needs -# to be removed from this directory. -ifneq (,$(filter clean distclean,$(MAKECMDGOALS))) -LINK_GTEST_DURING_COMPILE = 1 -endif - -# Don't build gkrust-gtest during MOZ_PROFILE_GENERATE, it doesn't get -# used during profiling anyway. -ifdef MOZ_PROFILE_GENERATE -LINK_GTEST_DURING_COMPILE = -endif - -ifndef LINK_GTEST_DURING_COMPILE -# Force to not include backend.mk unless LINK_GTEST_DURING_COMPILE is set. -# Not including backend.mk makes traversing this directory do nothing. -STANDALONE_MAKEFILE = 1 - -else - -include $(topsrcdir)/config/config.mk - -endif diff --git a/toolkit/library/gtest/rust/moz.build b/toolkit/library/gtest/rust/moz.build --- a/toolkit/library/gtest/rust/moz.build +++ b/toolkit/library/gtest/rust/moz.build @@ -1,9 +1,10 @@ # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. include('../../rust/gkrust-features.mozbuild') -RustLibrary('gkrust-gtest', gkrust_features, '../..') +RustLibrary('gkrust-gtest', gkrust_features, '../..', + output_category=None if CONFIG['LINK_GTEST_DURING_COMPILE'] else 'gtest') diff --git a/toolkit/library/moz.build b/toolkit/library/moz.build --- a/toolkit/library/moz.build +++ b/toolkit/library/moz.build @@ -7,24 +7,24 @@ @template def Libxul_defines(): LIBRARY_DEFINES['MOZILLA_INTERNAL_API'] = True LIBRARY_DEFINES['IMPL_LIBXUL'] = True if not CONFIG['JS_SHARED_LIBRARY']: LIBRARY_DEFINES['STATIC_EXPORTABLE_JS_API'] = True @template -def Libxul(name): +def Libxul(name, output_category=None): if CONFIG['MOZ_WIDGET_TOOLKIT'] in ('cocoa'): # This is going to be a framework named "XUL", not an ordinary library named # "libxul.dylib" - GeckoFramework(name, linkage=None) + GeckoFramework(name, output_category=output_category, linkage=None) SHARED_LIBRARY_NAME = 'XUL' else: - GeckoSharedLibrary(name, linkage=None) + GeckoSharedLibrary(name, output_category=output_category, linkage=None) SHARED_LIBRARY_NAME = 'xul' DELAYLOAD_DLLS += [ 'comdlg32.dll', 'netapi32.dll', 'secur32.dll', 'wininet.dll', 'winspool.drv'