# HG changeset patch # User Chris Manchester # Date 1534180061 0 # Node ID adadcbb4f742a70bedb5a79c96a67ecb1c93d9a4 # Parent b4b313bda21470c5320c2e4e460bffbf82df1b2b Bug 1478798 - Handle the CLOBBER file optimistically in the Tup backend. r=mshal MozReview-Commit-ID: DDIqlDwjKil Differential Revision: https://phabricator.services.mozilla.com/D2799 diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py --- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -25,16 +25,17 @@ from textwrap import ( ) try: import psutil except Exception: psutil = None from mach.mixin.logging import LoggingMixin +import mozfile from mozsystemmonitor.resourcemonitor import SystemResourceMonitor from mozterm.widgets import Footer import mozpack.path as mozpath from .clobber import ( Clobberer, ) @@ -89,16 +90,25 @@ The easiest and fastest way to perform a $ mach build If you did not modify any test files, it is safe to ignore this message \ and proceed with running tests. To do this run: $ touch {clobber_file} '''.splitlines()]) +CLOBBER_REQUESTED_MESSAGE = ''' +=================== +The CLOBBER file was updated prior to this build. A clobber build may be +required to succeed, but we weren't expecting it to. + +Please consider filing a bug for this failure if you have reason to believe +this is a clobber bug and not due to local changes. +=================== +'''.strip() BuildOutputResult = namedtuple('BuildOutputResult', ('warning', 'state_changed', 'message')) class TierStatus(object): """Represents the state and progress of tier traversal. @@ -1015,32 +1025,16 @@ class BuildDriver(MozbuildObject): outputs = fh.read().splitlines() for output in outputs: if not os.path.isfile(mozpath.join(self.topobjdir, output)): return True dep_file = '%s.in' % backend_file return build_out_of_date(backend_file, dep_file) - def maybe_invoke_backend(active_backend): - # Attempt to bypass the make-oriented logic below. Note this - # will only succeed in case we're building with a non-make - # backend (Tup), and otherwise be harmless. - if active_backend: - if backend_out_of_date(mozpath.join(self.topobjdir, - 'backend.%sBackend' % - active_backend)): - print('Build configuration changed. Regenerating backend.') - args = [config.substs['PYTHON'], - mozpath.join(self.topobjdir, 'config.status')] - self.run_process(args, cwd=self.topobjdir, pass_thru=True) - backend_cls = get_backend_class(active_backend)(config) - return backend_cls.build(self, output, jobs, verbose, what) - return None - monitor.start_resource_recording() config = None try: config = self.config_environment except Exception: pass @@ -1050,31 +1044,48 @@ class BuildDriver(MozbuildObject): if config_rc != 0: return config_rc config = self.config_environment status = None active_backend = config.substs.get('BUILD_BACKENDS', [None])[0] if active_backend and 'Make' not in active_backend: + # Record whether a clobber was requested so we can print + # a special message later if the build fails. + clobber_requested = False # Write out any changes to the current mozconfig in case # they should invalidate configure. self._write_mozconfig_json() # Even if we have a config object, it may be out of date # if something that influences its result has changed. if build_out_of_date(mozpath.join(self.topobjdir, 'config.status'), mozpath.join(self.topobjdir, 'config_status_deps.in')): + clobber_requested = self._clobber_configure() config_rc = self.configure(buildstatus_messages=True, line_handler=output.on_line) if config_rc != 0: return config_rc + elif backend_out_of_date(mozpath.join(self.topobjdir, + 'backend.%sBackend' % + active_backend)): + print('Build configuration changed. Regenerating backend.') + args = [config.substs['PYTHON'], + mozpath.join(self.topobjdir, 'config.status')] + self.run_process(args, cwd=self.topobjdir, pass_thru=True) - status = maybe_invoke_backend(active_backend) + backend_cls = get_backend_class(active_backend)(config) + status = backend_cls.build(self, output, jobs, verbose, what) + + if status and clobber_requested: + for line in CLOBBER_REQUESTED_MESSAGE.splitlines(): + self.log(logging.WARNING, 'clobber', + {'msg': line.rstrip()}, '{msg}') if what and status is None: # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) if directory is not None: @@ -1331,16 +1342,60 @@ class BuildDriver(MozbuildObject): # If we don't actually have a list of tests to install we install # test and support files wholesale. self._run_make(target='install-test-files', pass_thru=True, print_directory=False) else: install_test_files(mozpath.normpath(self.topsrcdir), self.topobjdir, '_tests', test_objs) + def _clobber_configure(self): + # This is an optimistic treatment of the CLOBBER file for when we have + # some trust in the build system: an update to the CLOBBER file is + # interpreted to mean that configure will fail during an incremental + # build, which is handled by removing intermediate configure artifacts + # and subsections of the objdir related to python and testing before + # proceeding. + clobberer = Clobberer(self.topsrcdir, self.topobjdir) + clobber_output = io.BytesIO() + res = clobberer.maybe_do_clobber(os.getcwd(), False, + clobber_output) + required, performed, message = res + assert not performed + if not required: + return False + + def remove_objdir_path(path): + path = mozpath.join(self.topobjdir, path) + self.log(logging.WARNING, + 'clobber', + {'path': path}, + 'CLOBBER file has been updated, removing {path}.') + mozfile.remove(path) + + # Remove files we think could cause "configure" clobber bugs. + for f in ('old-configure.vars', 'config.cache', 'configure.pkl'): + remove_objdir_path(f) + remove_objdir_path(mozpath.join('js', 'src', f)) + + rm_dirs = [ + # Stale paths in our virtualenv may cause build-backend + # to fail. + '_virtualenvs', + # Some tests may accumulate state in the objdir that may + # become invalid after srcdir changes. + '_tests', + ] + + for d in rm_dirs: + remove_objdir_path(d) + + os.utime(mozpath.join(self.topobjdir, 'CLOBBER'), None) + return True + def _write_mozconfig_json(self): mozconfig_json = os.path.join(self.topobjdir, '.mozconfig.json') with FileAvoidWrite(mozconfig_json) as fh: json.dump({ 'topsrcdir': self.topsrcdir, 'topobjdir': self.topobjdir, 'mozconfig': self.mozconfig, }, fh, sort_keys=True, indent=2)