# HG changeset patch # User Andrew Halberstadt # Date 1517512680 18000 # Node ID 37db56af5f70766af2122ae99aa78add4b76abb5 # Parent 3017a3a228963ff3247545244d57467571964583 Bug 1392391 - [reftest] Pre-parse the manifests in a separate Firefox instance, r=jmaher Instead of parsing the manifests and running the tests all in one go, this will spawn an extra Firefox instance at the beginning that does nothing but parse the manifest and dump them to a file. This will allow the python harness to load and manipulate the test objects, before sending them back to the JS harness as a list of tests to run. The main motivation for this change is to implement run-by-manifest, a mode where we restart the browser in between every test manifest. But there are other benefits as well, like sharing the chunking logic used by other harnesses and the ability for the python harness to stuff arbitrary metadata into the test objects. For now, Android will continue to parse the manifests and run the tests all in one go. Converting Android to this new mechanism will be left to a follow-up bug. MozReview-Commit-ID: AfUBmQpx3Zz diff --git a/layout/tools/reftest/globals.jsm b/layout/tools/reftest/globals.jsm --- a/layout/tools/reftest/globals.jsm +++ b/layout/tools/reftest/globals.jsm @@ -136,16 +136,17 @@ for (let [key, val] of Object.entries({ failedDisplayList: false, failedOpaqueLayer: false, failedOpaqueLayerMessages: [], failedAssignedLayer: false, failedAssignedLayerMessages: [], startAfter: undefined, suiteStarted: false, + manageSuite: false, // The enabled-state of the test-plugins, stored so they can be reset later testPluginEnabledStates: null, prefsToRestore: [], httpServerPort: -1, // whether to run slow tests or not runSlowTests: true, diff --git a/layout/tools/reftest/manifest.jsm b/layout/tools/reftest/manifest.jsm --- a/layout/tools/reftest/manifest.jsm +++ b/layout/tools/reftest/manifest.jsm @@ -1,16 +1,16 @@ /* -*- indent-tabs-mode: nil; js-indent-level: 4 -*- / /* vim: set shiftwidth=4 tabstop=8 autoindent cindent expandtab: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ "use strict"; -var EXPORTED_SYMBOLS = ["ReadTopManifest"]; +var EXPORTED_SYMBOLS = ["ReadTopManifest", "CreateUrls"]; ChromeUtils.import("chrome://reftest/content/globals.jsm", this); ChromeUtils.import("chrome://reftest/content/reftest.jsm", this); ChromeUtils.import("resource://gre/modules/Services.jsm"); ChromeUtils.import("resource://gre/modules/NetUtil.jsm"); const NS_SCRIPTSECURITYMANAGER_CONTRACTID = "@mozilla.org/scriptsecuritymanager;1"; const NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX = "@mozilla.org/network/protocol;1?name="; @@ -672,30 +672,30 @@ function CreateUrls(test) { return test; } function AddTestItem(aTest, aFilter) { if (!aFilter) aFilter = [null, [], false]; - aTest = CreateUrls(aTest); + var {url1, url2} = CreateUrls(Object.assign({}, aTest)); var globalFilter = aFilter[0]; var manifestFilter = aFilter[1]; var invertManifest = aFilter[2]; - if ((globalFilter && !globalFilter.test(aTest.url1.spec)) || + if ((globalFilter && !globalFilter.test(url1.spec)) || (manifestFilter && - !(invertManifest ^ manifestFilter.test(aTest.url1.spec)))) + !(invertManifest ^ manifestFilter.test(url1.spec)))) return; if (g.focusFilterMode == FOCUS_FILTER_NEEDS_FOCUS_TESTS && !aTest.needsFocus) return; if (g.focusFilterMode == FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS && aTest.needsFocus) return; - if (aTest.url2 !== null) - aTest.identifier = [aTest.url1.spec, aTest.type, aTest.url2.spec]; + if (url2 !== null) + aTest.identifier = [url1.spec, aTest.type, url2.spec]; else - aTest.identifier = aTest.url1.spec; + aTest.identifier = url1.spec; g.urls.push(aTest); } diff --git a/layout/tools/reftest/output.py b/layout/tools/reftest/output.py --- a/layout/tools/reftest/output.py +++ b/layout/tools/reftest/output.py @@ -1,14 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import json import threading +from collections import defaultdict from mozlog.formatters import TbplFormatter from mozrunner.utils import get_stack_fixer_function class ReftestFormatter(TbplFormatter): """ Formatter designed to preserve the legacy "tbpl" format in reftest. @@ -124,31 +125,36 @@ class OutputHandler(object): raw data logged from reftest.js to an appropriate structured log action, where applicable. """ def __init__(self, log, utilityPath, symbolsPath=None): self.stack_fixer_function = get_stack_fixer_function(utilityPath, symbolsPath) self.log = log self.proc_name = None + self.results = defaultdict(int) def __call__(self, line): # need to return processed messages to appease remoteautomation.py if not line.strip(): return [] line = line.decode('utf-8', errors='replace') try: data = json.loads(line) except ValueError: self.verbatim(line) return [line] if isinstance(data, dict) and 'action' in data: - self.log.log_raw(data) + if data['action'] == 'results': + for k, v in data['results'].items(): + self.results[k] += v + else: + self.log.log_raw(data) else: self.verbatim(json.dumps(data)) return [data] def verbatim(self, line): if self.stack_fixer_function: line = self.stack_fixer_function(line) diff --git a/layout/tools/reftest/reftest.jsm b/layout/tools/reftest/reftest.jsm --- a/layout/tools/reftest/reftest.jsm +++ b/layout/tools/reftest/reftest.jsm @@ -306,38 +306,79 @@ function ReadTests() { if (g.focusFilterMode != FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS) { g.browser.removeEventListener("focus", ReadTests, true); } g.urls = []; var prefs = Components.classes["@mozilla.org/preferences-service;1"]. getService(Components.interfaces.nsIPrefBranch); - // Parse reftest manifests - try { - var manifests = JSON.parse(prefs.getCharPref("reftest.manifests")); - g.urlsFilterRegex = manifests[null]; - } catch(e) { - logger.error("Unable to find reftest.manifests pref. Please ensure your profile is setup properly"); + /* There are three modes implemented here: + * 1) reftest.manifests + * 2) reftest.manifests and reftest.manifests.dumpTests + * 3) reftest.tests + * + * The first will parse the specified manifests, then immediately + * run the tests. The second will parse the manifests, save the test + * objects to a file and exit. The third will load a file of test + * objects and run them. + * + * The latter two modes are used to pass test data back and forth + * with python harness. + */ + let manifests = prefs.getCharPref("reftest.manifests", null); + let dumpTests = prefs.getCharPref("reftest.manifests.dumpTests", null); + let testList = prefs.getCharPref("reftest.tests", null); + + if ((testList && manifests) || !(testList || manifests)) { + logger.error("Exactly one of reftest.manifests or reftest.tests must be specified."); DoneTests(); } - var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null; - var manifestURLs = Object.keys(manifests); + if (testList) { + let promise = OS.File.read(testList).then(function onSuccess(array) { + let decoder = new TextDecoder(); + g.urls = JSON.parse(decoder.decode(array)).map(CreateUrls); + StartTests(); + }); + } else if (manifests) { + // Parse reftest manifests + manifests = JSON.parse(manifests); + g.urlsFilterRegex = manifests[null]; + + var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null; + var manifestURLs = Object.keys(manifests); - // Ensure we read manifests from higher up the directory tree first so that we - // process includes before reading the included manifest again - manifestURLs.sort(function(a,b) {return a.length - b.length}) - manifestURLs.forEach(function(manifestURL) { - logger.info("Reading manifest " + manifestURL); - var filter = manifests[manifestURL] ? new RegExp(manifests[manifestURL]) : null; - ReadTopManifest(manifestURL, [globalFilter, filter, false]); - }); + // Ensure we read manifests from higher up the directory tree first so that we + // process includes before reading the included manifest again + manifestURLs.sort(function(a,b) {return a.length - b.length}) + manifestURLs.forEach(function(manifestURL) { + logger.info("Reading manifest " + manifestURL); + var filter = manifests[manifestURL] ? new RegExp(manifests[manifestURL]) : null; + ReadTopManifest(manifestURL, [globalFilter, filter, false]); + }); - StartTests(); + if (dumpTests) { + let encoder = new TextEncoder(); + let tests = encoder.encode(JSON.stringify(g.urls)); + OS.File.writeAtomic(dumpTests, tests, {flush: true}).then( + function onSuccess() { + DoneTests(); + }, + function onFailure(reason) { + logger.error("failed to write test data: " + reason); + DoneTests(); + } + ) + } else { + g.manageSuite = true; + g.urls = g.urls.map(CreateUrls); + StartTests(); + } + } } catch(e) { ++g.testResults.Exception; logger.error("EXCEPTION: " + e); } } function StartTests() { @@ -409,17 +450,17 @@ function StartTests() end = g.thisChunk == g.totalChunks ? g.urls.length : g.urls.indexOf(tURLs[end + 1]) - 1; logger.info("Running chunk " + g.thisChunk + " out of " + g.totalChunks + " chunks. " + "tests " + (start+1) + "-" + end + "/" + g.urls.length); g.urls = g.urls.slice(start, end); } - if (g.startAfter === undefined && !g.suiteStarted) { + if (g.manageSuite && g.startAfter === undefined && !g.suiteStarted) { var ids = g.urls.map(function(obj) { return obj.identifier; }); var suite = prefs.getCharPref('reftest.suite', 'reftest'); logger.suiteStart(ids, suite, {"skipped": g.urls.length - numActiveTests}); g.suiteStarted = true } @@ -677,18 +718,22 @@ function StartCurrentURI(aURLTargetType) } else { SendLoadTest(type, g.currentURL, g.currentURLTargetType, g.loadTimeout); } } } function DoneTests() { - logger.suiteEnd({'results': g.testResults}); - g.suiteStarted = false + if (g.manageSuite) { + g.suiteStarted = false + logger.suiteEnd({'results': g.testResults}); + } else { + logger._logData('results', {results: g.testResults}); + } logger.info("Slowest test took " + g.slowestTestTime + "ms (" + g.slowestTestURL + ")"); logger.info("Total canvas count = " + g.recycledCanvases.length); if (g.failedUseWidgetLayers) { LogWidgetLayersFailure(); } function onStopped() { let appStartup = Cc["@mozilla.org/toolkit/app-startup;1"].getService(Ci.nsIAppStartup); diff --git a/layout/tools/reftest/remotereftest.py b/layout/tools/reftest/remotereftest.py --- a/layout/tools/reftest/remotereftest.py +++ b/layout/tools/reftest/remotereftest.py @@ -1,22 +1,22 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from contextlib import closing -import sys import logging import os import psutil import signal +import sys +import tempfile import time -import tempfile import traceback import urllib2 +from contextlib import closing import mozdevice import mozinfo from automation import Automation from remoteautomation import RemoteAutomation, fennecLogcatFilters from output import OutputHandler from runreftest import RefTest, ReftestResolver @@ -138,16 +138,17 @@ class ReftestServer: self.automation.log.info("Failed to shutdown server at %s" % self.shutdownURL) traceback.print_exc() self._process.kill() class RemoteReftest(RefTest): use_marionette = False + parse_manifest = False remoteApp = '' resolver_cls = RemoteReftestResolver def __init__(self, automation, devicemanager, options, scriptDir): RefTest.__init__(self) self.automation = automation self._devicemanager = devicemanager self.scriptDir = scriptDir @@ -162,21 +163,21 @@ class RemoteReftest(RefTest): self.SERVER_STARTUP_TIMEOUT = 180 else: self.SERVER_STARTUP_TIMEOUT = 90 self.automation.deleteANRs() self.automation.deleteTombstones() self._devicemanager.removeDir(self.remoteCache) self._populate_logger(options) - outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) + self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest # MessageLogger object to re-use this code path. - outputHandler.write = outputHandler.__call__ - self.automation._processArgs['messageLogger'] = outputHandler + self.outputHandler.write = self.outputHandler.__call__ + self.automation._processArgs['messageLogger'] = self.outputHandler def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path @@ -253,22 +254,22 @@ class RemoteReftest(RefTest): except Exception as e: self.log.info("Failed to kill process %d: %s" % (proc.pid, str(e))) else: self.log.info("NOT killing %s (not an orphan?)" % procd) except Exception: # may not be able to access process info for all processes continue - def createReftestProfile(self, options, manifest, startAfter=None): + def createReftestProfile(self, options, startAfter=None, **kwargs): profile = RefTest.createReftestProfile(self, options, - manifest, server=options.remoteWebServer, - port=options.httpPort) + port=options.httpPort, + **kwargs) if startAfter is not None: print ("WARNING: Continuing after a crash is not supported for remote " "reftest yet.") profileDir = profile.profile prefs = {} prefs["app.update.url.android"] = "" prefs["browser.firstrun.show.localepicker"] = False @@ -327,33 +328,46 @@ class RemoteReftest(RefTest): def buildBrowserEnv(self, options, profileDir): browserEnv = RefTest.buildBrowserEnv(self, options, profileDir) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] return browserEnv - def runApp(self, profile, binary, cmdargs, env, - timeout=None, debuggerInfo=None, - symbolsPath=None, options=None, - valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None): + def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None, symbolsPath=None, + valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, **profileArgs): + if cmdargs is None: + cmdargs = [] + + if self.use_marionette: + cmdargs.append('-marionette') + + binary = options.app + profile = self.createReftestProfile(options, **profileArgs) + + # browser environment + env = self.buildBrowserEnv(options, profile.profile) + + self.log.info("Running with e10s: {}".format(options.e10s)) status, lastTestSeen = self.automation.runApp(None, env, binary, profile.profile, cmdargs, utilityPath=options.utilityPath, xrePath=options.xrePath, debuggerInfo=debuggerInfo, symbolsPath=symbolsPath, timeout=timeout) if status == 1: # when max run time exceeded, avoid restart lastTestSeen = RefTest.TEST_SEEN_FINAL - return status, lastTestSeen + + self.cleanup(profile.profile) + return status, lastTestSeen, self.outputHandler.results def cleanup(self, profileDir): # Pull results back from device if self.remoteLogFile and \ self._devicemanager.fileExists(self.remoteLogFile): self._devicemanager.getFile(self.remoteLogFile, self.localLogName) else: print "WARNING: Unable to retrieve log file (%s) from remote " \ diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py --- a/layout/tools/reftest/runreftest.py +++ b/layout/tools/reftest/runreftest.py @@ -13,32 +13,35 @@ import json import multiprocessing import os import platform import re import shutil import signal import subprocess import sys +import tempfile import threading from datetime import datetime, timedelta SCRIPT_DIRECTORY = os.path.abspath( os.path.realpath(os.path.dirname(__file__))) if SCRIPT_DIRECTORY not in sys.path: sys.path.insert(0, SCRIPT_DIRECTORY) import mozcrash import mozdebug +import mozfile import mozinfo import mozleak import mozlog import mozprocess import mozprofile import mozrunner +from manifestparser import TestManifest from mozrunner.utils import get_stack_fixer_function, test_environment from mozscreenshot import printstatus, dump_screen try: from marionette_driver.addons import Addons from marionette_harness import Marionette except ImportError, e: # Defer ImportError until attempt to use Marionette @@ -221,26 +224,28 @@ class ReftestResolver(object): else: manifests[key] = "|".join(list(manifests[key])) return manifests class RefTest(object): TEST_SEEN_INITIAL = 'reftest' TEST_SEEN_FINAL = 'Main app process exited normally' + oldcwd = os.getcwd() + parse_manifest = True + resolver_cls = ReftestResolver use_marionette = True - oldcwd = os.getcwd() - resolver_cls = ReftestResolver def __init__(self): update_mozinfo() self.lastTestSeen = self.TEST_SEEN_INITIAL self.haveDumpedScreen = False self.resolver = self.resolver_cls() self.log = None + self.testDumpFile = os.path.join(tempfile.gettempdir(), 'reftests.json') def _populate_logger(self, options): if self.log: return self.log = getattr(options, 'log', None) if self.log: return @@ -254,35 +259,39 @@ class RefTest(object): options.log_tbpl_level = fmt_options['level'] = 'debug' self.log = mozlog.commandline.setup_logging( "reftest harness", options, {"tbpl": sys.stdout}, fmt_options) def getFullPath(self, path): "Get an absolute path relative to self.oldcwd." return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path))) - def createReftestProfile(self, options, manifests, server='localhost', port=0, - profile_to_clone=None, startAfter=None): + def createReftestProfile(self, options, tests=None, manifests=None, + server='localhost', port=0, profile_to_clone=None, + startAfter=None, prefs=None): """Sets up a profile for reftest. :param options: Object containing command line options - :param manifests: Dictionary of the form {manifest_path: [filters]} + :param tests: List of test objects to run + :param manifests: List of manifest files to parse (only takes effect + if tests were not passed in) :param server: Server name to use for http tests :param profile_to_clone: Path to a profile to use as the basis for the test profile + :param startAfter: Start running tests after the specified test id + :param prefs: Extra preferences to set in the profile """ - locations = mozprofile.permissions.ServerLocations() locations.add_host(server, scheme='http', port=port) locations.add_host(server, scheme='https', port=port) # Set preferences for communication between our command line arguments # and the reftest harness. Preferences that are required for reftest # to work should instead be set in reftest-preferences.js . - prefs = {} + prefs = prefs or {} prefs['reftest.timeout'] = options.timeout * 1000 if options.totalChunks: prefs['reftest.totalChunks'] = options.totalChunks if options.thisChunk: prefs['reftest.thisChunk'] = options.thisChunk if options.logFile: prefs['reftest.logFile'] = options.logFile if options.ignoreWindowSize: @@ -292,17 +301,16 @@ class RefTest(object): if options.repeat: prefs['reftest.repeat'] = options.repeat if options.runUntilFailure: prefs['reftest.runUntilFailure'] = True if options.cleanupCrashes: prefs['reftest.cleanupPendingCrashes'] = True prefs['reftest.focusFilterMode'] = options.focusFilterMode prefs['reftest.logLevel'] = options.log_tbpl_level or 'info' - prefs['reftest.manifests'] = json.dumps(manifests) prefs['reftest.suite'] = options.suite if startAfter not in (None, self.TEST_SEEN_INITIAL, self.TEST_SEEN_FINAL): self.log.info("Setting reftest.startAfter to %s" % startAfter) prefs['reftest.startAfter'] = startAfter # Unconditionally update the e10s pref. if options.e10s: @@ -372,16 +380,24 @@ class RefTest(object): 'preferences': prefs, 'locations': locations, 'whitelistpaths': sandbox_whitelist_paths} if profile_to_clone: profile = mozprofile.Profile.clone(profile_to_clone, **kwargs) else: profile = mozprofile.Profile(**kwargs) + if tests: + testlist = os.path.join(profile.profile, 'reftests.json') + with open(testlist, 'w') as fh: + json.dump(tests, fh) + profile.set_preferences({'reftest.tests': testlist}) + elif manifests: + profile.set_preferences({'reftest.manifests': json.dumps(manifests)}) + if os.path.join(here, 'chrome') not in options.extraProfileFiles: options.extraProfileFiles.append(os.path.join(here, 'chrome')) self.copyExtraFilesToProfile(options, profile) return profile def environment(self, **kwargs): kwargs['log'] = self.log @@ -651,20 +667,33 @@ class RefTest(object): process.kill(sig=signal.SIGABRT) except OSError: # https://bugzilla.mozilla.org/show_bug.cgi?id=921509 self.log.info("Can't trigger Breakpad, process no longer exists") return self.log.info("Can't trigger Breakpad, just killing process") process.kill() - def runApp(self, profile, binary, cmdargs, env, - timeout=None, debuggerInfo=None, - symbolsPath=None, options=None, - valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None): + def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None, + symbolsPath=None, valgrindPath=None, valgrindArgs=None, + valgrindSuppFiles=None, **profileArgs): + + if cmdargs is None: + cmdargs = [] + + if self.use_marionette: + cmdargs.append('-marionette') + + binary = options.app + profile = self.createReftestProfile(options, **profileArgs) + + # browser environment + env = self.buildBrowserEnv(options, profile.profile) + + self.log.info("Running with e10s: {}".format(options.e10s)) def timeoutHandler(): self.handleTimeout( timeout, proc, options.utilityPath, debuggerInfo) interactive = False debug_args = None if debuggerInfo: @@ -761,98 +790,119 @@ class RefTest(object): self.lastTestSeen = self.TEST_SEEN_FINAL crashed = mozcrash.log_crashes(self.log, os.path.join(profile.profile, 'minidumps'), symbolsPath, test=self.lastTestSeen) if not status and crashed: status = 1 runner.cleanup() + self.cleanup(profile.profile) if marionette_exception is not None: exc, value, tb = marionette_exception raise exc, value, tb - return status, self.lastTestSeen + self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s')) + return status, self.lastTestSeen, outputHandler.results + + def getActiveTests(self, manifests, options, testDumpFile=None): + # These prefs will cause reftest.jsm to parse the manifests, + # dump the resulting tests to a file, and exit. + prefs = { + 'reftest.manifests': json.dumps(manifests), + 'reftest.manifests.dumpTests': testDumpFile or self.testDumpFile, + } + cmdargs = [] # ['-headless'] + status, _, _ = self.runApp(options, cmdargs=cmdargs, prefs=prefs) + + with open(self.testDumpFile, 'r') as fh: + tests = json.load(fh) + + if os.path.isfile(self.testDumpFile): + mozfile.remove(self.testDumpFile) + + for test in tests: + # Name and path are expected by manifestparser, but not used in reftest. + test['name'] = test['path'] = test['url1'] + + mp = TestManifest(strict=False) + mp.tests = tests + + filters = [] + tests = mp.active_tests(exists=False, filters=filters) + return tests def runSerialTests(self, manifests, options, cmdargs=None): debuggerInfo = None if options.debugger: debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs, options.debuggerInteractive) - profileDir = None + tests = None + if self.parse_manifest: + tests = self.getActiveTests(manifests, options) + + ids = [t['identifier'] for t in tests] + self.log.suite_start(ids, name=options.suite) + startAfter = None # When the previous run crashed, we skip the tests we ran before prevStartAfter = None for i in itertools.count(): - try: - if cmdargs is None: - cmdargs = [] - - if self.use_marionette: - cmdargs.append('-marionette') - - profile = self.createReftestProfile(options, - manifests, - startAfter=startAfter) - profileDir = profile.profile # name makes more sense - - # browser environment - browserEnv = self.buildBrowserEnv(options, profileDir) + status, startAfter, results = self.runApp( + options, + tests=tests, + manifests=manifests, + cmdargs=cmdargs, + # We generally want the JS harness or marionette + # to handle timeouts if they can. + # The default JS harness timeout is currently + # 300 seconds (default options.timeout). + # The default Marionette socket timeout is + # currently 360 seconds. + # Give the JS harness extra time to deal with + # its own timeouts and try to usually exceed + # the 360 second marionette socket timeout. + # See bug 479518 and bug 1414063. + timeout=options.timeout + 70.0, + symbolsPath=options.symbolsPath, + debuggerInfo=debuggerInfo + ) + mozleak.process_leak_log(self.leakLogFile, + leak_thresholds=options.leakThresholds, + stack_fixer=get_stack_fixer_function(options.utilityPath, + options.symbolsPath)) - self.log.info("Running with e10s: {}".format(options.e10s)) - status, startAfter = self.runApp(profile, - binary=options.app, - cmdargs=cmdargs, - env=browserEnv, - # We generally want the JS harness or marionette - # to handle timeouts if they can. - # The default JS harness timeout is currently - # 300 seconds (default options.timeout). - # The default Marionette socket timeout is - # currently 360 seconds. - # Give the JS harness extra time to deal with - # its own timeouts and try to usually exceed - # the 360 second marionette socket timeout. - # See bug 479518 and bug 1414063. - timeout=options.timeout + 70.0, - symbolsPath=options.symbolsPath, - options=options, - debuggerInfo=debuggerInfo) - self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s')) - mozleak.process_leak_log(self.leakLogFile, - leak_thresholds=options.leakThresholds, - stack_fixer=get_stack_fixer_function(options.utilityPath, - options.symbolsPath)) - if status == 0: - break + if status == 0: + break + + if startAfter == self.TEST_SEEN_FINAL: + self.log.info("Finished running all tests, skipping resume " + "despite non-zero status code: %s" % status) + break - if startAfter == self.TEST_SEEN_FINAL: - self.log.info("Finished running all tests, skipping resume " - "despite non-zero status code: %s" % status) - break + if startAfter is not None and options.shuffle: + self.log.error("Can not resume from a crash with --shuffle " + "enabled. Please consider disabling --shuffle") + break + if startAfter is not None and options.maxRetries <= i: + self.log.error("Hit maximum number of allowed retries ({}) " + "in the test run".format(options.maxRetries)) + break + if startAfter == prevStartAfter: + # If the test stuck on the same test, or there the crashed + # test appeared more then once, stop + self.log.error("Force stop because we keep running into " + "test \"{}\"".format(startAfter)) + break + prevStartAfter = startAfter + # TODO: we need to emit an SUITE-END log if it crashed - if startAfter is not None and options.shuffle: - self.log.error("Can not resume from a crash with --shuffle " - "enabled. Please consider disabling --shuffle") - break - if startAfter is not None and options.maxRetries <= i: - self.log.error("Hit maximum number of allowed retries ({}) " - "in the test run".format(options.maxRetries)) - break - if startAfter == prevStartAfter: - # If the test stuck on the same test, or there the crashed - # test appeared more then once, stop - self.log.error("Force stop because we keep running into " - "test \"{}\"".format(startAfter)) - break - prevStartAfter = startAfter - # TODO: we need to emit an SUITE-END log if it crashed - finally: - self.cleanup(profileDir) + if self.parse_manifest: + self.log.suite_end(extra={'results': results}) return status def copyExtraFilesToProfile(self, options, profile): "Copy extra files or dirs specified on the command line to the testing profile." profileDir = profile.profile for f in options.extraProfileFiles: abspath = self.getFullPath(f) if os.path.isfile(abspath):