From 590c8fb62c1c8ff71f87704cf20acea7fc4ce1f9 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Tue, 8 May 2018 17:19:35 -0700 Subject: [PATCH 001/110] Refactor code involving resource import to not rely on os.name --- src/funfuzz/bot.py | 4 +++- src/funfuzz/js/js_interesting.py | 26 ++++++++++++++------------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 821218b82..52c2d687b 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -162,11 +162,13 @@ def printMachineInfo(): # pylint: disable=invalid-name for line in hgrc_contents: print(line.rstrip()) - if os.name == "posix": + try: # resource library is only applicable to Linux or Mac platforms. import resource # pylint: disable=import-error # pylint: disable=no-member print("Corefile size (soft limit, hard limit) is: %r" % (resource.getrlimit(resource.RLIMIT_CORE),)) + except ImportError: + print("Not checking corefile size as resource module is unavailable") def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 6599e315b..e98c8339a 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -83,9 +83,8 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa valgrindSuppressions() + runthis) - preexec_fn = ulimitSet if os.name == "posix" else None # logPrefix should be a string for timed_run in Lithium version 0.2.1 to work properly, apparently - runinfo = timed_run.timed_run(runthis, options.timeout, logPrefix.encode("utf-8"), preexec_fn=preexec_fn) + runinfo = timed_run.timed_run(runthis, options.timeout, logPrefix.encode("utf-8"), preexec_fn=set_ulimit) lev = JS_FINE issues = [] @@ -263,18 +262,21 @@ def deleteLogs(logPrefix): # pylint: disable=invalid-name,missing-param-doc,mis os.remove(logPrefix + "-core.gz") -def ulimitSet(): # pylint: disable=invalid-name - """When called as a preexec_fn, sets appropriate resource limits for the JS shell. Must only be called on POSIX.""" - # module only available on POSIX - import resource # pylint: disable=import-error +def set_ulimit(): + """Sets appropriate resource limits for the JS shell when on POSIX.""" + try: + import resource # pylint: disable=import-error - # Limit address space to 2GB (or 1GB on ARM boards such as ODROID). - GB = 2**30 # pylint: disable=invalid-name - resource.setrlimit(resource.RLIMIT_AS, (2 * GB, 2 * GB)) # pylint: disable=no-member + # log.debug("Limit address space to 2GB (or 1GB on ARM boards such as ODROID)") + giga_byte = 2**30 + resource.setrlimit(resource.RLIMIT_AS, (2 * giga_byte, 2 * giga_byte)) # pylint: disable=no-member - # Limit corefiles to 0.5 GB. - halfGB = int(GB // 2) # pylint: disable=invalid-name - resource.setrlimit(resource.RLIMIT_CORE, (halfGB, halfGB)) # pylint: disable=no-member + # log.debug("Limit corefiles to 0.5 GB") + half_giga_byte = int(giga_byte // 2) + resource.setrlimit(resource.RLIMIT_CORE, (half_giga_byte, half_giga_byte)) # pylint: disable=no-member + except ImportError: + # log.debug("Skipping resource import as a non-POSIX platform was detected: %s", platform.system()) + return def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc From d0c79895e131606ade28749f5a8c82a8352f47fb Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 19 Apr 2018 22:35:32 -0700 Subject: [PATCH 002/110] Replace various os/os.path usages with pathlib2/pathlib. --- src/funfuzz/autobisectjs/autobisectjs.py | 108 ++--- src/funfuzz/bot.py | 39 +- src/funfuzz/ccoverage/get_build.py | 10 +- src/funfuzz/js/__init__.py | 1 + src/funfuzz/js/build_options.py | 95 +++-- src/funfuzz/js/compare_jit.py | 34 +- src/funfuzz/js/compile_shell.py | 482 +++++++++++++---------- src/funfuzz/js/inspect_shell.py | 10 +- src/funfuzz/js/js_interesting.py | 16 +- src/funfuzz/js/link_fuzzer.py | 38 ++ src/funfuzz/js/loop.py | 68 ++-- src/funfuzz/util/__init__.py | 2 +- src/funfuzz/util/crashesat.py | 6 +- src/funfuzz/util/create_collector.py | 25 +- src/funfuzz/util/detect_malloc_errors.py | 2 +- src/funfuzz/util/file_manipulation.py | 2 +- src/funfuzz/util/fork_join.py | 31 +- src/funfuzz/util/hg_helpers.py | 133 ++++--- src/funfuzz/util/link_js.py | 39 -- src/funfuzz/util/lithium_helpers.py | 107 ++--- src/funfuzz/util/lock_dir.py | 12 +- src/funfuzz/util/os_ops.py | 373 ++++++++++++++++++ src/funfuzz/util/repos_update.py | 30 +- src/funfuzz/util/subprocesses.py | 344 ++-------------- tests/js/test_build_options.py | 53 +++ tests/js/test_compile_shell.py | 23 +- tests/js/test_link_fuzzer.py | 46 +++ tests/util/test_fork_join.py | 41 ++ tests/util/test_hg_helpers.py | 27 +- tests/util/test_os_ops.py | 50 +++ 30 files changed, 1363 insertions(+), 884 deletions(-) create mode 100644 src/funfuzz/js/link_fuzzer.py delete mode 100644 src/funfuzz/util/link_js.py create mode 100644 src/funfuzz/util/os_ops.py create mode 100644 tests/js/test_build_options.py create mode 100644 tests/js/test_link_fuzzer.py create mode 100644 tests/util/test_fork_join.py create mode 100644 tests/util/test_os_ops.py diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index 8936f6af1..32dd2b87a 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function # isort:skip from optparse import OptionParser # pylint: disable=deprecated-module -import os import re import shutil import subprocess @@ -30,6 +29,11 @@ from ..util import subprocesses as sps from ..util.lock_dir import LockDir +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc # pylint: disable=too-many-branches,too-complex,too-many-statements @@ -112,16 +116,17 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur print_("TBD: Bisection using downloaded shells is temporarily not supported.", flush=True) sys.exit(0) - options.build_options = build_options.parseShellOptions(options.build_options) + options.build_options = build_options.parse_shell_opts(options.build_options) options.skipRevs = " + ".join(kbew.known_broken_ranges(options.build_options)) - options.paramList = [sps.normExpUserPath(x) for x in options.parameters.split(" ") if x] + options.runtime_params = [x for x in options.parameters.split(" ") if x] + # First check that the testcase is present. - if "-e 42" not in options.parameters and not os.path.isfile(options.paramList[-1]): + if "-e 42" not in options.parameters and not Path(options.runtime_params[-1]).expanduser().is_file(): print_(flush=True) - print_("List of parameters to be passed to the shell is: %s" % " ".join(options.paramList), flush=True) + print_("List of parameters to be passed to the shell is: %s" % " ".join(options.runtime_params), flush=True) print_(flush=True) - raise Exception("Testcase at " + options.paramList[-1] + " is not present.") + raise OSError("Testcase at %s is not present." % options.runtime_params[-1]) assert options.compilationFailedLabel in ("bad", "good", "skip") @@ -141,24 +146,24 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur options.testAndLabel = internalTestAndLabel(options) earliestKnownQuery = kbew.earliest_known_working_rev( # pylint: disable=invalid-name - options.build_options, options.paramList + extraFlags, options.skipRevs) + options.build_options, options.runtime_params + extraFlags, options.skipRevs) earliestKnown = "" # pylint: disable=invalid-name if not options.useTreeherderBinaries: # pylint: disable=invalid-name - earliestKnown = hg_helpers.getRepoHashAndId(options.build_options.repoDir, repoRev=earliestKnownQuery)[0] + earliestKnown = hg_helpers.get_repo_hash_and_id(options.build_options.repo_dir, repo_rev=earliestKnownQuery)[0] if options.startRepo is None: if options.useTreeherderBinaries: options.startRepo = "default" else: options.startRepo = earliestKnown - # elif not (options.useTreeherderBinaries or hg_helpers.isAncestor(options.build_options.repoDir, + # elif not (options.useTreeherderBinaries or hg_helpers.isAncestor(options.build_options.repo_dir, # earliestKnown, options.startRepo)): # raise Exception("startRepo is not a descendant of kbew.earliestKnownWorkingRev for this configuration") # - # if not options.useTreeherderBinaries and not hg_helpers.isAncestor(options.build_options.repoDir, + # if not options.useTreeherderBinaries and not hg_helpers.isAncestor(options.build_options.repo_dir, # earliestKnown, options.endRepo): # raise Exception("endRepo is not a descendant of kbew.earliestKnownWorkingRev for this configuration") @@ -172,17 +177,18 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur return options -def findBlamedCset(options, repoDir, testRev): # pylint: disable=invalid-name,missing-docstring,too-complex +def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name,missing-docstring,too-complex # pylint: disable=too-many-locals,too-many-statements - print_("%s | Bisecting on: %s" % (time.asctime(), repoDir), flush=True) + repo_dir = str(repo_dir) + print_("%s | Bisecting on: %s" % (time.asctime(), repo_dir), flush=True) - hgPrefix = ["hg", "-R", repoDir] # pylint: disable=invalid-name + hgPrefix = ["hg", "-R", repo_dir] # pylint: disable=invalid-name # Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8". # pylint: disable=invalid-name - realStartRepo = sRepo = hg_helpers.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0] + realStartRepo = sRepo = hg_helpers.get_repo_hash_and_id(repo_dir, repo_rev=options.startRepo)[0] # pylint: disable=invalid-name - realEndRepo = eRepo = hg_helpers.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] + realEndRepo = eRepo = hg_helpers.get_repo_hash_and_id(repo_dir, repo_rev=options.endRepo)[0] sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo) # Refresh source directory (overwrite all local changes) to default tip if required. @@ -248,7 +254,7 @@ def findBlamedCset(options, repoDir, testRev): # pylint: disable=invalid-name,m print_("This iteration took %.3f seconds to run." % oneRunTime, flush=True) if blamedRev is not None: - checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, + checkBlameParents(repo_dir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, realEndRepo) sps.vdump("Resetting bisect") @@ -256,7 +262,7 @@ def findBlamedCset(options, repoDir, testRev): # pylint: disable=invalid-name,m sps.vdump("Resetting working directory") sps.captureStdout(hgPrefix + ["update", "-C", "-r", "default"], ignoreStderr=True) - hg_helpers.destroyPyc(repoDir) + hg_helpers.destroyPyc(repo_dir) print_(time.asctime(), flush=True) @@ -267,7 +273,7 @@ def internalTestAndLabel(options): # pylint: disable=invalid-name,missing-param def inner(shellFilename, _hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc,too-many-return-statements # pylint: disable=invalid-name - (stdoutStderr, exitCode) = inspect_shell.testBinary(shellFilename, options.paramList, + (stdoutStderr, exitCode) = inspect_shell.testBinary(shellFilename, options.runtime_params, options.build_options.runWithVg) if (stdoutStderr.find(options.output) != -1) and (options.output != ""): @@ -310,30 +316,32 @@ def externalTestAndLabel(options, interestingness): # pylint: disable=invalid-n def inner(shellFilename, hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList # pylint: disable=invalid-name - tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash) # pylint: disable=invalid-name - tempPrefix = os.path.join(tempDir, "t") # pylint: disable=invalid-name + # pylint: disable=invalid-name + conditionArgs = conditionArgPrefix + [str(shellFilename)] + options.runtime_params + temp_dir = Path(tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash)) + temp_prefix = temp_dir / "t" if hasattr(conditionScript, "init"): # Since we're changing the js shell name, call init() again! conditionScript.init(conditionArgs) - if conditionScript.interesting(conditionArgs, tempPrefix): + if conditionScript.interesting(conditionArgs, str(temp_prefix)): innerResult = ("bad", "interesting") # pylint: disable=invalid-name else: innerResult = ("good", "not interesting") # pylint: disable=invalid-name - if os.path.isdir(tempDir): - sps.rmTreeIncludingReadOnly(tempDir) + if temp_dir.is_dir(): + sps.rm_tree_incl_readonly(str(temp_dir)) return innerResult return inner # pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-many-arguments -def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo): +def checkBlameParents(repo_dir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo): """If bisect blamed a merge, try to figure out why.""" + repo_dir = str(repo_dir) bisectLied = False missedCommonAncestor = False - parents = sps.captureStdout(["hg", "-R", repoDir] + ["parent", "--template={node|short},", - "-r", blamedRev])[0].split(",")[:-1] + parents = sps.captureStdout(["hg", "-R", repo_dir] + ["parent", "--template={node|short},", + "-r", blamedRev])[0].split(",")[:-1] if len(parents) == 1: return @@ -343,8 +351,8 @@ def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, star if labels.get(p) is None: print_(flush=True) print_("Oops! We didn't test rev %s, a parent of the blamed revision! Let's do that now." % p, flush=True) - if not hg_helpers.isAncestor(repoDir, startRepo, p) and \ - not hg_helpers.isAncestor(repoDir, endRepo, p): + if not hg_helpers.isAncestor(repo_dir, startRepo, p) and \ + not hg_helpers.isAncestor(repo_dir, endRepo, p): print_("We did not test rev %s because it is not a descendant of either %s or %s." % ( p, startRepo, endRepo), flush=True) # Note this in case we later decide the bisect result is wrong. @@ -366,7 +374,7 @@ def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, star # Explain why bisect blamed the merge. if bisectLied: if missedCommonAncestor: - ca = hg_helpers.findCommonAncestor(repoDir, parents[0], parents[1]) + ca = hg_helpers.findCommonAncestor(repo_dir, parents[0], parents[1]) print_(flush=True) print_("Bisect blamed the merge because our initial range did not include one", flush=True) print_("of the parents.", flush=True) @@ -394,7 +402,7 @@ def sanitizeCsetMsg(msg, repo): # pylint: disable=missing-param-doc,missing-ret for line in msgList: if line.find("<") != -1 and line.find("@") != -1 and line.find(">") != -1: line = " ".join(line.split(" ")[:-1]) - elif line.startswith("changeset:") and "mozilla-central" in repo: + elif line.startswith("changeset:") and "mozilla-central" in str(repo): line = "changeset: https://hg.mozilla.org/mozilla-central/rev/" + line.split(":")[-1] sanitizedMsgList.append(line) return "\n".join(sanitizedMsgList) @@ -409,11 +417,11 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl outputLines = outputResult.split("\n") if options.build_options: - repoDir = options.build_options.repoDir + repo_dir = options.build_options.repo_dir if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]): print_(flush=True) - print_(sanitizeCsetMsg(outputResult, repoDir), flush=True) + print_(sanitizeCsetMsg(outputResult, repo_dir), flush=True) print_(flush=True) return None, None, None, startRepo, endRepo @@ -424,7 +432,7 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl print_(flush=True) print_("autobisectjs shows this is probably related to the following changeset:", flush=True) print_(flush=True) - print_(sanitizeCsetMsg(outputResult, repoDir), flush=True) + print_(sanitizeCsetMsg(outputResult, repo_dir), flush=True) print_(flush=True) blamedGoodOrBad = m.group(1) blamedRev = hg_helpers.get_cset_hash_from_bisect_msg(outputLines[1]) @@ -440,7 +448,7 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl if currRev is None: print_("Resetting to default revision...", flush=True) subprocess.check_call(hgPrefix + ["update", "-C", "default"]) - hg_helpers.destroyPyc(repoDir) + hg_helpers.destroyPyc(repo_dir) raise Exception("hg did not suggest a changeset to test!") # Update the startRepo/endRepo values. @@ -456,8 +464,15 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl return None, None, currRev, start, end -def rmOldLocalCachedDirs(cacheDir): # pylint: disable=missing-param-doc,missing-type-doc - """Remove old local cached directories, which were created four weeks ago.""" +def rm_old_local_cached_dirs(cache_dir): + """Remove old local cached directories, which were created four weeks ago. + + Args: + cache_dir (Path): Full path to the cache directory + """ + assert isinstance(cache_dir, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + cache_dir = cache_dir.expanduser() + # This is in autobisectjs because it has a lock so we do not race while removing directories # Adapted from http://stackoverflow.com/a/11337407 SECONDS_IN_A_DAY = 24 * 60 * 60 @@ -467,14 +482,13 @@ def rmOldLocalCachedDirs(cacheDir): # pylint: disable=missing-param-doc,missing else: NUMBER_OF_DAYS = 28 - cacheDir = sps.normExpUserPath(cacheDir) - names = [os.path.join(cacheDir, fname) for fname in os.listdir(cacheDir)] + names = [cache_dir / x for x in cache_dir.iterdir()] for name in names: - if os.path.isdir(name): - timediff = time.mktime(time.gmtime()) - os.stat(name).st_atime + if name.is_dir(): + timediff = time.mktime(time.gmtime()) - Path.stat(name).st_atime if timediff > SECONDS_IN_A_DAY * NUMBER_OF_DAYS: - shutil.rmtree(name) + shutil.rmtree(str(name)) def main(): @@ -482,16 +496,16 @@ def main(): options = parseOpts() if options.build_options: - repoDir = options.build_options.repoDir + repo_dir = options.build_options.repo_dir - with LockDir(compile_shell.getLockDirPath(options.nameOfTreeherderBranch, tboxIdentifier="Tbox") - if options.useTreeherderBinaries else compile_shell.getLockDirPath(repoDir)): + with LockDir(compile_shell.get_lock_dir_path(Path.home(), options.nameOfTreeherderBranch, tbox_id="Tbox") + if options.useTreeherderBinaries else compile_shell.get_lock_dir_path(Path.home(), repo_dir)): if options.useTreeherderBinaries: print_("TBD: We need to switch to the autobisect repository.", flush=True) sys.exit(0) else: # Bisect using local builds - findBlamedCset(options, repoDir, compile_shell.makeTestRev(options)) + findBlamedCset(options, repo_dir, compile_shell.makeTestRev(options)) # Last thing we do while we have a lock. # Note that this only clears old *local* cached directories, not remote ones. - rmOldLocalCachedDirs(compile_shell.ensureCacheDir()) + rm_old_local_cached_dirs(compile_shell.ensure_cache_dir(Path.home())) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 52c2d687b..4f4ef299a 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -13,7 +13,6 @@ from builtins import object # pylint: disable=redefined-builtin import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module -import os import platform import shutil import sys @@ -30,9 +29,11 @@ from .util.lock_dir import LockDir if sys.version_info.major == 2: + from pathlib2 import Path import psutil +else: + from pathlib import Path # pylint: disable=import-error -path0 = os.path.dirname(os.path.abspath(__file__)) # pylint: disable=invalid-name JS_SHELL_DEFAULT_TIMEOUT = 24 # see comments in loop for tradeoffs @@ -84,7 +85,7 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur if args: print("Warning: bot does not use positional arguments") - if not options.useTreeherderBuilds and not os.path.isdir(build_options.DEFAULT_TREES_LOCATION): + if not options.useTreeherderBuilds and not build_options.DEFAULT_TREES_LOCATION.is_dir(): # We don't have trees, so we must use treeherder builds. options.useTreeherderBuilds = True print() @@ -106,7 +107,7 @@ def main(): # pylint: disable=missing-docstring options = parseOpts() - collector = create_collector.createCollector("jsfunfuzz") + collector = create_collector.make_collector() try: collector.refresh() except RuntimeError as ex: @@ -118,10 +119,10 @@ def main(): # pylint: disable=missing-docstring print(options.tempDir) build_info = ensureBuild(options) - assert os.path.isdir(build_info.buildDir) + assert build_info.buildDir.is_dir() number_of_processes = multiprocessing.cpu_count() - if "-asan" in build_info.buildDir: + if "-asan" in str(build_info.buildDir): # This should really be based on the amount of RAM available, but I don't know how to compute that in Python. # I could guess 1 GB RAM per core, but that wanders into sketchyville. number_of_processes = max(number_of_processes // 2, 1) @@ -143,7 +144,7 @@ def printMachineInfo(): # pylint: disable=invalid-name except (KeyboardInterrupt, Exception) as ex: # pylint: disable=broad-except print("Error involving gdb is: %r" % (ex,)) - # FIXME: Should have if os.path.exists(path to git) or something # pylint: disable=fixme + # FIXME: Should have if which(git) or something # pylint: disable=fixme # print("git version: %s" % sps.captureStdout(["git", "--version"], combineStderr=True, # ignoreStderr=True, ignoreExitCode=True)[0]) print("Python version: %s" % sys.version.split()[0]) @@ -154,10 +155,10 @@ def printMachineInfo(): # pylint: disable=invalid-name rootdir_free_space = shutil.disk_usage("/").free / (1024 ** 3) # pylint: disable=no-member print("Free space (GB): %.2f" % rootdir_free_space) - hgrc_path = os.path.join(path0, ".hg", "hgrc") - if os.path.isfile(hgrc_path): + hgrc_path = Path("~/.hg/hgrc").expanduser() + if hgrc_path.is_file(): print("The hgrc of this repository is:") - with open(hgrc_path, "r") as f: + with open(str(hgrc_path), "r") as f: hgrc_contents = f.readlines() for line in hgrc_contents: print(line.rstrip()) @@ -180,16 +181,16 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss bRev = "" # pylint: disable=invalid-name manyTimedRunArgs = [] # pylint: disable=invalid-name elif not options.useTreeherderBuilds: - options.build_options = build_options.parseShellOptions(options.build_options) + options.build_options = build_options.parse_shell_opts(options.build_options) options.timeout = options.timeout or (300 if options.build_options.runWithVg else JS_SHELL_DEFAULT_TIMEOUT) - with LockDir(compile_shell.getLockDirPath(options.build_options.repoDir)): - bRev = hg_helpers.getRepoHashAndId(options.build_options.repoDir)[0] # pylint: disable=invalid-name + with LockDir(compile_shell.get_lock_dir_path(Path.home(), options.build_options.repo_dir)): + bRev = hg_helpers.get_repo_hash_and_id(options.build_options.repo_dir)[0] # pylint: disable=invalid-name cshell = compile_shell.CompiledShell(options.build_options, bRev) - updateLatestTxt = (options.build_options.repoDir == "mozilla-central") # pylint: disable=invalid-name + updateLatestTxt = (options.build_options.repo_dir == "mozilla-central") # pylint: disable=invalid-name compile_shell.obtainShell(cshell, updateLatestTxt=updateLatestTxt) - bDir = cshell.getShellCacheDir() # pylint: disable=invalid-name + bDir = cshell.get_shell_cache_dir() # pylint: disable=invalid-name # Strip out first 3 chars or else the dir name in fuzzing jobs becomes: # js-js-dbg-opt-64-dm-linux bType = build_options.computeShellType(options.build_options)[3:] # pylint: disable=invalid-name @@ -202,7 +203,7 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss "==============================================\n\n" % ( "funfuzz.js.compile_shell", options.build_options.build_options_str, - options.build_options.repoDir, + options.build_options.repo_dir, bRev, cshell.getRepoName(), time.asctime() @@ -219,7 +220,7 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss def loopFuzzingAndReduction(options, buildInfo, collector, i): # pylint: disable=invalid-name,missing-docstring - tempDir = tempfile.mkdtemp("loop" + str(i)) # pylint: disable=invalid-name + tempDir = Path(tempfile.mkdtemp("loop" + str(i))) # pylint: disable=invalid-name loop.many_timed_runs(options.targetTime, tempDir, buildInfo.mtrArgs, collector) @@ -227,7 +228,7 @@ def mtrArgsCreation(options, cshell): # pylint: disable=invalid-name,missing-pa # pylint: disable=missing-return-type-doc,missing-type-doc """Create many_timed_run arguments for compiled builds.""" manyTimedRunArgs = [] # pylint: disable=invalid-name - manyTimedRunArgs.append("--repo=" + sps.normExpUserPath(options.build_options.repoDir)) + manyTimedRunArgs.append("--repo=%s" % options.build_options.repo_dir) manyTimedRunArgs.append("--build=" + options.build_options.build_options_str) if options.build_options.runWithVg: manyTimedRunArgs.append("--valgrind") @@ -240,7 +241,7 @@ def mtrArgsCreation(options, cshell): # pylint: disable=invalid-name,missing-pa # Ordering of elements in manyTimedRunArgs is important. manyTimedRunArgs.append(str(options.timeout)) manyTimedRunArgs.append(cshell.getRepoName()) # known bugs' directory - manyTimedRunArgs.append(cshell.getShellCacheFullPath()) + manyTimedRunArgs.append(cshell.get_shell_cache_js_bin_path()) return manyTimedRunArgs diff --git a/src/funfuzz/ccoverage/get_build.py b/src/funfuzz/ccoverage/get_build.py index e7a2f96b1..40ec48f3a 100644 --- a/src/funfuzz/ccoverage/get_build.py +++ b/src/funfuzz/ccoverage/get_build.py @@ -11,12 +11,17 @@ import io import logging -import os +import sys import tarfile import zipfile import requests +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + RUN_COV_LOG = logging.getLogger("funfuzz") @@ -41,7 +46,8 @@ def get_coverage_build(dirpath, args): js_cov_bin_name = "js" js_cov_bin = extract_folder / "dist" / "bin" / js_cov_bin_name - os.chmod(str(js_cov_bin), os.stat(str(js_cov_bin)).st_mode | 0o111) # Ensure the js binary is executable + + Path.chmod(js_cov_bin, Path.stat(js_cov_bin).st_mode | 0o111) # Ensure the js binary is executable assert js_cov_bin.is_file() # Check that the binary is non-debug. diff --git a/src/funfuzz/js/__init__.py b/src/funfuzz/js/__init__.py index f4bf3a0fa..91b691b62 100644 --- a/src/funfuzz/js/__init__.py +++ b/src/funfuzz/js/__init__.py @@ -13,5 +13,6 @@ from . import compile_shell from . import inspect_shell from . import js_interesting +from . import link_fuzzer from . import loop from . import shell_flags diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 0c9ffb6d9..99adec5df 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -12,7 +12,6 @@ import argparse from builtins import object # pylint: disable=redefined-builtin import hashlib -import os import platform import random import sys @@ -20,9 +19,13 @@ from past.builtins import range # pylint: disable=redefined-builtin from ..util import hg_helpers -from ..util import subprocesses as sps -DEFAULT_TREES_LOCATION = sps.normExpUserPath(os.path.join("~", "trees")) +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + +DEFAULT_TREES_LOCATION = Path.home() / "trees" def chance(p): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -67,11 +70,13 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): # pylint action="store_true", default=False, help='Chooses sensible random build options. Defaults to "%(default)s".') - parser.add_argument("-R", "--repoDir", - dest="repoDir", + parser.add_argument("-R", "--repodir", + dest="repo_dir", + type=Path, help="Sets the source repository.") parser.add_argument("-P", "--patch", - dest="patchFile", + dest="patch_file", + type=Path, help="Define the path to a single JS patch. Ensure mq is installed.") # Basic spidermonkey options @@ -156,11 +161,17 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): # pylint return parser, randomizer -def parseShellOptions(inputArgs): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Return a "build_options" object, which is intended to be immutable.""" +def parse_shell_opts(args): + """Parses shell options into a build_options object. + + Args: + args (object): Arguments to be parsed + + Returns: + build_options: An immutable build_options object + """ parser, randomizer = addParserOptions() - build_options = parser.parse_args(inputArgs.split()) + build_options = parser.parse_args(args.split()) if platform.system() == "Darwin": build_options.buildWithClang = True # Clang seems to be the only supported compiler @@ -171,31 +182,29 @@ def parseShellOptions(inputArgs): # pylint: disable=invalid-name,missing-param- if build_options.enableRandom: build_options = generateRandomConfigurations(parser, randomizer) else: - build_options.build_options_str = inputArgs + build_options.build_options_str = args valid = areArgsValid(build_options) if not valid[0]: print("WARNING: This set of build options is not tested well because: %s" % valid[1]) # Ensures releng machines do not enter the if block and assumes mozilla-central always exists - if os.path.isdir(DEFAULT_TREES_LOCATION): + if DEFAULT_TREES_LOCATION.is_dir(): # Repositories do not get randomized if a repository is specified. - if build_options.repoDir is None: + if build_options.repo_dir is None: # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead - if build_options.enableRandom and not build_options.patchFile: - build_options.repoDir = getRandomValidRepo(DEFAULT_TREES_LOCATION) + if build_options.enableRandom and not build_options.patch_file: + build_options.repo_dir = get_random_valid_repo(DEFAULT_TREES_LOCATION) else: - build_options.repoDir = os.path.realpath(sps.normExpUserPath( - os.path.join(DEFAULT_TREES_LOCATION, "mozilla-central"))) + build_options.repo_dir = DEFAULT_TREES_LOCATION / "mozilla-central" - if not os.path.isdir(build_options.repoDir): - sys.exit("repoDir is not specified, and a default repository location cannot be confirmed. Exiting...") + if not build_options.repo_dir.is_dir(): + sys.exit("repo_dir is not specified, and a default repository location cannot be confirmed. Exiting...") - assert hg_helpers.isRepoValid(build_options.repoDir) + assert (build_options.repo_dir / ".hg" / "hgrc").is_file() - if build_options.patchFile: - hg_helpers.ensureMqEnabled() - build_options.patchFile = sps.normExpUserPath(build_options.patchFile) - assert os.path.isfile(build_options.patchFile) + if build_options.patch_file: + hg_helpers.ensure_mq_enabled() + assert build_options.patch_file.resolve().is_file() else: sys.exit("DEFAULT_TREES_LOCATION not found at: %s. Exiting..." % DEFAULT_TREES_LOCATION) @@ -230,10 +239,10 @@ def computeShellType(build_options): # pylint: disable=invalid-name,missing-par if build_options.enableSimulatorArm32 or build_options.enableSimulatorArm64: fileName.append("armSim") fileName.append("windows" if platform.system() == "Windows" else platform.system().lower()) - if build_options.patchFile: + if build_options.patch_file: # We take the name before the first dot, so Windows (hopefully) does not get confused. - fileName.append(os.path.basename(build_options.patchFile).split(".")[0]) - with open(os.path.abspath(build_options.patchFile), "r") as f: + fileName.append(build_options.patch_file.name) + with open(str(build_options.patch_file.resolve()), "r") as f: readResult = f.read() # pylint: disable=invalid-name # Append the patch hash, but this is not equivalent to Mercurial's hash of the patch. fileName.append(hashlib.sha512(readResult).hexdigest()[:12]) @@ -332,20 +341,28 @@ def generateRandomConfigurations(parser, randomizer): # pylint: disable=inconsi return build_options -def getRandomValidRepo(treeLocation): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - validRepos = [] # pylint: disable=invalid-name - for repo in ["mozilla-central", "mozilla-beta"]: - if os.path.isfile(sps.normExpUserPath(os.path.join( - treeLocation, repo, ".hg", "hgrc"))): - validRepos.append(repo) +def get_random_valid_repo(tree): + """Given a path to Mozilla Mercurial repositories, return a randomly chosen valid one. + + Args: + tree (str): Intended location of Mozilla Mercurial repositories + + Returns: + Path: Location of a valid Mozilla repository + """ + assert isinstance(tree, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + tree = tree.resolve() + + valid_repos = [] + for _ in ["mozilla-central", "mozilla-beta"]: + if (tree / _ / ".hg" / "hgrc").is_file(): + valid_repos.append(_) # After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen - if "mozilla-beta" in validRepos and chance(0.5): - validRepos.remove("mozilla-beta") + if "mozilla-beta" in valid_repos and chance(0.5): + valid_repos.remove("mozilla-beta") - return os.path.realpath(sps.normExpUserPath( - os.path.join(treeLocation, random.choice(validRepos)))) + return tree / random.choice(valid_repos) def main(): # pylint: disable=missing-docstring @@ -363,7 +380,7 @@ def main(): # pylint: disable=missing-docstring print() print("Running this file directly doesn't do anything, but here's our subparser help:") print() - parseShellOptions("--help") + parse_shell_opts("--help") if __name__ == "__main__": diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 1280e8411..3a43081a5 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function # isort:skip from optparse import OptionParser # pylint: disable=deprecated-module -import os import sys # These pylint errors exist because FuzzManager is not Python 3-compatible yet @@ -25,6 +24,11 @@ from ..util import lithium_helpers from ..util import subprocesses as sps +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + gOptions = "" # pylint: disable=invalid-name lengthLimit = 1000000 # pylint: disable=invalid-name @@ -88,6 +92,8 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi # we also use it directly for knownPath, timeout, and collector # Return: (lev, crashInfo) or (js_interesting.JS_FINE, None) + assert isinstance(infilename, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + combos = shell_flags.basic_flag_sets(jsEngine) if quickMode: @@ -97,7 +103,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi if flags: combos.insert(0, flags) - commands = [[jsEngine] + combo + [infilename] for combo in combos] + commands = [[jsEngine] + combo + [str(infilename)] for combo in combos] for i in range(0, len(commands)): # pylint: disable=consider-using-enumerate prefix = logPrefix + "-r" + str(i) @@ -115,17 +121,17 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi js_interesting.deleteLogs(prefix) elif r.lev > js_interesting.JS_OVERALL_MISMATCH: # would be more efficient to run lithium on one or the other, but meh - print("%s | %s" % (infilename, + print("%s | %s" % (str(infilename), js_interesting.summaryString(r.issues + ["compare_jit found a more serious bug"], r.lev, r.runinfo.elapsedtime))) - with open(logPrefix + "-summary.txt", "w") as f: + with open(str(logPrefix + "-summary.txt"), "w") as f: f.write("\n".join(r.issues + [" ".join(quote(x) for x in command), "compare_jit found a more serious bug"]) + "\n") print(" %s" % " ".join(quote(x) for x in command)) return (r.lev, r.crashInfo) elif r.lev != js_interesting.JS_FINE or r.return_code != 0: - print("%s | %s" % (infilename, js_interesting.summaryString( + print("%s | %s" % (str(infilename), js_interesting.summaryString( r.issues + ["compare_jit is not comparing output, because the shell exited strangely"], r.lev, r.runinfo.elapsedtime))) print(" %s" % " ".join(quote(x) for x in command)) @@ -136,7 +142,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi # If the shell or python hit a memory limit, we consider the rest of the computation # "tainted" for the purpose of correctness comparison. message = "compare_jit is not comparing output: OOM" - print("%s | %s" % (infilename, js_interesting.summaryString( + print("%s | %s" % (str(infilename), js_interesting.summaryString( r.issues + [message], r.lev, r.runinfo.elapsedtime))) js_interesting.deleteLogs(prefix) if not i: @@ -168,13 +174,13 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr "--timeout=" + str(options.timeout), options.knownPath, jsEngine, - os.path.basename(infilename)]) + str(infilename.name)]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = (" " + " ".join(quote(x) for x in commands[0]) + "\n " + " ".join(quote(x) for x in command) + "\n\n" + summary) - with open(logPrefix + "-summary.txt", "w") as f: + with open(str(logPrefix + "-summary.txt"), "w") as f: f.write(rerunCommand + "\n\n" + summary) - print("%s | %s" % (infilename, js_interesting.summaryString( + print("%s | %s" % (str(infilename), js_interesting.summaryString( issues, js_interesting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime))) if quickMode: print(rerunCommand) @@ -249,17 +255,17 @@ def parseOptions(args): # pylint: disable=invalid-name options, args = parser.parse_args(args) if len(args) != 3: raise Exception("Wrong number of positional arguments. Need 3 (knownPath, jsengine, infilename).") - options.knownPath = args[0] - options.jsengine = args[1] - options.infilename = args[2] + options.knownPath = Path(args[0]).resolve() + options.jsengine = Path(args[1]).resolve() + options.infilename = Path(args[2]).resolve() options.flags = options.flagsSpaceSep.split(" ") if options.flagsSpaceSep else [] - if not os.path.exists(options.jsengine): + if not options.jsengine.is_file(): raise Exception("js shell does not exist: " + options.jsengine) # For js_interesting: options.valgrind = False options.shellIsDeterministic = True # We shouldn't be in compare_jit with a non-deterministic build - options.collector = create_collector.createCollector("jsfunfuzz") + options.collector = create_collector.make_collector() return options diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index ea8546937..44cc4e3a3 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -7,11 +7,13 @@ """Compiles SpiderMonkey shells on different platforms using various specified configuration parameters. """ +# reset ; rg -g '!*subprocesses.py' -g '!*crashesat.py' -g '!*s3cache.py' -g '!*test_shell_flags.py' +# -g '!*test_compile_shell.py' -g '!*known_broken*.py' -t py "import os$" + from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin import copy -import ctypes import io import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module @@ -33,9 +35,12 @@ from ..util import subprocesses as sps from ..util.lock_dir import LockDir -if sys.version_info.major == 2 and os.name == "posix": - import subprocess32 as subprocess # pylint: disable=import-error +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path else: + from pathlib import Path # pylint: disable=import-error import subprocess S3_SHELL_CACHE_DIRNAME = "shell-cache" # Used by autobisectjs @@ -68,18 +73,18 @@ class CompiledShellError(Exception): class CompiledShell(object): # pylint: disable=missing-docstring,too-many-instance-attributes,too-many-public-methods - def __init__(self, buildOpts, hgHash): - self.shellNameWithoutExt = build_options.computeShellName(buildOpts, hgHash) # pylint: disable=invalid-name - self.hgHash = hgHash # pylint: disable=invalid-name - self.build_opts = buildOpts + def __init__(self, build_opts, hg_hash): + self.shell_name_without_ext = build_options.computeShellName(build_opts, hg_hash) + self.hg_hash = hg_hash # pylint: disable=invalid-name + self.build_opts = build_opts - self.jsObjdir = "" # pylint: disable=invalid-name + self.js_objdir = "" # pylint: disable=invalid-name self.cfg = "" self.destDir = "" # pylint: disable=invalid-name self.addedEnv = "" # pylint: disable=invalid-name self.fullEnv = "" # pylint: disable=invalid-name - self.jsCfgFile = "" # pylint: disable=invalid-name + self.js_cfg_file = "" # pylint: disable=invalid-name self.jsMajorVersion = "" # pylint: disable=invalid-name self.jsVersion = "" # pylint: disable=invalid-name @@ -118,17 +123,17 @@ def run(argv=None): # pylint: disable=missing-param-doc,missing-return-doc,miss help="Specify revision to build") options = parser.parse_args(argv)[0] - options.build_opts = build_options.parseShellOptions(options.build_opts) + options.build_opts = build_options.parse_shell_opts(options.build_opts) - with LockDir(getLockDirPath(options.build_opts.repoDir)): + with LockDir(get_lock_dir_path(Path.home(), options.build_opts.repo_dir)): if options.revision: shell = CompiledShell(options.build_opts, options.revision) else: - local_orig_hg_hash = hg_helpers.getRepoHashAndId(options.build_opts.repoDir)[0] + local_orig_hg_hash = hg_helpers.get_repo_hash_and_id(options.build_opts.repo_dir)[0] shell = CompiledShell(options.build_opts, local_orig_hg_hash) obtainShell(shell, updateToRev=options.revision) - print(shell.getShellCacheFullPath()) + print(shell.get_shell_cache_js_bin_path()) return 0 @@ -151,66 +156,112 @@ def setEnvFull(self, fullEnv): # pylint: disable=invalid-name,missing-docstring def getEnvFull(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc return self.fullEnv - def getHgHash(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.hgHash + def get_hg_hash(self): + """Retrieve the hash of the current changeset of the repository - def getJsCfgPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - self.jsCfgFile = sps.normExpUserPath(os.path.join(self.getRepoDirJsSrc(), "configure")) - assert os.path.isfile(self.jsCfgFile) - return self.jsCfgFile + Returns: + str: Changeset hash + """ + return self.hg_hash - def getJsObjdir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.jsObjdir + def get_js_cfg_path(self): + """Retrieve the configure file in a js/src directory. - def setJsObjdir(self, oDir): # pylint: disable=invalid-name,missing-docstring - self.jsObjdir = oDir + Returns: + Path: Full path to the configure file + """ + self.js_cfg_file = self.get_repo_dir() / "js" / "src" / "configure" + return self.js_cfg_file - def getRepoDir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.build_opts.repoDir + def get_js_objdir(self): + """Retrieve the objdir of the js shell to be compiled. - def getRepoDirJsSrc(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(self.getRepoDir(), "js", "src")) + Returns: + Path: Full path to the js shell objdir + """ + return self.js_objdir + + def set_js_objdir(self, objdir): + """Set the objdir of the js shell to be compiled. + + Args: + objdir (Path): Full path to the objdir of the js shell to be compiled + """ + self.js_objdir = objdir + + def get_repo_dir(self): + """Retrieve the directory of a Mercurial repository. + + Returns: + Path: Full path to the repository + """ + return self.build_opts.repo_dir def getRepoName(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return hg_helpers.getRepoNameFromHgrc(self.build_opts.repoDir) + return hg_helpers.hgrc_repo_name(self.build_opts.repo_dir) def getS3TarballWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - return self.getShellNameWithoutExt() + ".tar.bz2" + return self.get_shell_name_without_ext() + ".tar.bz2" - def getS3TarballWithExtFullPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getS3TarballWithExt())) + def get_s3_tar_with_ext_full_path(self): + """Retrieve the path to the tarball downloaded from S3. - def getShellCacheDir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getShellNameWithoutExt())) + Returns: + Path: Full path to the tarball in the local shell cache directory + """ + return ensure_cache_dir(Path.home()) / self.getS3TarballWithExt() - def getShellCacheFullPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(self.getShellCacheDir(), self.getShellNameWithExt())) + def get_shell_cache_dir(self): + """Retrieve the shell cache directory of the intended js binary. - def getShellCompiledPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath( - os.path.join(self.getJsObjdir(), "dist", "bin", "js" + (".exe" if platform.system() == "Windows" else ""))) + Returns: + Path: Full path to the shell cache directory of the intended js binary + """ + return ensure_cache_dir(Path.home()) / self.get_shell_name_without_ext() - def getShellCompiledRunLibsPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - libs_list = [ - sps.normExpUserPath(os.path.join(self.getJsObjdir(), "dist", "bin", runLib)) - for runLib in inspect_shell.ALL_RUN_LIBS + def get_shell_cache_js_bin_path(self): + """Retrieve the full path to the js binary located in the shell cache. + + Returns: + Path: Full path to the js binary in the shell cache + """ + return ensure_cache_dir(Path.home()) / self.get_shell_name_without_ext() / self.get_shell_name_with_ext() + + def get_shell_compiled_path(self): + """Retrieve the full path to the original location of js binary compiled in the shell cache. + + Returns: + Path: Full path to the original location of js binary compiled in the shell cache + """ + full_path = self.get_js_objdir() / "dist" / "bin" / "js" + return full_path.with_suffix(".exe") if platform.system() == "Windows" else full_path + + def get_shell_compiled_runlibs_path(self): + """Retrieve the full path to the original location of the libraries of js binary compiled in the shell cache. + + Returns: + Path: Full path to the original location of the libraries of js binary compiled in the shell cache + """ + return [ + self.get_js_objdir() / "dist" / "bin" / runlib for runlib in inspect_shell.ALL_RUN_LIBS ] - return libs_list - def getShellNameWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.shellNameWithoutExt + (".exe" if platform.system() == "Windows" else "") + def get_shell_name_with_ext(self): + """Retrieve the name of the compiled js shell with the file extension. - def getShellNameWithoutExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.shellNameWithoutExt + Returns: + str: Name of the compiled js shell with the file extension + """ + return self.shell_name_without_ext + (".exe" if platform.system() == "Windows" else "") + + def get_shell_name_without_ext(self): + """Retrieve the name of the compiled js shell without the file extension. + + Returns: + str: Name of the compiled js shell without the file extension + """ + return self.shell_name_without_ext # Version numbers def getMajorVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc @@ -227,61 +278,54 @@ def setVersion(self, jsVersion): # pylint: disable=invalid-name,missing-docstri self.jsVersion = jsVersion -def ensureCacheDir(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc - """Return a cache directory for compiled shells to live in, and create one if needed.""" - cache_dir = os.path.join(sps.normExpUserPath("~"), "shell-cache") - ensureDir(cache_dir) +def ensure_cache_dir(base_dir): + """Retrieve a cache directory for compiled shells to live in, and create one if needed. - # Expand long Windows paths (overcome legacy MS-DOS 8.3 stuff) - # This has to occur after the shell-cache directory is created - if platform.system() == "Windows": # adapted from http://stackoverflow.com/a/3931799 - if sys.version_info.major == 2: - utext = unicode # noqa pylint: disable=redefined-builtin,undefined-variable,unicode-builtin - else: - utext = str - win_temp_dir = utext(cache_dir) - get_long_path_name = ctypes.windll.kernel32.GetLongPathNameW - unicode_buf = ctypes.create_unicode_buffer(get_long_path_name(win_temp_dir, 0, 0)) - get_long_path_name(win_temp_dir, unicode_buf, len(unicode_buf)) - cache_dir = sps.normExpUserPath(str(unicode_buf.value)) # convert back to a str + Args: + base_dir (Path): Base directory to create the cache directory in + Returns: + Path: Returns the full shell-cache path + """ + if not base_dir: + base_dir = Path.home() + cache_dir = base_dir / "shell-cache" + cache_dir.mkdir(exist_ok=True) return cache_dir -def ensureDir(directory): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Create a directory, if it does not already exist.""" - if not os.path.exists(directory): - os.mkdir(directory) - assert os.path.isdir(directory) - +def autoconf_run(working_dir): + """Run autoconf binaries corresponding to the platform. -def autoconfRun(cwDir): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Run autoconf binaries corresponding to the platform.""" + Args: + working_dir (Path): Directory to be set as the current working directory + """ if platform.system() == "Darwin": autoconf213_mac_bin = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" if which("brew") else "autoconf213" # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 - if not os.path.isfile(sps.normExpUserPath(autoconf213_mac_bin)): + if not Path(autoconf213_mac_bin).is_file(): autoconf213_mac_bin = "autoconf213" - subprocess.check_call([autoconf213_mac_bin], cwd=cwDir) + subprocess.check_call([autoconf213_mac_bin], cwd=str(working_dir)) elif platform.system() == "Linux": if which("autoconf2.13"): - subprocess.run(["autoconf2.13"], check=True, cwd=cwDir) + subprocess.run(["autoconf2.13"], check=True, cwd=str(working_dir)) elif which("autoconf-2.13"): - subprocess.run(["autoconf-2.13"], check=True, cwd=cwDir) + subprocess.run(["autoconf-2.13"], check=True, cwd=str(working_dir)) elif which("autoconf213"): - subprocess.run(["autoconf213"], check=True, cwd=cwDir) + subprocess.run(["autoconf213"], check=True, cwd=str(working_dir)) elif platform.system() == "Windows": # Windows needs to call sh to be able to find autoconf. - subprocess.check_call(["sh", "autoconf-2.13"], cwd=cwDir) + subprocess.check_call(["sh", "autoconf-2.13"], cwd=str(working_dir)) def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc """Configures, compiles and copies a js shell according to required parameters.""" print("Compiling...") # Print *with* a trailing newline to avoid breaking other stuff - os.mkdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js"))) - shell.setJsObjdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js"))) + js_objdir_path = shell.get_shell_cache_dir() / "objdir-js" + js_objdir_path.mkdir() + shell.set_js_objdir(js_objdir_path) - autoconfRun(shell.getRepoDirJsSrc()) + autoconf_run(shell.get_repo_dir() / "js" / "src") configure_try_count = 0 while True: try: @@ -301,9 +345,8 @@ def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missi compileJs(shell) inspect_shell.verifyBinary(shell) - compile_log = sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), - shell.getShellNameWithoutExt() + ".fuzzmanagerconf")) - if not os.path.isfile(compile_log): + compile_log = shell.get_shell_cache_dir() / (shell.get_shell_name_without_ext() + ".fuzzmanagerconf") + if not compile_log.is_file(): envDump(shell, compile_log) @@ -314,38 +357,34 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_env = copy.deepcopy(os.environ) orig_cfg_env = copy.deepcopy(os.environ) cfg_env["AR"] = "ar" - if shell.build_opts.enable32 and os.name == "posix": + if shell.build_opts.enable32 and platform.system() == "Linux": # 32-bit shell on 32/64-bit x86 Linux - if platform.system() == "Linux": - cfg_env["PKG_CONFIG_LIBDIR"] = "/usr/lib/pkgconfig" - if shell.build_opts.buildWithClang: - cfg_env["CC"] = cfg_env["HOST_CC"] = str( - "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) - cfg_env["CXX"] = cfg_env["HOST_CXX"] = str( - "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) - else: - # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. - cfg_env["CC"] = "gcc -m32 %s" % SSE2_FLAGS - cfg_env["CXX"] = "g++ -m32 %s" % SSE2_FLAGS - if shell.build_opts.buildWithAsan: - cfg_env["CC"] += " " + CLANG_ASAN_PARAMS - cfg_env["CXX"] += " " + CLANG_ASAN_PARAMS - cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) - cfg_cmds.append("--target=i686-pc-linux") - if shell.build_opts.buildWithAsan: - cfg_cmds.append("--enable-address-sanitizer") - if shell.build_opts.enableSimulatorArm32: - # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 - # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated - # Newer configure.in changes mean that things blow up if unknown/removed configure - # options are entered, so specify it only if it's requested. - if shell.build_opts.enableArmSimulatorObsolete: - cfg_cmds.append("--enable-arm-simulator") - cfg_cmds.append("--enable-simulator=arm") + cfg_env["PKG_CONFIG_LIBDIR"] = "/usr/lib/pkgconfig" + if shell.build_opts.buildWithClang: + cfg_env["CC"] = cfg_env["HOST_CC"] = str( + "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) + cfg_env["CXX"] = cfg_env["HOST_CXX"] = str( + "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) else: - cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. + cfg_env["CC"] = "gcc -m32 %s" % SSE2_FLAGS + cfg_env["CXX"] = "g++ -m32 %s" % SSE2_FLAGS + if shell.build_opts.buildWithAsan: + cfg_env["CC"] += " " + CLANG_ASAN_PARAMS + cfg_env["CXX"] += " " + CLANG_ASAN_PARAMS + cfg_cmds.append("sh") + cfg_cmds.append(str(shell.get_js_cfg_path())) + cfg_cmds.append("--target=i686-pc-linux") + if shell.build_opts.buildWithAsan: + cfg_cmds.append("--enable-address-sanitizer") + if shell.build_opts.enableSimulatorArm32: + # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 + # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated + # Newer configure.in changes mean that things blow up if unknown/removed configure + # options are entered, so specify it only if it's requested. + if shell.build_opts.enableArmSimulatorObsolete: + cfg_cmds.append("--enable-arm-simulator") + cfg_cmds.append("--enable-simulator=arm") # 64-bit shell on Mac OS X 10.11 El Capitan and greater elif parse_version(platform.mac_ver()[0]) >= parse_version("10.11") and not shell.build_opts.enable32: cfg_env["CC"] = "clang " + CLANG_PARAMS @@ -356,7 +395,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ if which("brew"): cfg_env["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append(str(shell.get_js_cfg_path())) cfg_cmds.append("--target=x86_64-apple-darwin15.6.0") # El Capitan 10.11.6 cfg_cmds.append("--disable-xcode-checks") if shell.build_opts.buildWithAsan: @@ -380,7 +419,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_env["HOST_LDFLAGS"] = " " cfg_env["LIB"] += r"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows" cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append(str(shell.get_js_cfg_path())) if shell.build_opts.enable32: if shell.build_opts.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 @@ -406,7 +445,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_env["CC"] += " " + CLANG_ASAN_PARAMS cfg_env["CXX"] += " " + CLANG_ASAN_PARAMS cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append(str(shell.get_js_cfg_path())) if shell.build_opts.buildWithAsan: cfg_cmds.append("--enable-address-sanitizer") @@ -452,12 +491,11 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_cmds.append("--enable-debug-symbols") # gets debug symbols on opt shells cfg_cmds.append("--disable-tests") - if os.name == "nt": + if platform.system() == "Windows": # FIXME: Replace this with shellescape's quote # pylint: disable=fixme counter = 0 for entry in cfg_cmds: if os.sep in entry: - assert platform.system() == "Windows" # MozillaBuild on Windows sometimes confuses "/" and "\". cfg_cmds[counter] = cfg_cmds[counter].replace(os.sep, "//") counter = counter + 1 @@ -471,8 +509,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ sps.vdump("Command to be run is: " + " ".join(quote(x) for x in env_vars) + " " + " ".join(quote(x) for x in cfg_cmds)) - js_objdir = shell.getJsObjdir() - assert os.path.isdir(js_objdir) + assert shell.get_js_objdir().is_dir() if platform.system() == "Windows": changed_cfg_cmds = [] @@ -483,9 +520,9 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ if "\\" in entry: entry = entry.replace("\\", "/") changed_cfg_cmds.append(entry) - sps.captureStdout(changed_cfg_cmds, ignoreStderr=True, currWorkingDir=js_objdir, env=cfg_env) + sps.captureStdout(changed_cfg_cmds, ignoreStderr=True, currWorkingDir=str(shell.get_js_objdir()), env=cfg_env) else: - sps.captureStdout(cfg_cmds, ignoreStderr=True, currWorkingDir=js_objdir, env=cfg_env) + sps.captureStdout(cfg_cmds, ignoreStderr=True, currWorkingDir=str(shell.get_js_objdir()), env=cfg_env) shell.setEnvAdded(env_vars) shell.setEnvFull(cfg_env) @@ -495,9 +532,9 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc """Compile and copy a binary.""" try: - cmd_list = [MAKE_BINARY, "-C", shell.getJsObjdir(), "-j" + str(COMPILATION_JOBS), "-s"] + cmd_list = [MAKE_BINARY, "-C", str(shell.get_js_objdir()), "-j" + str(COMPILATION_JOBS), "-s"] out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, - currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] + currWorkingDir=str(shell.get_js_objdir()), env=shell.getEnvFull())[0] except Exception as ex: # pylint: disable=broad-except # This exception message is returned from sps.captureStdout via cmd_list. if (platform.system() == "Linux" or platform.system() == "Darwin") and \ @@ -505,28 +542,28 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- # FIXME: Absolute hack to retry after hitting OOM. # pylint: disable=fixme print("Trying once more due to the compiler running out of memory...") out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, - currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] + currWorkingDir=shell.get_js_objdir(), env=shell.getEnvFull())[0] # A non-zero error can be returned during make, but eventually a shell still gets compiled. - if os.path.exists(shell.getShellCompiledPath()): + if shell.get_shell_compiled_path().is_file(): print("A shell was compiled even though there was a non-zero exit code. Continuing...") else: print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) raise - if os.path.exists(shell.getShellCompiledPath()): - shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) - for run_lib in shell.getShellCompiledRunLibsPath(): - if os.path.isfile(run_lib): - shutil.copy2(run_lib, shell.getShellCacheDir()) + if shell.get_shell_compiled_path().is_file(): + shutil.copy2(str(shell.get_shell_compiled_path()), str(shell.get_shell_cache_js_bin_path())) + for run_lib in shell.get_shell_compiled_runlibs_path(): + if run_lib.is_file(): + shutil.copy2(str(run_lib), str(shell.get_shell_cache_dir())) - version = extractVersions(shell.getJsObjdir()) + version = extract_vers(shell.get_js_objdir()) shell.setMajorVersion(version.split(".")[0]) shell.setVersion(version) if platform.system() == "Linux": # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) # files in the objdir or else the stacks from failing testcases will lack symbols. - shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js"))) + shutil.rmtree(str(shell.get_shell_cache_dir() / "objdir-js")) else: print(out.decode("utf-8", errors="replace")) raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.") @@ -534,7 +571,7 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Create a .busted file with the exception message and backtrace included.""" - with open(filename, "w") as f: + with open(str(filename), "w") as f: f.write("Caught exception %r (%s)\n" % (e, e)) f.write("Backtrace:\n") f.write(traceback.format_exc() + "\n") @@ -555,12 +592,12 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi elif platform.system() == "Windows": fmconf_os = "windows" - with open(log, "a") as f: + with open(str(log), "a") as f: f.write("# Information about shell:\n# \n") f.write("# Create another shell in shell-cache like this one:\n") f.write('# python -u -m %s -b "%s" -r %s\n# \n' % ("funfuzz.js.compile_shell", - shell.build_opts.build_options_str, shell.getHgHash())) + shell.build_opts.build_options_str, shell.get_hg_hash())) f.write("# Full environment is:\n") f.write("# %s\n# \n" % str(shell.getEnvFull())) @@ -574,49 +611,61 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi f.write("[Main]\n") f.write("platform = %s\n" % fmconf_platform) f.write("product = %s\n" % shell.getRepoName()) - f.write("product_version = %s\n" % shell.getHgHash()) + f.write("product_version = %s\n" % shell.get_hg_hash()) f.write("os = %s\n" % fmconf_os) f.write("\n") f.write("[Metadata]\n") f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) f.write("majorVersion = %s\n" % shell.getMajorVersion()) - f.write("pathPrefix = %s%s\n" % (shell.getRepoDir(), - "/" if not shell.getRepoDir().endswith("/") else "")) + f.write("pathPrefix = %s/\n" % shell.get_repo_dir()) f.write("version = %s\n" % shell.getVersion()) -def extractVersions(objdir): # pylint: disable=inconsistent-return-statements,invalid-name,missing-param-doc - # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc - """Extract the version from js.pc and put it into *.fuzzmanagerconf.""" - jspc_dir = sps.normExpUserPath(os.path.join(objdir, "js", "src")) - jspc_name = os.path.join(jspc_dir, "js.pc") +def extract_vers(objdir): # pylint: disable=inconsistent-return-statements + """Extract the version from js.pc and put it into *.fuzzmanagerconf. + + Args: + objdir (Path): Full path to the objdir + + Raises: + OSError: Raises when js.pc is not found + + Returns: + str: Version number of the compiled js shell + """ + jspc_file_path = objdir / "js" / "src" / "js.pc" # Moved to /js/src/build/, see bug 1262241, Fx55 m-c rev 351194:2159959522f4 - jspc_new_dir = os.path.join(jspc_dir, "build") - jspc_new_name = os.path.join(jspc_new_dir, "js.pc") + jspc_new_file_path = objdir / "js" / "src" / "build" / "js.pc" + + if jspc_file_path.is_file(): + actual_path = jspc_file_path + elif jspc_new_file_path.is_file(): + actual_path = jspc_new_file_path + else: + raise OSError("js.pc file not found - needed to extract the version number") + + with io.open(str(actual_path), mode="r", encoding="utf-8", errors="replace") as f: + for line in f: + if line.startswith("Version: "): # Sample line: "Version: 47.0a2" + return line.split(": ")[1].rstrip() - def fixateVer(pcfile): # pylint: disable=inconsistent-return-statements,invalid-name,missing-param-doc - # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc - """Returns the current version number (47.0a2).""" - with io.open(pcfile, mode="r", encoding="utf-8", errors="replace") as f: - for line in f: - if line.startswith("Version: "): - # Sample line: "Version: 47.0a2" - return line.split(": ")[1].rstrip() - if os.path.isfile(jspc_name): - return fixateVer(jspc_name) - elif os.path.isfile(jspc_new_name): - return fixateVer(jspc_new_name) +def get_lock_dir_path(cache_dir_base, repo_dir, tbox_id=""): + """Return the name of the lock directory. + Args: + cache_dir_base (Path): Base directory where the cache directory is located + repo_dir (Path): Full path to the repository + tbox_id (str): Tinderbox entry id -def getLockDirPath(repoDir, tboxIdentifier=""): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Return the name of the lock directory, which is in the cache directory by default.""" - lockdir_name = ["shell", os.path.basename(repoDir), "lock"] - if tboxIdentifier: - lockdir_name.append(tboxIdentifier) - return os.path.join(ensureCacheDir(), "-".join(lockdir_name)) + Returns: + Path: Full path to the shell cache lock directory + """ + lockdir_name = "shell-%s-lock" % repo_dir.name + if tbox_id: + lockdir_name += "-%s" % tbox_id + return ensure_cache_dir(cache_dir_base) / lockdir_name def makeTestRev(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -630,81 +679,85 @@ def testRev(rev): # pylint: disable=invalid-name,missing-docstring,missing-retu return (options.compilationFailedLabel, "compilation failed") print("Testing...", end=" ") - return options.testAndLabel(shell.getShellCacheFullPath(), rev) + return options.testAndLabel(shell.get_shell_cache_js_bin_path(), rev) return testRev def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disable=invalid-name,missing-param-doc # pylint: disable=missing-raises-doc,missing-type-doc,too-many-branches,too-complex,too-many-statements """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" - assert os.path.isdir(getLockDirPath(shell.build_opts.repoDir)) - cached_no_shell = shell.getShellCacheFullPath() + ".busted" + assert get_lock_dir_path(Path.home(), shell.build_opts.repo_dir).is_dir() + cached_no_shell = shell.get_shell_cache_js_bin_path().with_suffix(".busted") - if os.path.isfile(shell.getShellCacheFullPath()): + if shell.get_shell_cache_js_bin_path().is_file(): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print("Found cached shell...") # Assuming that since the binary is present, everything else (e.g. symbols) is also present - verifyFullWinPageHeap(shell.getShellCacheFullPath()) + if platform.system() == "Windows": + verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) return - elif os.path.isfile(cached_no_shell): + elif cached_no_shell.is_file(): raise Exception("Found a cached shell that failed compilation...") - elif os.path.isdir(shell.getShellCacheDir()): + elif shell.get_shell_cache_dir().is_dir(): print("Found a cache dir without a successful/failed shell...") - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) + sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) - os.mkdir(shell.getShellCacheDir()) - hg_helpers.destroyPyc(shell.build_opts.repoDir) + shell.get_shell_cache_dir().mkdir() + hg_helpers.destroyPyc(shell.build_opts.repo_dir) s3cache_obj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) use_s3cache = s3cache_obj.connect() if use_s3cache: - if s3cache_obj.downloadFile(shell.getShellNameWithoutExt() + ".busted", - shell.getShellCacheFullPath() + ".busted"): - raise Exception("Found a .busted file for rev " + shell.getHgHash()) + if s3cache_obj.downloadFile(str(shell.get_shell_name_without_ext() + ".busted"), + str(shell.get_shell_cache_js_bin_path() + ".busted")): + raise Exception("Found a .busted file for rev " + shell.get_hg_hash()) - if s3cache_obj.downloadFile(shell.getShellNameWithoutExt() + ".tar.bz2", - shell.getS3TarballWithExtFullPath()): + if s3cache_obj.downloadFile(str(shell.get_shell_name_without_ext() + ".tar.bz2"), + str(shell.get_s3_tar_with_ext_full_path())): print("Extracting shell...") - with tarfile.open(shell.getS3TarballWithExtFullPath(), "r") as f: - f.extractall(shell.getShellCacheDir()) + with tarfile.open(str(shell.get_s3_tar_with_ext_full_path()), "r") as f: + f.extractall(str(shell.get_shell_cache_dir())) # Delete tarball after downloading from S3 - os.remove(shell.getS3TarballWithExtFullPath()) - verifyFullWinPageHeap(shell.getShellCacheFullPath()) + shell.get_s3_tar_with_ext_full_path().unlink() + if platform.system() == "Windows": + verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) return try: if updateToRev: - updateRepo(shell.build_opts.repoDir, updateToRev) - if shell.build_opts.patchFile: - hg_helpers.patchHgRepoUsingMq(shell.build_opts.patchFile, shell.getRepoDir()) + updateRepo(shell.build_opts.repo_dir, updateToRev) + if shell.build_opts.patch_file: + hg_helpers.patch_hg_repo_with_mq(shell.build_opts.patch_file, shell.get_repo_dir()) cfgJsCompile(shell) - verifyFullWinPageHeap(shell.getShellCacheFullPath()) + if platform.system() == "Windows": + verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) except KeyboardInterrupt: - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) + sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) raise except Exception as ex: # Remove the cache dir, but recreate it with only the .busted file. - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) - os.mkdir(shell.getShellCacheDir()) + sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) + shell.get_shell_cache_dir().mkdir() createBustedFile(cached_no_shell, ex) if use_s3cache: - s3cache_obj.uploadFileToS3(shell.getShellCacheFullPath() + ".busted") + s3cache_obj.uploadFileToS3(str(shell.get_shell_cache_js_bin_path() + ".busted")) raise finally: - if shell.build_opts.patchFile: - hg_helpers.hgQpopQrmAppliedPatch(shell.build_opts.patchFile, shell.getRepoDir()) + if shell.build_opts.patch_file: + hg_helpers.qpop_qrm_applied_patch(shell.build_opts.patch_file, shell.get_repo_dir()) if use_s3cache: - s3cache_obj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) + s3cache_obj.compressAndUploadDirTarball(str(shell.get_shell_cache_dir()), + str(shell.get_s3_tar_with_ext_full_path())) if updateLatestTxt: # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. - txt_info = "-".join(shell.getS3TarballWithExt().split("-")[:-1] + ["latest"]) + ".txt" - s3cache_obj.uploadStrToS3("", txt_info, shell.getS3TarballWithExt()) - os.remove(shell.getS3TarballWithExtFullPath()) + txt_info = "-".join(str(shell.getS3TarballWithExt()).split("-")[:-1] + ["latest"]) + ".txt" + s3cache_obj.uploadStrToS3("", txt_info, str(shell.getS3TarballWithExt())) + shell.get_s3_tar_with_ext_full_path().unlink() def updateRepo(repo, rev): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc @@ -715,15 +768,18 @@ def updateRepo(repo, rev): # pylint: disable=invalid-name,missing-param-doc,mis sps.captureStdout(["hg", "-R", repo, "update", "-C", "-r", rev], ignoreStderr=True) -def verifyFullWinPageHeap(shellPath): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Turn on full page heap verification on Windows.""" +def verify_full_win_pageheap(shell_path): + """Turn on full page heap verification on Windows. + + Args: + shell_path (Path): Path to the compiled js shell + """ # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ - if platform.system() == "Windows": - gflags_bin_path = os.path.join(os.getenv("PROGRAMW6432"), "Debugging Tools for Windows (x64)", "gflags.exe") - if os.path.isfile(gflags_bin_path) and os.path.isfile(shellPath): - print(subprocess.check_output([gflags_bin_path.decode("utf-8", errors="replace"), - "-p", "/enable", shellPath.decode("utf-8", errors="replace"), "/full"])) + gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" + if gflags_bin_path.is_file() and shell_path.is_file(): + print(subprocess.check_output([str(gflags_bin_path).decode("utf-8", errors="replace"), + "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"])) def main(): diff --git a/src/funfuzz/js/inspect_shell.py b/src/funfuzz/js/inspect_shell.py index 2e3299114..64d96eba0 100644 --- a/src/funfuzz/js/inspect_shell.py +++ b/src/funfuzz/js/inspect_shell.py @@ -9,7 +9,6 @@ from __future__ import absolute_import, print_function # isort:skip -import os import platform from lithium.interestingness.utils import env_with_path @@ -78,7 +77,7 @@ def archOfBinary(binary): # pylint: disable=inconsistent-return-statements,inva # pylint: disable=missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc """Test if a binary is 32-bit or 64-bit.""" # We can possibly use the python-magic-bin PyPI library in the future - unsplit_file_type = sps.captureStdout(["file", binary])[0] + unsplit_file_type = sps.captureStdout(["file", str(binary)])[0] filetype = unsplit_file_type.decode("utf-8", errors="replace").split(":", 1)[1] if platform.system() == "Windows": assert "MS Windows" in filetype @@ -133,11 +132,10 @@ def shellSupports(shellPath, args): # pylint: disable=invalid-name,missing-para def testBinary(shellPath, args, useValgrind): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Test the given shell with the given args.""" - test_cmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args + test_cmd = (constructVgCmdList() if useValgrind else []) + [str(shellPath)] + args sps.vdump("The testing command is: " + " ".join(quote(x) for x in test_cmd)) out, return_code = sps.captureStdout(test_cmd, combineStderr=True, ignoreStderr=True, - ignoreExitCode=True, env=env_with_path( - os.path.dirname(os.path.abspath(shellPath)))) + ignoreExitCode=True, env=env_with_path(str(shellPath.parent))) sps.vdump("The exit code is: " + str(return_code)) return out, return_code @@ -158,7 +156,7 @@ def queryBuildConfiguration(s, parameter): # pylint: disable=invalid-name,missi def verifyBinary(sh): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Verify that the binary is compiled as intended.""" - binary = sh.getShellCacheFullPath() + binary = sh.get_shell_cache_js_bin_path() assert archOfBinary(binary) == ("32" if sh.build_opts.enable32 else "64") diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index e98c8339a..af7134a4f 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -26,7 +26,7 @@ from . import inspect_shell from ..util import create_collector from ..util import detect_malloc_errors -from ..util import subprocesses as sps +from ..util import os_ops if sys.version_info.major == 2: if os.name == "posix": @@ -92,9 +92,9 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and "readlines" everything into memory. - with open(logPrefix + "-out.txt") as f: + with open(str(logPrefix + "-out.txt")) as f: out = f.readlines() - with open(logPrefix + "-err.txt") as f: + with open(str(logPrefix + "-err.txt")) as f: err = f.readlines() if options.valgrind and runinfo.return_code == VALGRIND_ERROR_EXIT_CODE: @@ -105,8 +105,8 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if valgrindErrorPrefix and line.startswith(valgrindErrorPrefix): issues.append(line.rstrip()) elif runinfo.sta == timed_run.CRASHED: - if sps.grabCrashLog(runthis[0], runinfo.pid, logPrefix, True): - with open(logPrefix + "-crash.txt") as f: + if os_ops.grab_crash_log(runthis[0], runinfo.pid, logPrefix, True): + with open(str(logPrefix + "-crash.txt")) as f: auxCrashData = [line.strip() for line in f.readlines()] elif detect_malloc_errors.amiss(logPrefix): issues.append("malloc error") @@ -170,7 +170,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa print("%s | %s" % (logPrefix, summaryString(issues, lev, runinfo.elapsedtime))) if lev != JS_FINE: - with open(logPrefix + "-summary.txt", "w") as f: + with open(str(logPrefix + "-summary.txt"), "w") as f: f.writelines(["Number: " + logPrefix + "\n", "Command: " + " ".join(quote(x) for x in runthis) + "\n"] + ["Status: " + i + "\n" for i in issues]) @@ -237,7 +237,7 @@ def summaryString(issues, level, elapsedtime): # pylint: disable=invalid-name,m def truncateFile(fn, maxSize): # pylint: disable=invalid-name,missing-docstring if os.path.exists(fn) and os.path.getsize(fn) > maxSize: - with open(fn, "r+") as f: + with open(str(fn), "r+") as f: f.truncate(maxSize) @@ -303,7 +303,7 @@ def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missin raise Exception("Not enough positional arguments") options.knownPath = args[0] options.jsengineWithArgs = args[1:] - options.collector = create_collector.createCollector("jsfunfuzz") + options.collector = create_collector.make_collector() if not os.path.exists(options.jsengineWithArgs[0]): raise Exception("js shell does not exist: " + options.jsengineWithArgs[0]) options.shellIsDeterministic = inspect_shell.queryBuildConfiguration( diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py new file mode 100644 index 000000000..e017f3666 --- /dev/null +++ b/src/funfuzz/js/link_fuzzer.py @@ -0,0 +1,38 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Concatenate js files to create jsfunfuzz. +""" + +from __future__ import absolute_import, print_function # isort:skip + +import sys + +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + + +def link_fuzzer(target_path, prologue=""): + """Links files together to create the full jsfunfuzz file. + + Args: + target_path (Path): Target file with full path, to be created + prologue (str): Contents to be prepended to the target file + """ + base_dir = Path(__file__).parent + + with open(str(target_path), "w") as f: # Create the full jsfunfuzz file + if prologue: + f.write(prologue) + + for entry in (base_dir / "files_to_link.txt").read_text().split(): + entry = entry.rstrip() + if entry and not entry.startswith("#"): + file_path = base_dir / Path(entry) + f.write("\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:]) + f.write(file_path.read_text()) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index b1b3e89ac..330c5bd93 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -18,13 +18,18 @@ from . import compare_jit from . import js_interesting +from . import link_fuzzer from . import shell_flags from ..util import create_collector from ..util import file_manipulation -from ..util import link_js from ..util import lithium_helpers from ..util import subprocesses as sps +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc parser = OptionParser() @@ -39,8 +44,8 @@ def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-r default=False, help="Pass a random set of flags (e.g. --ion-eager) to the js engine") parser.add_option("--repo", - action="store", dest="repo", - default=os.path.expanduser("~/trees/mozilla-central/"), + action="store", + dest="repo", help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection") parser.add_option("--build", action="store", dest="build_options_str", @@ -52,6 +57,12 @@ def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-r help="use valgrind with a reasonable set of options") options, args = parser.parse_args(args) + # optparse does not recognize pathlib - we will need to move to argparse + if options.repo: + options.repo = Path(options.repo) + else: + options.repo = Path.home() / "trees" / "mozilla-central" + if options.valgrind and options.use_compare_jit: print("Note: When running compare_jit, the --valgrind option will be ignored") @@ -63,7 +74,7 @@ def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-r # FIXME: We can probably remove args[1] # pylint: disable=fixme options.knownPath = "mozilla-central" - options.jsEngine = args[2] + options.jsEngine = Path(args[2]) options.engineFlags = args[3:] return options @@ -82,37 +93,29 @@ def showtail(filename): # pylint: disable=missing-docstring print() -def linkFuzzer(target_fn, prologue): # pylint: disable=invalid-name,missing-docstring - source_base = os.path.dirname(os.path.abspath(__file__)) - file_list_fn = sps.normExpUserPath(os.path.join(source_base, "files_to_link.txt")) - link_js.link_js(target_fn, file_list_fn, source_base, prologue) - - def makeRegressionTestPrologue(repo): # pylint: disable=invalid-name,missing-docstring,missing-param-doc # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests.""" - repo = sps.normExpUserPath(repo) + os.sep - return """ const regressionTestsRoot = %s; const libdir = regressionTestsRoot + %s; // needed by jit-tests const regressionTestList = %s; -""" % (json.dumps(repo), - json.dumps(os.path.join("js", "src", "jit-test", "lib") + os.sep), +""" % (json.dumps(str(repo) + os.sep), + json.dumps(os.sep.join(["js", "src", "jit-test", "lib"]) + os.sep), json.dumps(inTreeRegressionTests(repo))) def inTreeRegressionTests(repo): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - jit_tests = jsFilesIn(len(repo), os.path.join(repo, "js", "src", "jit-test", "tests")) - js_tests = jsFilesIn(len(repo), os.path.join(repo, "js", "src", "tests")) + jit_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "jit-test" / "tests") + js_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests") return jit_tests + js_tests def jsFilesIn(repoPathLength, root): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc return [os.path.join(path, filename)[repoPathLength:] - for path, _dirs, files in os.walk(sps.normExpUserPath(root)) + for path, _dirs, files in os.walk(str(root)) for filename in files if filename.endswith(".js")] @@ -124,21 +127,22 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in engineFlags = options.engineFlags # pylint: disable=invalid-name startTime = time.time() # pylint: disable=invalid-name - if os.path.isdir(sps.normExpUserPath(options.repo)): + if options.repo.is_dir(): regressionTestPrologue = makeRegressionTestPrologue(options.repo) # pylint: disable=invalid-name else: regressionTestPrologue = "" # pylint: disable=invalid-name - fuzzjs = sps.normExpUserPath(os.path.join(wtmpDir, "jsfunfuzz.js")) - linkFuzzer(fuzzjs, regressionTestPrologue) + fuzzjs = wtmpDir / "jsfunfuzz.js" + assert fuzzjs.is_file() + link_fuzzer.link_fuzzer(fuzzjs, regressionTestPrologue) iteration = 0 while True: if targetTime and time.time() > startTime + targetTime: print("Out of time!") - os.remove(fuzzjs) - if not os.listdir(wtmpDir): - os.rmdir(wtmpDir) + fuzzjs.unlink() + if not os.listdir(str(wtmpDir)): + wtmpDir.remove() break # Construct command needed to loop jsfunfuzz fuzzing. @@ -156,7 +160,7 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in js_interesting_options = js_interesting.parseOptions(js_interesting_args) iteration += 1 - logPrefix = sps.normExpUserPath(os.path.join(wtmpDir, "w" + str(iteration))) # pylint: disable=invalid-name + logPrefix = wtmpDir / ("w" + str(iteration)) # pylint: disable=invalid-name res = js_interesting.ShellResult(js_interesting_options, # pylint: disable=no-member @@ -170,12 +174,12 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in filenameToReduce = logPrefix + "-reduced.js" # pylint: disable=invalid-name [before, after] = file_manipulation.fuzzSplice(fuzzjs) - with open(logPrefix + "-out.txt", "r") as f: + with open(str(logPrefix + "-out.txt"), "r") as f: newfileLines = before + [ # pylint: disable=invalid-name l.replace("/*FRC-", "/*") for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after - with open(logPrefix + "-orig.js", "w") as f: + with open(str(logPrefix + "-orig.js"), "w") as f: f.writelines(newfileLines) - with open(filenameToReduce, "w") as f: + with open(str(filenameToReduce), "w") as f: f.writelines(newfileLines) # Run Lithium and autobisectjs (make a reduced testcase and find a regression window) @@ -219,14 +223,14 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in js_interesting_options.shellIsDeterministic and are_flags_deterministic: linesToCompare = jitCompareLines(logPrefix + "-out.txt", "/*FCM*/") # pylint: disable=invalid-name jitcomparefilename = logPrefix + "-cj-in.js" - with open(jitcomparefilename, "w") as f: + with open(str(jitcomparefilename), "w") as f: f.writelines(linesToCompare) # pylint: disable=invalid-name anyBug = compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename, logPrefix + "-cj", options.repo, options.build_options_str, targetTime, js_interesting_options) if not anyBug: - os.remove(jitcomparefilename) + jitcomparefilename.unlink() js_interesting.deleteLogs(logPrefix) @@ -251,7 +255,7 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid "wasmIsSupported = function() { return true; };\n", "// DDBEGIN\n" ] - with open(jsfunfuzzOutputFilename, "r") as f: + with open(str(jsfunfuzzOutputFilename), "r") as f: for line in f: if line.startswith(marker): sline = line[len(marker):] @@ -269,5 +273,5 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid if __name__ == "__main__": # pylint: disable=no-member - many_timed_runs(None, sps.createWtmpDir(os.getcwdu() if sys.version_info.major == 2 else os.getcwd()), - sys.argv[1:], create_collector.createCollector("jsfunfuzz")) + many_timed_runs(None, sps.make_wtmp_dir(Path(os.getcwdu() if sys.version_info.major == 2 else os.getcwd())), + sys.argv[1:], create_collector.make_collector()) diff --git a/src/funfuzz/util/__init__.py b/src/funfuzz/util/__init__.py index 30015c124..cf7a0f18e 100644 --- a/src/funfuzz/util/__init__.py +++ b/src/funfuzz/util/__init__.py @@ -14,9 +14,9 @@ from . import file_manipulation from . import fork_join from . import hg_helpers -from . import link_js from . import lithium_helpers from . import lock_dir +from . import os_ops from . import repos_update from . import s3cache from . import subprocesses diff --git a/src/funfuzz/util/crashesat.py b/src/funfuzz/util/crashesat.py index 4128052a9..5205d8cd8 100644 --- a/src/funfuzz/util/crashesat.py +++ b/src/funfuzz/util/crashesat.py @@ -6,7 +6,7 @@ """Lithium's "crashesat" interestingness test to assess whether a binary crashes at a desired location. -Not merged into Lithium, unsure if this still works for now. Still relies on grabCrashLog. +Not merged into Lithium, unsure if this still works for now. Still relies on grab_crash_log. """ from __future__ import absolute_import, print_function # isort:skip @@ -17,7 +17,7 @@ import lithium.interestingness.timed_run as timed_run from lithium.interestingness.utils import file_contains -from . import subprocesses as sps +from . import os_ops def parse_options(arguments): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc @@ -44,7 +44,7 @@ def interesting(cli_args, temp_prefix): # pylint: disable=missing-docstring,mis # Examine stack for crash signature, this is needed if crash_sig is specified. runinfo = timed_run.timed_run(args, timeout, temp_prefix) if runinfo.sta == timed_run.CRASHED: - sps.grabCrashLog(args[0], runinfo.pid, temp_prefix, True) + os_ops.grab_crash_log(args[0], runinfo.pid, temp_prefix, True) time_str = " (%.3f seconds)" % runinfo.elapsedtime diff --git a/src/funfuzz/util/create_collector.py b/src/funfuzz/util/create_collector.py index 96a4df6c2..104954b4b 100644 --- a/src/funfuzz/util/create_collector.py +++ b/src/funfuzz/util/create_collector.py @@ -9,20 +9,25 @@ from __future__ import absolute_import, print_function # isort:skip -import os +import sys from Collector.Collector import Collector +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error -def createCollector(tool): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - assert tool == "jsfunfuzz" - cache_dir = os.path.normpath(os.path.expanduser(os.path.join("~", "sigcache"))) - try: - os.mkdir(cache_dir) - except OSError: - pass # cache_dir already exists - collector = Collector(sigCacheDir=cache_dir, tool=tool) - return collector + +def make_collector(): + """Creates a jsfunfuzz collector specifying ~/sigcache as the signature cache dir + + Returns: + Collector: jsfunfuzz collector object + """ + sigcache_path = Path.home() / "sigcache" + sigcache_path.mkdir(exist_ok=True) + return Collector(sigCacheDir=str(sigcache_path), tool="jsfunfuzz") def printCrashInfo(crashInfo): # pylint: disable=invalid-name,missing-docstring diff --git a/src/funfuzz/util/detect_malloc_errors.py b/src/funfuzz/util/detect_malloc_errors.py index 7a6c5568e..c97101711 100644 --- a/src/funfuzz/util/detect_malloc_errors.py +++ b/src/funfuzz/util/detect_malloc_errors.py @@ -21,7 +21,7 @@ def amiss(log_prefix): # pylint: disable=missing-docstring,missing-return-doc,m PLINE = "" PPLINE = "" - with open(log_prefix + "-err.txt") as f: + with open(str(log_prefix + "-err.txt")) as f: for line in f: if scanLine(line): found_something = True diff --git a/src/funfuzz/util/file_manipulation.py b/src/funfuzz/util/file_manipulation.py index 56d29b006..6ad8232df 100644 --- a/src/funfuzz/util/file_manipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -15,7 +15,7 @@ def fuzzSplice(filename): # pylint: disable=invalid-name,missing-param-doc,miss """Return the lines of a file, minus the ones between the two lines containing SPLICE.""" before = [] after = [] - with open(filename, "r") as f: + with open(str(filename), "r") as f: for line in f: before.append(line) if line.find("SPLICE") != -1: diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index 419181d04..fa7d76631 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -10,11 +10,15 @@ from __future__ import absolute_import, print_function # isort:skip import multiprocessing -import os import sys from past.builtins import range # pylint: disable=redefined-builtin +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + # Call |fun| in a bunch of separate processes, then wait for them all to finish. # fun is called with someArgs, plus an additional argument with a numeric ID. @@ -24,7 +28,7 @@ def forkJoin(logDir, numProcesses, fun, *someArgs): # pylint: disable=invalid-n def showFile(fn): # pylint: disable=invalid-name,missing-docstring print("==== %s ====" % fn) print() - with open(fn) as f: + with open(str(fn)) as f: for line in f: print(line.rstrip()) print() @@ -45,20 +49,29 @@ def showFile(fn): # pylint: disable=invalid-name,missing-docstring p.join() print("=== Child process #%d exited with code %d ===" % (i, p.exitcode)) print() - showFile(logFileName(logDir, i, "out")) - showFile(logFileName(logDir, i, "err")) + showFile(log_name(logDir, i, "out")) + showFile(log_name(logDir, i, "err")) print() # Functions used by forkJoin are top-level so they can be "pickled" (required on Windows) -def logFileName(logDir, i, t): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return os.path.join(logDir, "forkjoin-" + str(i) + "-" + t + ".txt") +def log_name(log_dir, i, log_type): + """Returns the path of the forkjoin log file as a string. + + Args: + log_dir (str): Directory of the log file + i (int): Log number + log_type (str): Log type + + Returns: + str: The forkjoin log file path + """ + return str(Path(log_dir) / ("forkjoin-%s-%s.txt" % (i, log_type))) def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring - sys.stdout = open(logFileName(logDir, i, "out"), "w", buffering=0) - sys.stderr = open(logFileName(logDir, i, "err"), "w", buffering=0) + sys.stdout = open(log_name(logDir, i, "out"), "w", buffering=0) + sys.stderr = open(log_name(logDir, i, "err"), "w", buffering=0) fun(*(someArgs + (i,))) diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index 3ab1a49a4..c106f8370 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -18,51 +18,61 @@ from . import subprocesses as sps +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error -def destroyPyc(repoDir): # pylint: disable=invalid-name,missing-docstring + +def destroyPyc(repo_dir): # pylint: disable=invalid-name,missing-docstring # This is roughly equivalent to ["hg", "purge", "--all", "--include=**.pyc"]) # but doesn't run into purge's issues (incompatbility with -R, requiring an hg extension) - for root, dirs, files in os.walk(repoDir): + for root, dirs, files in os.walk(str(repo_dir)): for fn in files: # pylint: disable=invalid-name if fn.endswith(".pyc"): - os.remove(os.path.join(root, fn)) + (Path(root) / fn).unlink() if ".hg" in dirs: # Don't visit .hg dir dirs.remove(".hg") -def ensureMqEnabled(): # pylint: disable=invalid-name,missing-raises-doc - """Ensure that mq is enabled in the ~/.hgrc file.""" - user_hgrc = os.path.join(os.path.expanduser("~"), ".hgrc") - assert os.path.isfile(user_hgrc) +def ensure_mq_enabled(): + """Ensure that mq is enabled in the ~/.hgrc file. + + Raises: + NoOptionError: Raises if an mq entry is not found in [extensions] + """ + user_hgrc = Path.home() / ".hgrc" + assert user_hgrc.is_file() user_hgrc_cfg = configparser.SafeConfigParser() - user_hgrc_cfg.read(user_hgrc) + user_hgrc_cfg.read(str(user_hgrc)) try: user_hgrc_cfg.get("extensions", "mq") except configparser.NoOptionError: - raise Exception('Please first enable mq in ~/.hgrc by having "mq =" in [extensions].') + print('Please first enable mq in ~/.hgrc by having "mq =" in [extensions].') + raise -def findCommonAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-docstring,missing-return-doc +def findCommonAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - return sps.captureStdout(["hg", "-R", repoDir, "log", "-r", "ancestor(" + a + "," + b + ")", + return sps.captureStdout(["hg", "-R", str(repo_dir), "log", "-r", "ancestor(" + a + "," + b + ")", "--template={node|short}"])[0] -def isAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc +def isAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist.""" - return sps.captureStdout(["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", + return sps.captureStdout(["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"])[0] != "" -def existsAndIsAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc +def existsAndIsAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Return true iff |a| exists and is an ancestor of |b|.""" # Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error - out = sps.captureStdout(["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", + out = sps.captureStdout(["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"], combineStderr=True, ignoreExitCode=True)[0] return out != "" and out.decode("utf-8", errors="replace").find("abort: unknown revision") < 0 @@ -86,14 +96,23 @@ def get_cset_hash_from_bisect_msg(msg): raise ValueError("Bisection output format required for hash extraction unavailable. The variable msg is: %s" % msg) -def getRepoHashAndId(repoDir, repoRev="parents() and default"): # pylint: disable=invalid-name,missing-param-doc - # pylint: disable=missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc +def get_repo_hash_and_id(repo_dir, repo_rev="parents() and default"): """Return the repository hash and id, and whether it is on default. It will also ask what the user would like to do, should the repository not be on default. + + Args: + repo_dir (Path): Full path to the repository + repo_rev (str): Intended Mercurial changeset details to retrieve + + Raises: + ValueError: Raises if the input is invalid + + Returns: + tuple: Changeset hash, local numerical ID, boolean on whether the repository is on default tip """ # This returns null if the repository is not on default. - hg_log_template_cmds = ["hg", "-R", repoDir, "log", "-r", repoRev, + hg_log_template_cmds = ["hg", "-R", str(repo_dir), "log", "-r", repo_rev, "--template", "{node|short} {rev}"] hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] is_on_default = bool(hg_id_full) @@ -105,13 +124,13 @@ def getRepoHashAndId(repoDir, repoRev="parents() and default"): # pylint: disab print("Aborting...") sys.exit(0) elif update_default == "d": - subprocess.check_call(["hg", "-R", repoDir, "update", "default"]) + subprocess.check_call(["hg", "-R", str(repo_dir), "update", "default"]) is_on_default = True elif update_default == "u": - hg_log_template_cmds = ["hg", "-R", repoDir, "log", "-r", "parents()", "--template", + hg_log_template_cmds = ["hg", "-R", str(repo_dir), "log", "-r", "parents()", "--template", "{node|short} {rev}"] else: - raise Exception("Invalid choice.") + raise ValueError("Invalid choice.") hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] assert hg_id_full != "" (hg_id_hash, hg_id_local_num) = hg_id_full.decode("utf-8", errors="replace").split(" ") @@ -119,66 +138,82 @@ def getRepoHashAndId(repoDir, repoRev="parents() and default"): # pylint: disab return hg_id_hash, hg_id_local_num, is_on_default -def getRepoNameFromHgrc(repoDir): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Look in the hgrc file in the .hg directory of the repository and return the name.""" - assert isRepoValid(repoDir) +def hgrc_repo_name(repo_dir): + """Look in the hgrc file in the .hg directory of the Mercurial repository and return the name. + + Args: + repo_dir (Path): Mercurial repository directory + + Returns: + str: Returns the name of the Mercurial repository as indicated in the .hgrc + """ hgrc_cfg = configparser.SafeConfigParser() - hgrc_cfg.read(sps.normExpUserPath(os.path.join(repoDir, ".hg", "hgrc"))) + hgrc_cfg.read(str(repo_dir / ".hg" / "hgrc")) # Not all default entries in [paths] end with "/". return [i for i in hgrc_cfg.get("paths", "default").split("/") if i][-1] -def isRepoValid(repo): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Check that a repository is valid by ensuring that the hgrc file is around.""" - return os.path.isfile(sps.normExpUserPath(os.path.join(repo, ".hg", "hgrc"))) +def patch_hg_repo_with_mq(patch_file, repo_dir=None): + """Use mq to patch the Mercurial repository + Args: + patch_file (Path): Full path to the patch + repo_dir (Path): Working directory path -def patchHgRepoUsingMq(patchFile, workingDir=None): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - workingDir = workingDir or ( + Raises: + OSError: Raises when `hg qimport` or `hg qpush` did not return a return code of 0 + + Returns: + str: Returns the name of the patch file + """ + repo_dir = str(repo_dir) or ( os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member # We may have passed in the patch with or without the full directory. - patch_abs_path = os.path.abspath(sps.normExpUserPath(patchFile)) - pname = os.path.basename(patch_abs_path) - assert pname != "" - qimport_output, qimport_return_code = sps.captureStdout(["hg", "-R", workingDir, "qimport", patch_abs_path], + patch_abs_path = patch_file.resolve() + pname = patch_abs_path.name + qimport_output, qimport_return_code = sps.captureStdout(["hg", "-R", str(repo_dir), "qimport", str(patch_abs_path)], combineStderr=True, ignoreStderr=True, ignoreExitCode=True) if qimport_return_code != 0: if "already exists" in qimport_output: print("A patch with the same name has already been qpush'ed. Please qremove it first.") - raise Exception("Return code from `hg qimport` is: " + str(qimport_return_code)) + raise OSError("Return code from `hg qimport` is: " + str(qimport_return_code)) print("Patch qimport'ed...", end=" ") - qpush_output, qpush_return_code = sps.captureStdout(["hg", "-R", workingDir, "qpush", pname], + qpush_output, qpush_return_code = sps.captureStdout(["hg", "-R", str(repo_dir), "qpush", pname], combineStderr=True, ignoreStderr=True) assert " is empty" not in qpush_output, "Patch to be qpush'ed should not be empty." if qpush_return_code != 0: - hgQpopQrmAppliedPatch(patchFile, workingDir) + qpop_qrm_applied_patch(patch_file, repo_dir) print("You may have untracked .rej or .orig files in the repository.") - print("`hg status` output of the repository of interesting files in %s :" % workingDir) - subprocess.check_call(["hg", "-R", workingDir, "status", "--modified", "--added", + print("`hg status` output of the repository of interesting files in %s :" % repo_dir) + subprocess.check_call(["hg", "-R", str(repo_dir), "status", "--modified", "--added", "--removed", "--deleted"]) - raise Exception("Return code from `hg qpush` is: " + str(qpush_return_code)) + raise OSError("Return code from `hg qpush` is: " + str(qpush_return_code)) print("Patch qpush'ed. Continuing...", end=" ") return pname -def hgQpopQrmAppliedPatch(patchFile, repoDir): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc - # pylint: disable=missing-type-doc - """Remove applied patch using `hg qpop` and `hg qdelete`.""" - qpop_output, qpop_return_code = sps.captureStdout(["hg", "-R", repoDir, "qpop"], +def qpop_qrm_applied_patch(patch_file, repo_dir): + """Remove applied patch using `hg qpop` and `hg qdelete`. + + Args: + patch_file (Path): Full path to the patch + repo_dir (Path): Working directory path + + Raises: + OSError: Raises when `hg qpop` did not return a return code of 0 + """ + qpop_output, qpop_return_code = sps.captureStdout(["hg", "-R", str(repo_dir), "qpop"], combineStderr=True, ignoreStderr=True, ignoreExitCode=True) if qpop_return_code != 0: print("`hg qpop` output is: " + qpop_output) - raise Exception("Return code from `hg qpop` is: " + str(qpop_return_code)) + raise OSError("Return code from `hg qpop` is: " + str(qpop_return_code)) print("Patch qpop'ed...", end=" ") - subprocess.check_call(["hg", "-R", repoDir, "qdelete", os.path.basename(patchFile)]) + subprocess.check_call(["hg", "-R", str(repo_dir), "qdelete", patch_file.name]) print("Patch qdelete'd.") diff --git a/src/funfuzz/util/link_js.py b/src/funfuzz/util/link_js.py deleted file mode 100644 index dfa88c5d0..000000000 --- a/src/funfuzz/util/link_js.py +++ /dev/null @@ -1,39 +0,0 @@ -# coding=utf-8 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -"""Functions to concatenate files, with one specially for js files. -""" - -from __future__ import absolute_import, print_function # isort:skip - -import os - - -def link_js(target_fn, file_list_fn, source_base, prologue="", module_dirs=None): - # pylint: disable=missing-docstring - module_dirs = module_dirs or [] - with open(target_fn, "w") as target: - target.write(prologue) - - # Add files listed in file_list_fn - with open(file_list_fn) as file_list: - for source_fn in file_list: - source_fn = source_fn.replace("/", os.path.sep).strip() - if source_fn and source_fn[0] != "#": - add_contents(os.path.join(source_base, source_fn), target) - - # Add all *.js files in module_dirs - for module_base in module_dirs: - for module_fn in os.listdir(module_base): - if module_fn.endswith(".js"): - add_contents(os.path.join(module_base, module_fn), target) - - -def add_contents(source_fn, target): # pylint: disable=missing-docstring - target.write("\n\n// " + source_fn + "\n\n") - with open(source_fn) as source: - for line in source: - target.write(line) diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 557c0b82e..5e51d9000 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -9,7 +9,6 @@ from __future__ import absolute_import, print_function # isort:skip -import os import re import shutil import subprocess @@ -25,6 +24,11 @@ from ..js.js_interesting import JS_OVERALL_MISMATCH from ..js.js_interesting import JS_VG_AMISS +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + runlithiumpy = [sys.executable, "-u", "-m", "lithium"] # pylint: disable=invalid-name # Status returns for runLithium and many_timed_runs @@ -43,7 +47,7 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis """ lithArgs = itest + [jsEngine] + engineFlags + [infilename] # pylint: disable=invalid-name - (lithResult, lithDetails) = strategicReduction( # pylint: disable=invalid-name + (lithResult, lithDetails) = reduction_strat( # pylint: disable=invalid-name logPrefix, infilename, lithArgs, targetTime, suspiciousLevel) print() @@ -62,10 +66,10 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ) print(" ".join(quote(x) for x in autobisectCmd)) autoBisectLogFilename = logPrefix + "-autobisect.txt" # pylint: disable=invalid-name - subprocess.call(autobisectCmd, stdout=open(autoBisectLogFilename, "w"), stderr=subprocess.STDOUT) + subprocess.call(autobisectCmd, stdout=open(str(autoBisectLogFilename), "w"), stderr=subprocess.STDOUT) print("Done running autobisectjs. Log: %s" % autoBisectLogFilename) - with open(autoBisectLogFilename, "r") as f: + with open(str(autoBisectLogFilename), "r") as f: lines = f.readlines() autoBisectLog = file_manipulation.truncateMid(lines, 50, ["..."]) # pylint: disable=invalid-name else: @@ -74,7 +78,7 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis return (lithResult, lithDetails, autoBisectLog) -def runLithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc +def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level. @@ -89,12 +93,12 @@ def runLithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name else: # loop is being run standalone lithtmp = logPrefix + "-lith-tmp" - os.mkdir(lithtmp) + Path.mkdir(lithtmp) lithArgs = ["--tempdir=" + lithtmp] + lithArgs lithlogfn = logPrefix + "-lith-out.txt" print("Preparing to run Lithium, log file %s" % lithlogfn) print(" ".join(quote(x) for x in runlithiumpy + lithArgs)) - subprocess.call(runlithiumpy + lithArgs, stdout=open(lithlogfn, "w"), stderr=subprocess.STDOUT) + subprocess.call(runlithiumpy + lithArgs, stdout=open(str(lithlogfn), "w"), stderr=subprocess.STDOUT) print("Done running Lithium") if deletableLithTemp: shutil.rmtree(deletableLithTemp) @@ -105,7 +109,7 @@ def runLithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name def readLithiumResult(lithlogfn): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - with open(lithlogfn) as f: + with open(str(lithlogfn)) as f: for line in f: if line.startswith("Lithium result"): print(line.rstrip()) @@ -125,43 +129,52 @@ def readLithiumResult(lithlogfn): # pylint: disable=invalid-name,missing-docstr return (LITH_BUSTED, None) -def strategicReduction(logPrefix, infilename, lithArgs, targetTime, lev): # pylint: disable=invalid-name +def reduction_strat(logPrefix, infilename, lithArgs, targetTime, lev): # pylint: disable=invalid-name # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc,too-complex # pylint: disable=too-many-branches,too-many-locals,too-many-statements """Reduce jsfunfuzz output files using Lithium by using various strategies.""" + # RMassert isinstance(cacheDir, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + # This is an array because Python does not like assigning to upvars. reductionCount = [0] # pylint: disable=invalid-name - backupFilename = infilename + "-backup" # pylint: disable=invalid-name + backup_file = infilename + "-backup" - def lithReduceCmd(strategy): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc - """Lithium reduction commands accepting various strategies.""" + """Lithium reduction commands accepting various strategies. + + Args: + strategy (str): Intended strategy to use + + Returns: + (tuple): The finished Lithium run result and details + """ reductionCount[0] += 1 # Remove empty elements - fullLithArgs = [x for x in (strategy + lithArgs) if x] # pylint: disable=invalid-name - print(" ".join(quote(x) for x in [sys.executable, "-u", "-m", "lithium"] + fullLithArgs)) + full_lith_args = [x for x in (strategy + lithArgs) if x] + print(" ".join(quote(x) for x in [sys.executable, "-u", "-m", "lithium"] + full_lith_args)) desc = "-chars" if strategy == "--char" else "-lines" - (lithResult, lithDetails) = runLithium( # pylint: disable=invalid-name - fullLithArgs, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) - if lithResult == LITH_FINISHED: - shutil.copy2(infilename, backupFilename) + (lith_result, lith_details) = run_lithium( # pylint: disable=invalid-name + full_lith_args, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) + if lith_result == LITH_FINISHED: + shutil.copy2(infilename, backup_file) - return lithResult, lithDetails + return lith_result, lith_details print() print("Running the first line reduction...") print() # Step 1: Run the first instance of line reduction. - lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce([]) # pylint: disable=invalid-name - if lithDetails is not None: # lithDetails can be None if testcase no longer becomes interesting - origNumOfLines = int(lithDetails.split()[0]) # pylint: disable=invalid-name + if lith_details is not None: # lith_details can be None if testcase no longer becomes interesting + origNumOfLines = int(lith_details.split()[0]) # pylint: disable=invalid-name hasTryItOut = False # pylint: disable=invalid-name hasTryItOutRegex = re.compile(r'count=[0-9]+; tryItOut\("') # pylint: disable=invalid-name - with open(infilename, "r") as f: + with open(str(infilename), "r") as f: for line in file_manipulation.linesWith(f, '; tryItOut("'): # Checks if testcase came from jsfunfuzz or compare_jit. # Do not use .match here, it only matches from the start of the line: @@ -171,29 +184,29 @@ def lithReduceCmd(strategy): # pylint: disable=invalid-name,missing-param-doc,m break # Step 2: Run 1 instance of 1-line reduction after moving tryItOut and count=X around. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: tryItOutAndCountRegex = re.compile(r'"\);\ncount=([0-9]+); tryItOut\("', # pylint: disable=invalid-name re.MULTILINE) - with open(infilename, "r") as f: + with open(str(infilename), "r") as f: infileContents = f.read() # pylint: disable=invalid-name infileContents = re.sub(tryItOutAndCountRegex, # pylint: disable=invalid-name ';\\\n"); count=\\1; tryItOut("\\\n', infileContents) - with open(infilename, "w") as f: + with open(str(infilename), "w") as f: f.write(infileContents) print() print("Running 1 instance of 1-line reduction after moving tryItOut and count=X...") print() # --chunksize=1: Reduce only individual lines, for only 1 round. - lithResult, lithDetails = lithReduceCmd(["--chunksize=1"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--chunksize=1"]) # pylint: disable=invalid-name # Step 3: Run 1 instance of 2-line reduction after moving count=X to its own line and add a # 1-line offset. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: intendedLines = [] # pylint: disable=invalid-name - with open(infilename, "r") as f: + with open(str(infilename), "r") as f: for line in f: # The testcase is likely to already be partially reduced. if "dumpln(cookie" not in line: # jsfunfuzz-specific line ignore # This should be simpler than re.compile. @@ -202,61 +215,61 @@ def lithReduceCmd(strategy): # pylint: disable=invalid-name,missing-param-doc,m # The 1-line offset is added here. .replace("SPLICE DDBEGIN", "SPLICE DDBEGIN\n")) - with open(infilename, "w") as f: + with open(str(infilename), "w") as f: f.writelines(intendedLines) print() print("Running 1 instance of 2-line reduction after moving count=X to its own line...") print() - lithResult, lithDetails = lithReduceCmd(["--chunksize=2"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--chunksize=2"]) # pylint: disable=invalid-name # Step 4: Run 1 instance of 2-line reduction again, e.g. to remove pairs of STRICT_MODE lines. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: print() print("Running 1 instance of 2-line reduction again...") print() - lithResult, lithDetails = lithReduceCmd(["--chunksize=2"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--chunksize=2"]) # pylint: disable=invalid-name isLevOverallMismatchAsmJsAvailable = (lev == JS_OVERALL_MISMATCH and # pylint: disable=invalid-name - file_contains_str(infilename, "isAsmJSCompilationAvailable")) + file_contains_str(str(infilename), "isAsmJSCompilationAvailable")) # Step 5 (not always run): Run character reduction within interesting lines. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and targetTime is None and \ + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and targetTime is None and \ lev >= JS_OVERALL_MISMATCH and not isLevOverallMismatchAsmJsAvailable: print() print("Running character reduction...") print() - lithResult, lithDetails = lithReduceCmd(["--char"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--char"]) # pylint: disable=invalid-name # Step 6: Run line reduction after activating SECOND DDBEGIN with a 1-line offset. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: infileContents = [] # pylint: disable=invalid-name - with open(infilename, "r") as f: + with open(str(infilename), "r") as f: for line in f: if "NIGEBDD" in line: infileContents.append(line.replace("NIGEBDD", "DDBEGIN")) infileContents.append("\n") # The 1-line offset is added here. continue infileContents.append(line) - with open(infilename, "w") as f: + with open(str(infilename), "w") as f: f.writelines(infileContents) print() print("Running line reduction with a 1-line offset...") print() - lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce([]) # pylint: disable=invalid-name # Step 7: Run line reduction for a final time. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: print() print("Running the final line reduction...") print() - lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce([]) # pylint: disable=invalid-name # Restore from backup if testcase can no longer be reproduced halfway through reduction. - if lithResult != LITH_FINISHED and lithResult != LITH_PLEASE_CONTINUE: + if lith_result != LITH_FINISHED and lith_result != LITH_PLEASE_CONTINUE: # Probably can move instead of copy the backup, once this has stabilised. - if os.path.isfile(backupFilename): - shutil.copy2(backupFilename, infilename) + if backup_file.is_file(): + shutil.copy2(str(backup_file), infilename) else: - print("DEBUG! backupFilename is supposed to be: %s" % backupFilename) + print("DEBUG! backup_file is supposed to be: %s" % backup_file) - return lithResult, lithDetails + return lith_result, lith_details diff --git a/src/funfuzz/util/lock_dir.py b/src/funfuzz/util/lock_dir.py index edf9e6c39..9b0297e77 100644 --- a/src/funfuzz/util/lock_dir.py +++ b/src/funfuzz/util/lock_dir.py @@ -11,15 +11,17 @@ from __future__ import absolute_import, print_function # isort:skip from builtins import object # pylint: disable=redefined-builtin -import os -class LockDir(object): # pylint: disable=missing-param-doc,missing-type-doc,too-few-public-methods +class LockDir(object): # pylint: disable=too-few-public-methods """Create a filesystem-based lock while in scope. Use: with LockDir(path): # No other code is concurrently using LockDir(path) + + Args: + directory (str): Lock directory name """ def __init__(self, directory): @@ -27,10 +29,10 @@ def __init__(self, directory): def __enter__(self): try: - os.mkdir(self.directory) + self.directory.mkdir() except OSError: - print("Lock file exists: %s" % self.directory) + print("Lock directory exists: %s" % self.directory) raise def __exit__(self, exc_type, exc_val, exc_tb): - os.rmdir(self.directory) + self.directory.rmdir() diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py new file mode 100644 index 000000000..3812a8db6 --- /dev/null +++ b/src/funfuzz/util/os_ops.py @@ -0,0 +1,373 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Functions dealing with files and their contents. +""" + +from __future__ import absolute_import, print_function # isort:skip + +import os +import platform +import shutil +import sys +import time + +from pkg_resources import parse_version +from shellescape import quote + +from . import subprocesses as sps + +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + +NO_DUMP_MSG = r""" +WARNING: Minidumps are not being generated, so all crashes will be uninteresting. +WARNING: Make sure the following key value exists in this key: +WARNING: HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps +WARNING: Name: DumpType Type: REG_DWORD +WARNING: http://msdn.microsoft.com/en-us/library/windows/desktop/bb787181%28v=vs.85%29.aspx +""" + + +def make_cdb_cmd(prog_full_path, crashed_pid): + """Construct a command that uses the Windows debugger (cdb.exe) to turn a minidump file into a stack trace. + + Args: + prog_full_path (Path): Full path to the program + crashed_pid (int): PID of the program + + Returns: + list: cdb command list + """ + assert platform.system() == "Windows" + # Look for a minidump. + dump_name = Path.home() / "AppData" / "Local" / "CrashDumps" / "%s.%s.dmp" % (prog_full_path.name, crashed_pid) + + if platform.uname()[2] == "10": # Windows 10 + win64_debugging_folder = Path(os.getenv("PROGRAMFILES(X86)")) / "Windows Kits" / "10" / "Debuggers" / "x64" + else: + win64_debugging_folder = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" + + # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. + cdb_path = win64_debugging_folder / "cdb.exe" + if cdb_path.is_file(): + print() + print("WARNING: cdb.exe is not found - all crashes will be interesting.") + print() + return [] + + if is_win_dumping_to_default(): + loops = 0 + max_loops = 300 + while True: + if dump_name.is_file(): + dbggr_cmd_path = Path(__file__).parent / "cdb_cmds.txt" + assert dbggr_cmd_path.is_file() + + cdb_cmd_list = [] + cdb_cmd_list.append("$<" + str(dbggr_cmd_path)) + + # See bug 902706 about -g. + return [cdb_path, "-g", "-c", ";".join(cdb_cmd_list), "-z", str(dump_name)] + + time.sleep(0.200) + loops += 1 + if loops > max_loops: + # Windows may take some time to generate the dump. + print("make_cdb_cmd waited a long time, but %s never appeared!" % str(dump_name)) + return [] + else: + return [] + + +def make_gdb_cmd(prog_full_path, crashed_pid): + """Construct a command that uses the POSIX debugger (gdb) to turn a minidump file into a stack trace. + + Args: + prog_full_path (Path): Full path to the program + crashed_pid (int): PID of the program + + Returns: + list: gdb command list + """ + assert os.name == "posix" + # On Mac and Linux, look for a core file. + core_name = None + if platform.system() == "Darwin": + # Core files will be generated if you do: + # mkdir -p /cores/ + # ulimit -c 2147483648 (or call resource.setrlimit from a preexec_fn hook) + core_name = "/cores/core." + str(crashed_pid) + elif platform.system() == "Linux": + is_pid_used = False + core_uses_pid_path = Path("/proc/sys/kernel/core_uses_pid") + if core_uses_pid_path.is_file(): + with open(str(core_uses_pid_path)) as f: + is_pid_used = bool(int(f.read()[0])) # Setting [0] turns the input to a str. + core_name = "core." + str(crashed_pid) if is_pid_used else "core" + core_name_path = Path.cwd() / core_name + if not core_name_path.is_file(): + core_name_path = Path.home() / core_name # try the home dir + + if core_name and core_name_path.is_file(): + dbggr_cmd_path = Path(__file__).parent / "gdb_cmds.txt" + assert dbggr_cmd_path.is_file() + + # Run gdb and move the core file. Tip: gdb gives more info for: + # (debug with intact build dir > debug > opt with frame pointers > opt) + return ["gdb", "-n", "-batch", "-x", str(dbggr_cmd_path), str(prog_full_path), str(core_name)] + return [] + + +def disable_corefile(): + """When called as a preexec_fn, sets appropriate resource limits for the JS shell. Must only be called on POSIX.""" + import resource # module only available on POSIX pylint: disable=import-error + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # pylint: disable=no-member + + +def get_core_limit(): + """Returns the maximum core file size that the current process can create. + + Returns: + int: Maximum size (in bytes) of a core file that the current process can create. + """ + import resource # module only available on POSIX pylint: disable=import-error + return resource.getrlimit(resource.RLIMIT_CORE) # pylint: disable=no-member + + +# pylint: disable=inconsistent-return-statements +def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): + # pylint: disable=too-complex,too-many-branches + """Return the crash log if found. + + Args: + prog_full_path (Path): Full path to the program + crashed_pid (int): PID of the crashed program + log_prefix (str): Log prefix + want_stack (bool): Boolean on whether a stack is desired + + Returns: + str: Returns the path to the crash log + """ + progname = prog_full_path.name + + use_logfiles = isinstance(log_prefix, ("".__class__, u"".__class__)) + crash_log_path = Path(log_prefix + "-crash.txt") + core_path = Path(log_prefix + "-core") + + if use_logfiles: + if crash_log_path.is_file(): + crash_log_path.unlink() + if core_path.is_file(): + core_path.unlink() + + if not want_stack or progname == "valgrind": + return "" + + # This has only been tested on 64-bit Windows 7 and higher + if platform.system() == "Windows": + dbggr_cmd = make_cdb_cmd(prog_full_path, crashed_pid) + elif os.name == "posix": + dbggr_cmd = make_gdb_cmd(prog_full_path, crashed_pid) + else: + dbggr_cmd = None + + if dbggr_cmd: + sps.vdump(" ".join(dbggr_cmd)) + core_file = Path(dbggr_cmd[-1]) + assert core_file.is_file() + dbbgr_exit_code = subprocess.call( + dbggr_cmd, + stdin=None, + stderr=subprocess.STDOUT, + stdout=open(str(crash_log_path), "w") if use_logfiles else None, + # It would be nice to use this everywhere, but it seems to be broken on Windows + # (http://docs.python.org/library/subprocess.html) + close_fds=(os.name == "posix"), + # Do not generate a core_file if gdb crashes in Linux + preexec_fn=(disable_corefile if platform.system() == "Linux" else None) + ) + if dbbgr_exit_code != 0: + print("Debugger exited with code %d : %s" % (dbbgr_exit_code, " ".join(quote(x) for x in dbggr_cmd))) + if use_logfiles: + if core_file.is_file(): + shutil.move(str(core_file), str(core_path)) + subprocess.call(["gzip", "-f", str(core_path)]) + # chmod here, else the uploaded -core.gz files do not have sufficient permissions. + subprocess.check_call(["chmod", "og+r", "%s.gz" % core_path]) + return str(crash_log_path) + else: + print("I don't know what to do with a core file when log_prefix is null") + + # On Mac, look for a crash log generated by Mac OS X Crash Reporter + elif platform.system() == "Darwin": + loops = 0 + max_loops = 500 if progname.startswith("firefox") else 450 + while True: + crash_log_found = grab_mac_crash_log(crashed_pid, log_prefix, use_logfiles) + if crash_log_found is not None: + return crash_log_found + + # print "[grab_crash_log] Waiting for the crash log to appear..." + time.sleep(0.200) + loops += 1 + if loops > max_loops: + # I suppose this might happen if the process corrupts itself so much that + # the crash reporter gets confused about the process name, for example. + print("grab_crash_log waited a long time, but a crash log for %s [%s] never appeared!" % ( + progname, crashed_pid)) + break + + elif platform.system() == "Linux": + print("Warning: grab_crash_log() did not find a core file for PID %d." % crashed_pid) + print("Note: Your soft limit for core file sizes is currently %d. " + 'You can increase it with "ulimit -c" in bash.' % get_core_limit()[0]) + + +def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): + """Find the required crash log in the given crash reporter directory. + + Args: + crash_pid (str): PID value of the crashed process + log_prefix (str): Prefix (may include dirs) of the log file + use_log_files (bool): Boolean that decides whether *-crash.txt log files should be used + + Returns: + str: Absolute (if use_log_files is False) or relative (if use_log_files is True) path to crash log file + """ + assert parse_version(platform.mac_ver()[0]) >= parse_version("10.6") + + for base_dir in [Path.home(), Path("/")]: + # Sometimes the crash reports end up in the root directory. + # This possibly happens when the value of : + # defaults write com.apple.CrashReporter DialogType + # is none, instead of server, or some other option. + # It also happens when ssh'd into a computer. + # And maybe when the computer is under heavy load. + # See http://en.wikipedia.org/wiki/Crash_Reporter_%28Mac_OS_X%29 + reports_dir = base_dir / "Library" / "Logs" / "DiagnosticReports" + # Find a crash log for the right process name and pid, preferring + # newer crash logs (which sort last). + if reports_dir.is_dir(): + crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) + else: + crash_logs = [] + + for file_name in crash_logs: + full_report_path = reports_dir / file_name + try: + with open(str(full_report_path)) as f: + first_line = f.readline() + if first_line.rstrip().endswith("[%s]" % crash_pid): + if use_log_files: + # Copy, don't rename, because we might not have permissions + # (especially for the system rather than user crash log directory) + # Use copyfile, as we do not want to copy the permissions metadata over + shutil.copyfile(str(full_report_path), log_prefix + "-crash.txt") + subprocess.run(["chmod", "og+r", log_prefix + "-crash.txt"], + # pylint: disable=no-member + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), + check=True, + timeout=9) + return log_prefix + "-crash.txt" + return str(full_report_path) + + except OSError: + # Maybe the log was rotated out between when we got the list + # of files and when we tried to open this file. If so, it's + # clearly not The One. + pass + return None + + +def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branches + """Check whether Windows minidumps are enabled and set to go to Windows' default location. + + Returns: + bool: Returns True when Windows has dumping enabled, and is dumping to the default location, otherwise False + """ + if sys.version_info.major == 2: + import _winreg as winreg # pylint: disable=import-error + else: + import winreg # pylint: disable=import-error + # For now, this code does not edit the Windows Registry because we tend to be in a 32-bit + # version of Python and if one types in regedit in the Run dialog, opens up the 64-bit registry. + # If writing a key, we most likely need to flush. For the moment, no keys are written. + try: + with winreg.OpenKey(winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE), + r"Software\Microsoft\Windows\Windows Error Reporting\LocalDumps", + # Read key from 64-bit registry, which also works for 32-bit + 0, (winreg.KEY_WOW64_64KEY + winreg.KEY_READ)) as key: + + try: + dump_type_reg_value = winreg.QueryValueEx(key, "DumpType") + if not (dump_type_reg_value[0] == 1 and dump_type_reg_value[1] == winreg.REG_DWORD): + print(NO_DUMP_MSG) + return False + except WindowsError as ex: # pylint: disable=undefined-variable + if ex.errno == 2: + print(NO_DUMP_MSG) + return False + else: + raise + + try: + dump_folder_reg_value = winreg.QueryValueEx(key, "DumpFolder") + # %LOCALAPPDATA%\CrashDumps is the default location. + if not (dump_folder_reg_value[0] == r"%LOCALAPPDATA%\CrashDumps" and + dump_folder_reg_value[1] == winreg.REG_EXPAND_SZ): + print() + print("WARNING: Dumps are instead appearing at: %s - " + "all crashes will be uninteresting." % dump_folder_reg_value[0]) + print() + return False + except WindowsError as ex: # pylint: disable=undefined-variable + # If the key value cannot be found, the dumps will be put in the default location + if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": + return True + else: + raise + + return True + except WindowsError as ex: # pylint: disable=undefined-variable + # If the LocalDumps registry key cannot be found, dumps will be put in the default location. + if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": + print() + print("WARNING: The registry key HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\" + "Windows\\Windows Error Reporting\\LocalDumps cannot be found.") + print() + return False + else: + raise + + +def make_wtmp_dir(base_dir): + """Create wtmp directory, incrementing the number if one is already found. + + Args: + base_dir (Path): Base directory to create the wtmp directories + + Returns: + Path: Full path to the numbered wtmp directory + """ + assert isinstance(base_dir, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + + i = 1 + while True: + numbered_tmp_dir = "wtmp%s" % i + full_dir = base_dir / numbered_tmp_dir + try: + full_dir.mkdir() # To avoid race conditions, we use try/except instead of exists/create + break # break out of the while loop + except OSError: + i += 1 + + return full_dir diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index 4d71243e4..a7c875bb6 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -37,11 +37,11 @@ if platform.system() == "Windows": # pylint: disable=invalid-name - git_64bit_path = os.path.normpath(os.path.join(os.getenv("PROGRAMFILES"), "Git", "bin", "git.exe")) - git_32bit_path = os.path.normpath(os.path.join(os.getenv("PROGRAMFILES(X86)"), "Git", "bin", "git.exe")) - if os.path.isfile(git_64bit_path): + git_64bit_path = Path(os.getenv("PROGRAMFILES")) / "Git" / "bin" / "git.exe" + git_32bit_path = Path(os.getenv("PROGRAMFILES(X86)")) / "Git" / "bin" / "git.exe" + if git_64bit_path.is_file(): GITBINARY = git_64bit_path - elif os.path.isfile(git_32bit_path): + elif git_32bit_path.is_file(): GITBINARY = git_32bit_path else: raise OSError("Git binary not found") @@ -77,7 +77,7 @@ def typeOfRepo(r): # pylint: disable=invalid-name,missing-param-doc,missing-rai repo_types.append(".hg") repo_types.append(".git") for rtype in repo_types: - if os.path.isdir(os.path.join(r, rtype)): + if (r / rtype).is_dir(): return rtype[1:] raise Exception("Type of repository located at " + r + " cannot be determined.") @@ -85,20 +85,20 @@ def typeOfRepo(r): # pylint: disable=invalid-name,missing-param-doc,missing-rai def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Update a repository. Return False if missing; return True if successful; raise an exception if updating fails.""" - assert os.path.isdir(repo) + repo.is_dir() repo_type = typeOfRepo(repo) if repo_type == "hg": hg_pull_cmd = ["hg", "--time", "pull", "-u"] logger.info("\nRunning `%s` now..\n", " ".join(hg_pull_cmd)) - out_hg_pull = subprocess.run(hg_pull_cmd, check=True, cwd=repo, stderr=subprocess.PIPE) + out_hg_pull = subprocess.run(hg_pull_cmd, check=True, cwd=str(repo), stderr=subprocess.PIPE) logger.info('"%s" had the above output and took - %s', subprocess.list2cmdline(out_hg_pull.args), out_hg_pull.stderr) hg_log_default_cmd = ["hg", "--time", "log", "-r", "default"] logger.info("\nRunning `%s` now..\n", " ".join(hg_log_default_cmd)) - out_hg_log_default = subprocess.run(hg_log_default_cmd, check=True, cwd=repo, + out_hg_log_default = subprocess.run(hg_log_default_cmd, check=True, cwd=str(repo), stderr=subprocess.PIPE) logger.info('"%s" had the above output and took - %s', subprocess.list2cmdline(out_hg_log_default.args), @@ -108,7 +108,7 @@ def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing- gitenv = deepcopy(os.environ) if platform.system() == "Windows": gitenv["GIT_SSH_COMMAND"] = "~/../../mozilla-build/msys/bin/ssh.exe -F ~/.ssh/config" - time_cmd([GITBINARY, "pull"], cwd=repo, env=gitenv) + time_cmd([GITBINARY, "pull"], cwd=str(repo), env=gitenv) else: raise Exception("Unknown repository type: " + repo_type) @@ -117,15 +117,15 @@ def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing- def updateRepos(): # pylint: disable=invalid-name """Update Mercurial and Git repositories located in ~ and ~/trees .""" - home_dir = sps.normExpUserPath("~") + home_dir = Path.home() trees = [ - os.path.normpath(os.path.join(home_dir)), - os.path.normpath(os.path.join(home_dir, "trees")) + home_dir, + home_dir / "trees" ] for tree in trees: - for name in sorted(os.listdir(tree)): - name_path = os.path.join(tree, name) - if os.path.isdir(name_path) and (name in REPOS or (name.startswith("funfuzz") and "-" in name)): + for name in sorted(os.listdir(str(tree))): + name_path = Path(tree) / name + if name_path.is_dir() and (name in REPOS or (name.startswith("funfuzz") and "-" in name)): logger.info("Updating %s ...", name) updateRepo(name_path) diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index 414bed0a0..f20bbcd08 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -16,29 +16,18 @@ import stat import subprocess import sys -import time -from pkg_resources import parse_version from shellescape import quote verbose = False # pylint: disable=invalid-name -# pylint: disable=invalid-name -noMinidumpMsg = r""" -WARNING: Minidumps are not being generated, so all crashes will be uninteresting. -WARNING: Make sure the following key value exists in this key: -WARNING: HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps -WARNING: Name: DumpType Type: REG_DWORD -WARNING: http://msdn.microsoft.com/en-us/library/windows/desktop/bb787181%28v=vs.85%29.aspx -""" - # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc # pylint: disable=missing-type-doc,too-complex,too-many-arguments,too-many-branches,too-many-statements def captureStdout(inputCmd, ignoreStderr=False, combineStderr=False, ignoreExitCode=False, currWorkingDir=None, env="NOTSET", verbosity=False): """Capture standard output, return the output as a string, along with the return value.""" - currWorkingDir = currWorkingDir or ( + currWorkingDir = str(currWorkingDir) or ( os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member if env == "NOTSET": vdump(" ".join(quote(x) for x in inputCmd)) @@ -114,317 +103,50 @@ def captureStdout(inputCmd, ignoreStderr=False, combineStderr=False, ignoreExitC return stdout.rstrip(), p.returncode -def createWtmpDir(tmpDirBase): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Create wtmp directory, incrementing the number if one is already found.""" - i = 1 - while True: - tmpDirWithNum = "wtmp" + str(i) - tmpDir = os.path.join(tmpDirBase, tmpDirWithNum) - try: - os.mkdir(tmpDir) # To avoid race conditions, we use try/except instead of exists/create - break - except OSError: - i += 1 - vdump(tmpDirWithNum + os.sep) # Even if not verbose, wtmp is also dumped: wtmp1/w1: NORMAL - return tmpDirWithNum - - -def disableCorefile(): - """When called as a preexec_fn, sets appropriate resource limits for the JS shell. Must only be called on POSIX.""" - import resource # module only available on POSIX pylint: disable=import-error - resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # pylint: disable=no-member - - -def getCoreLimit(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - import resource # module only available on POSIX pylint: disable=import-error - return resource.getrlimit(resource.RLIMIT_CORE) # pylint: disable=no-member - - -def grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles): # pylint: disable=invalid-name,missing-param-doc - # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc - """Find the required crash log in the given crash reporter directory.""" - assert parse_version(platform.mac_ver()[0]) >= parse_version("10.6") - reportDirList = [os.path.expanduser("~"), "/"] - for baseDir in reportDirList: - # Sometimes the crash reports end up in the root directory. - # This possibly happens when the value of : - # defaults write com.apple.CrashReporter DialogType - # is none, instead of server, or some other option. - # It also happens when ssh'd into a computer. - # And maybe when the computer is under heavy load. - # See http://en.wikipedia.org/wiki/Crash_Reporter_%28Mac_OS_X%29 - reportDir = os.path.join(baseDir, "Library/Logs/DiagnosticReports/") - # Find a crash log for the right process name and pid, preferring - # newer crash logs (which sort last). - if os.path.exists(reportDir): - crashLogs = os.listdir(reportDir) - else: - crashLogs = [] - # Firefox sometimes still runs as firefox-bin, at least on Mac (likely bug 658850) - crashLogs = [x for x in crashLogs - if x.startswith(progname + "_") or x.startswith(progname + "-bin_")] - crashLogs.sort(reverse=True) - for fn in crashLogs: - fullfn = os.path.join(reportDir, fn) - try: - with open(fullfn) as c: - firstLine = c.readline() - if firstLine.rstrip().endswith("[" + str(crashedPID) + "]"): - if useLogFiles: - # Copy, don't rename, because we might not have permissions - # (especially for the system rather than user crash log directory) - # Use copyfile, as we do not want to copy the permissions metadata over - shutil.copyfile(fullfn, logPrefix + "-crash.txt") - captureStdout(["chmod", "og+r", logPrefix + "-crash.txt"]) - return logPrefix + "-crash.txt" - return fullfn - # return open(fullfn).read() - - except (OSError, IOError): # pylint: disable=overlapping-except - # Maybe the log was rotated out between when we got the list - # of files and when we tried to open this file. If so, it's - # clearly not The One. - pass - return None - - -def grabCrashLog(progfullname, crashedPID, logPrefix, wantStack): # pylint: disable=inconsistent-return-statements - # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc - # pylint: disable=too-complex,too-many-branches - """Return the crash log if found.""" - progname = os.path.basename(progfullname) - - useLogFiles = isinstance(logPrefix, ("".__class__, u"".__class__)) - if useLogFiles: - if os.path.exists(logPrefix + "-crash.txt"): - os.remove(logPrefix + "-crash.txt") - if os.path.exists(logPrefix + "-core"): - os.remove(logPrefix + "-core") - - if not wantStack or progname == "valgrind": - return - - # This has only been tested on 64-bit Windows 7 and higher - if platform.system() == "Windows": - debuggerCmd = constructCdbCommand(progfullname, crashedPID) - elif os.name == "posix": - debuggerCmd = constructGdbCommand(progfullname, crashedPID) - else: - debuggerCmd = None - - if debuggerCmd: - vdump(" ".join(debuggerCmd)) - coreFile = debuggerCmd[-1] - assert os.path.isfile(coreFile) - debuggerExitCode = subprocess.call( - debuggerCmd, - stdin=None, - stderr=subprocess.STDOUT, - stdout=open(logPrefix + "-crash.txt", "w") if useLogFiles else None, - # It would be nice to use this everywhere, but it seems to be broken on Windows - # (http://docs.python.org/library/subprocess.html) - close_fds=(os.name == "posix"), - # Do not generate a corefile if gdb crashes in Linux - preexec_fn=(disableCorefile if platform.system() == "Linux" else None) - ) - if debuggerExitCode != 0: - print("Debugger exited with code %d : %s" % (debuggerExitCode, " ".join(quote(x) for x in debuggerCmd))) - if useLogFiles: - if os.path.isfile(coreFile): - shutil.move(coreFile, logPrefix + "-core") - subprocess.call(["gzip", "-f", logPrefix + "-core"]) - # chmod here, else the uploaded -core.gz files do not have sufficient permissions. - subprocess.check_call(["chmod", "og+r", logPrefix + "-core.gz"]) - return logPrefix + "-crash.txt" - else: - print("I don't know what to do with a core file when logPrefix is null") - - # On Mac, look for a crash log generated by Mac OS X Crash Reporter - elif platform.system() == "Darwin": - loops = 0 - maxLoops = 500 if progname.startswith("firefox") else 450 - while True: - cLogFound = grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles) - if cLogFound is not None: - return cLogFound - - # print "[grabCrashLog] Waiting for the crash log to appear..." - time.sleep(0.200) - loops += 1 - if loops > maxLoops: - # I suppose this might happen if the process corrupts itself so much that - # the crash reporter gets confused about the process name, for example. - print("grabCrashLog waited a long time, but a crash log for %s [%s] never appeared!" % ( - progname, crashedPID)) - break - - elif platform.system() == "Linux": - print("Warning: grabCrashLog() did not find a core file for PID %d." % crashedPID) - print("Note: Your soft limit for core file sizes is currently %d. " - 'You can increase it with "ulimit -c" in bash.' % getCoreLimit()[0]) - - -def constructCdbCommand(progfullname, crashedPID): # pylint: disable=inconsistent-return-statements,invalid-name - # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc - """Construct a command that uses the Windows debugger (cdb.exe) to turn a minidump file into a stack trace.""" - assert platform.system() == "Windows" - # Look for a minidump. - dumpFilename = normExpUserPath(os.path.join( - "~", "AppData", "Local", "CrashDumps", os.path.basename(progfullname) + "." + str(crashedPID) + ".dmp")) - if platform.uname()[2] == "10": # Windows 10 - win64bitDebuggerFolder = os.path.join(os.getenv("PROGRAMFILES(X86)"), "Windows Kits", "10", "Debuggers", "x64") - else: - win64bitDebuggerFolder = os.path.join(os.getenv("PROGRAMW6432"), "Debugging Tools for Windows (x64)") - # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. - cdbPath = os.path.join(win64bitDebuggerFolder, "cdb.exe") - if not os.path.exists(cdbPath): - print() - print("WARNING: cdb.exe is not found - all crashes will be interesting.") - print() - return None - - if isWinDumpingToDefaultLocation(): - loops = 0 - maxLoops = 300 - while True: - if os.path.exists(dumpFilename): - debuggerCmdPath = getAbsPathForAdjacentFile("cdb_cmds.txt") - assert os.path.exists(debuggerCmdPath) - - cdbCmdList = [] - cdbCmdList.append("$<" + debuggerCmdPath) - - # See bug 902706 about -g. - return [cdbPath, "-g", "-c", ";".join(cdbCmdList), "-z", dumpFilename] - - time.sleep(0.200) - loops += 1 - if loops > maxLoops: - # Windows may take some time to generate the dump. - print("constructCdbCommand waited a long time, but %s never appeared!" % dumpFilename) - return None - else: - return None - - -def isWinDumpingToDefaultLocation(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc - # pylint: disable=too-complex,too-many-branches - """Check whether Windows minidumps are enabled and set to go to Windows' default location.""" - if sys.version_info.major == 2: - import _winreg as winreg # pylint: disable=import-error - else: - import winreg # pylint: disable=import-error - # For now, this code does not edit the Windows Registry because we tend to be in a 32-bit - # version of Python and if one types in regedit in the Run dialog, opens up the 64-bit registry. - # If writing a key, we most likely need to flush. For the moment, no keys are written. - try: - with winreg.OpenKey(winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE), - r"Software\Microsoft\Windows\Windows Error Reporting\LocalDumps", - # Read key from 64-bit registry, which also works for 32-bit - 0, (winreg.KEY_WOW64_64KEY + winreg.KEY_READ)) as key: - - try: - dumpTypeRegValue = winreg.QueryValueEx(key, "DumpType") - if not (dumpTypeRegValue[0] == 1 and dumpTypeRegValue[1] == winreg.REG_DWORD): - print(noMinidumpMsg) - return False - except WindowsError as e: # pylint: disable=undefined-variable - if e.errno == 2: - print(noMinidumpMsg) - return False - else: - raise - - try: - dumpFolderRegValue = winreg.QueryValueEx(key, "DumpFolder") - # %LOCALAPPDATA%\CrashDumps is the default location. - if not (dumpFolderRegValue[0] == r"%LOCALAPPDATA%\CrashDumps" and - dumpFolderRegValue[1] == winreg.REG_EXPAND_SZ): - print() - print("WARNING: Dumps are instead appearing at: %s - " - "all crashes will be uninteresting." % dumpFolderRegValue[0]) - print() - return False - except WindowsError as e: # pylint: disable=undefined-variable - # If the key value cannot be found, the dumps will be put in the default location - if e.errno == 2 and e.strerror == "The system cannot find the file specified": - return True - else: - raise - - return True - except WindowsError as e: # pylint: disable=undefined-variable - # If the LocalDumps registry key cannot be found, dumps will be put in the default location. - if e.errno == 2 and e.strerror == "The system cannot find the file specified": - print() - print("WARNING: The registry key HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\" - "Windows\\Windows Error Reporting\\LocalDumps cannot be found.") - print() - return None - else: - raise - - -def constructGdbCommand(progfullname, crashedPID): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Construct a command that uses the POSIX debugger (gdb) to turn a minidump file into a stack trace.""" - assert os.name == "posix" - # On Mac and Linux, look for a core file. - core_name = None - if platform.system() == "Darwin": - # Core files will be generated if you do: - # mkdir -p /cores/ - # ulimit -c 2147483648 (or call resource.setrlimit from a preexec_fn hook) - core_name = "/cores/core." + str(crashedPID) - elif platform.system() == "Linux": - is_pid_used = False - if os.path.exists("/proc/sys/kernel/core_uses_pid"): - with open("/proc/sys/kernel/core_uses_pid") as f: - is_pid_used = bool(int(f.read()[0])) # Setting [0] turns the input to a str. - core_name = "core." + str(crashedPID) if is_pid_used else "core" # relative path - if not os.path.isfile(core_name): - core_name = normExpUserPath(os.path.join("~", core_name)) # try the home dir - - if core_name and os.path.exists(core_name): - debuggerCmdPath = getAbsPathForAdjacentFile("gdb_cmds.txt") # pylint: disable=invalid-name - assert os.path.exists(debuggerCmdPath) - - # Run gdb and move the core file. Tip: gdb gives more info for: - # (debug with intact build dir > debug > opt with frame pointers > opt) - return ["gdb", "-n", "-batch", "-x", debuggerCmdPath, progfullname, core_name] - return None - - def getAbsPathForAdjacentFile(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Get the absolute path of a particular file, given its base directory and filename.""" return os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) -def rmTreeIncludingReadOnly(dirTree): # pylint: disable=invalid-name,missing-docstring - shutil.rmtree(dirTree, onerror=handleRemoveReadOnly if platform.system() == "Windows" else None) +def rm_tree_incl_readonly(dir_tree): + """Remove a directory tree including all read-only files. + + Args: + dir_tree (Path): Directory tree of files to be removed + """ + shutil.rmtree(str(dir_tree), onerror=handle_rm_readonly if platform.system() == "Windows" else None) + + +# This test needs updates for the move to pathlib, and needs to move to pytest +# def test_rm_tree_incl_readonly(): # pylint: disable=invalid-name +# """Run this function in the same directory as subprocesses to test.""" +# test_dir = "test_rm_tree_incl_readonly" +# os.mkdir(test_dir) +# read_only_dir = os.path.join(test_dir, "nestedReadOnlyDir") +# os.mkdir(read_only_dir) +# filename = os.path.join(read_only_dir, "test.txt") +# with open(filename, "w") as f: +# f.write("testing\n") +# os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) +# os.chmod(read_only_dir, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) -def test_rmTreeIncludingReadOnly(): # pylint: disable=invalid-name - """Run this function in the same directory as subprocesses to test.""" - test_dir = "test_rmTreeIncludingReadOnly" - os.mkdir(test_dir) - read_only_dir = os.path.join(test_dir, "nestedReadOnlyDir") - os.mkdir(read_only_dir) - filename = os.path.join(read_only_dir, "test.txt") - with open(filename, "w") as f: - f.write("testing\n") +# rm_tree_incl_readonly(test_dir) # Should pass here - os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - os.chmod(read_only_dir, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - rmTreeIncludingReadOnly(test_dir) # Should pass here +def handle_rm_readonly(func, path, exc): + """Handle read-only files. Adapted from http://stackoverflow.com/q/1213706 and some docs below adapted from + Python 2.7 official docs. + Args: + func (function): Function which raised the exception + path (str): Path name passed to function + exc (exception): Exception information returned by sys.exc_info() -def handleRemoveReadOnly(func, path, exc): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc - # pylint: disable=missing-type-doc - """Handle read-only files. Adapted from http://stackoverflow.com/q/1213706 .""" + Raises: + OSError: Raised if the read-only files are unable to be handled + """ if func in (os.rmdir, os.remove) and exc[1].errno == errno.EACCES: if os.name == "posix": # Ensure parent directory is also writeable. diff --git a/tests/js/test_build_options.py b/tests/js/test_build_options.py new file mode 100644 index 000000000..08774c66a --- /dev/null +++ b/tests/js/test_build_options.py @@ -0,0 +1,53 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import logging +import sys +import unittest + +from _pytest.monkeypatch import MonkeyPatch +import pytest + +from funfuzz.js import build_options + +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +def mock_chance(i): + """Overwrite the chance function to return True or False depending on a specific condition. + + Args: + i (float): Intended probability between 0 < i < 1 + + Returns: + bool: True if i > 0, False otherwise. + """ + return True if i > 0 else False + + +class BuildOptionsTests(unittest.TestCase): + """"TestCase class for functions in build_options.py""" + monkeypatch = MonkeyPatch() + trees_location = Path.home() / "trees" + + @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), + reason="requires a Mozilla Mercurial repository") + def test_get_random_valid_repo(self): + """Test that a valid repository can be obtained.""" + BuildOptionsTests.monkeypatch.setattr(build_options, "chance", mock_chance) + self.assertEqual(build_options.get_random_valid_repo(self.trees_location), + self.trees_location / "mozilla-central") diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index 2c8febb95..a5caee9fa 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -10,6 +10,7 @@ import logging import os +import platform import sys import unittest @@ -20,8 +21,10 @@ if sys.version_info.major == 2: from functools32 import lru_cache # pylint: disable=import-error + from pathlib2 import Path else: from functools import lru_cache # pylint: disable=no-name-in-module + from pathlib import Path # pylint: disable=import-error FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") logging.basicConfig(level=logging.DEBUG) @@ -30,15 +33,19 @@ class CompileShellTests(unittest.TestCase): """"TestCase class for functions in compile_shell.py""" + # Paths + mc_hg_repo = Path.home() / "trees" / "mozilla-central" + shell_cache = Path.home() / "shell-cache" + @pytest.mark.slow @lru_cache(maxsize=None) def test_shell_compile(self): """Test compilation of shells depending on the specified environment variable. Returns: - str: Path to the compiled shell. + Path: Path to the compiled shell. """ - self.assertTrue(os.path.isdir(os.path.join(os.path.expanduser("~"), "trees", "mozilla-central"))) + self.assertTrue(self.mc_hg_repo.is_dir()) # pylint: disable=no-member # Change the repository location by uncommenting this line and specifying the right one # "-R ~/trees/mozilla-central/") @@ -47,8 +54,8 @@ def test_shell_compile(self): # Remember to update the corresponding BUILD build parameters in .travis.yml as well build_opts = os.environ.get("BUILD", default_parameters_debug) - opts_parsed = js.build_options.parseShellOptions(build_opts) - hg_hash_of_default = util.hg_helpers.getRepoHashAndId(opts_parsed.repoDir)[0] + opts_parsed = js.build_options.parse_shell_opts(build_opts) + hg_hash_of_default = util.hg_helpers.get_repo_hash_and_id(opts_parsed.repo_dir)[0] # Ensure exit code is 0 self.assertTrue(not js.compile_shell.CompiledShell(opts_parsed, hg_hash_of_default).run(["-b", build_opts])) @@ -60,7 +67,9 @@ def test_shell_compile(self): # This set of builds should also have the following: 32-bit with ARM, with asan, and with clang file_name = "js-64-profDisabled-intlDisabled-linux-" + hg_hash_of_default - compiled_bin = os.path.join(os.path.expanduser("~"), "shell-cache", file_name, file_name) - self.assertTrue(os.path.isfile(compiled_bin)) + js_bin_path = self.shell_cache / file_name / file_name + if platform.system() == "Windows": + js_bin_path.with_suffix(".exe") + self.assertTrue(js_bin_path.is_file()) - return compiled_bin + return js_bin_path diff --git a/tests/js/test_link_fuzzer.py b/tests/js/test_link_fuzzer.py new file mode 100644 index 000000000..5bcf0be3e --- /dev/null +++ b/tests/js/test_link_fuzzer.py @@ -0,0 +1,46 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import logging +import sys +import unittest + +from funfuzz.js import link_fuzzer + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class LinkFuzzerTests(unittest.TestCase): + """"TestCase class for functions in link_fuzzer.py""" + def test_link_fuzzer(self): + """Test that a full jsfunfuzz file can be created.""" + with tempfile.TemporaryDirectory(suffix="link_fuzzer_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + jsfunfuzz_tmp = tmp_dir / "jsfunfuzz.js" + + link_fuzzer.link_fuzzer(jsfunfuzz_tmp) + + found = False + with open(str(jsfunfuzz_tmp)) as f: + for line in f: + if "It's looking good" in line: + found = True + break + + self.assertTrue(found) diff --git a/tests/util/test_fork_join.py b/tests/util/test_fork_join.py new file mode 100644 index 000000000..58e4deacd --- /dev/null +++ b/tests/util/test_fork_join.py @@ -0,0 +1,41 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import io +import logging +import sys +import unittest + +from funfuzz.util import fork_join + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class ForkJoinTests(unittest.TestCase): + """"TestCase class for functions in fork_join.py""" + def test_log_name(self): + """Test that incrementally numbered wtmp directories can be created""" + with tempfile.TemporaryDirectory(suffix="make_wtmp_dir_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + log_path = tmp_dir / "forkjoin-1-out.txt" + + with io.open(str(log_path), "w") as f: + f.writelines("test") + + self.assertEqual(fork_join.log_name(tmp_dir, 1, "out"), str(log_path)) diff --git a/tests/util/test_hg_helpers.py b/tests/util/test_hg_helpers.py index 2fccf731b..133bd9448 100644 --- a/tests/util/test_hg_helpers.py +++ b/tests/util/test_hg_helpers.py @@ -12,7 +12,14 @@ import sys import unittest -import funfuzz +import pytest + +from funfuzz.util import hg_helpers + +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") logging.basicConfig(level=logging.DEBUG) @@ -30,14 +37,22 @@ def assertRaisesRegex(self, *args, **kwds): # pylint: disable=arguments-differ, class HgHelpersTests(TestCase): """"TestCase class for functions in hg_helpers.py""" + trees_location = Path.home() / "trees" + def test_get_cset_hash_in_bisectmsg(self): """Test that we are able to extract the changeset hash from bisection output.""" - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("x 12345:abababababab"), "abababababab") - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("x 12345:123412341234"), "123412341234") - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("12345:abababababab y"), "abababababab") - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg( + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg("x 12345:abababababab"), "abababababab") + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg("x 12345:123412341234"), "123412341234") + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg("12345:abababababab y"), "abababababab") + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg( "Testing changeset 41831:4f4c01fb42c3 (2 changesets remaining, ~1 tests)"), "4f4c01fb42c3") with self.assertRaisesRegex(ValueError, (r"^Bisection output format required for hash extraction unavailable. " "The variable msg is:")): - funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("1a2345 - abababababab") + hg_helpers.get_cset_hash_from_bisect_msg("1a2345 - abababababab") + + @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), + reason="requires a Mozilla Mercurial repository") + def test_hgrc_repo_name(self): + """Test that we are able to extract the repository name from the hgrc file.""" + self.assertEqual(hg_helpers.hgrc_repo_name(self.trees_location / "mozilla-central"), "mozilla-central") diff --git a/tests/util/test_os_ops.py b/tests/util/test_os_ops.py new file mode 100644 index 000000000..f9426b794 --- /dev/null +++ b/tests/util/test_os_ops.py @@ -0,0 +1,50 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import logging +import sys +import unittest + +from funfuzz.util import os_ops + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class OsOpsTests(unittest.TestCase): + """"TestCase class for functions in os_ops.py""" + def test_make_wtmp_dir(self): + """Test that incrementally numbered wtmp directories can be created""" + with tempfile.TemporaryDirectory(suffix="make_wtmp_dir_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + + wtmp_dir_1 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_1.is_dir()) + self.assertTrue(wtmp_dir_1.name.endswith("1")) + + wtmp_dir_2 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_2.is_dir()) + self.assertTrue(wtmp_dir_2.name.endswith("2")) + + wtmp_dir_3 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_3.is_dir()) + self.assertTrue(wtmp_dir_3.name.endswith("3")) + + wtmp_dir_4 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_4.is_dir()) + self.assertTrue(wtmp_dir_4.name.endswith("4")) From 42865b604d750dd8140518751a56b9f3b834dbf5 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 9 May 2018 16:04:19 -0700 Subject: [PATCH 003/110] Ignore pylint "no-member" false positive error for now. --- src/funfuzz/bot.py | 1 + src/funfuzz/js/build_options.py | 2 +- src/funfuzz/js/compile_shell.py | 3 ++- src/funfuzz/js/link_fuzzer.py | 4 ++-- src/funfuzz/util/hg_helpers.py | 2 +- src/funfuzz/util/os_ops.py | 10 +++++----- tests/js/test_build_options.py | 1 + tests/util/test_hg_helpers.py | 1 + tests/util/test_os_ops.py | 8 ++++---- 9 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 4f4ef299a..40c5ac65d 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -85,6 +85,7 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur if args: print("Warning: bot does not use positional arguments") + # pylint: disable=no-member if not options.useTreeherderBuilds and not build_options.DEFAULT_TREES_LOCATION.is_dir(): # We don't have trees, so we must use treeherder builds. options.useTreeherderBuilds = True diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 99adec5df..dc76df261 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -188,7 +188,7 @@ def parse_shell_opts(args): print("WARNING: This set of build options is not tested well because: %s" % valid[1]) # Ensures releng machines do not enter the if block and assumes mozilla-central always exists - if DEFAULT_TREES_LOCATION.is_dir(): + if DEFAULT_TREES_LOCATION.is_dir(): # pylint: disable=no-member # Repositories do not get randomized if a repository is specified. if build_options.repo_dir is None: # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 44cc4e3a3..7fcff5d4f 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -9,6 +9,7 @@ # reset ; rg -g '!*subprocesses.py' -g '!*crashesat.py' -g '!*s3cache.py' -g '!*test_shell_flags.py' # -g '!*test_compile_shell.py' -g '!*known_broken*.py' -t py "import os$" +# disable no-member pylint messages can be removed after https://github.com/PyCQA/pylint/issues/1660 lands on 1.8 from __future__ import absolute_import, print_function, unicode_literals # isort:skip @@ -777,7 +778,7 @@ def verify_full_win_pageheap(shell_path): # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" - if gflags_bin_path.is_file() and shell_path.is_file(): + if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member print(subprocess.check_output([str(gflags_bin_path).decode("utf-8", errors="replace"), "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"])) diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index e017f3666..d0497f7d9 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -30,9 +30,9 @@ def link_fuzzer(target_path, prologue=""): if prologue: f.write(prologue) - for entry in (base_dir / "files_to_link.txt").read_text().split(): + for entry in (base_dir / "files_to_link.txt").read_text().split(): # pylint: disable=no-member entry = entry.rstrip() if entry and not entry.startswith("#"): file_path = base_dir / Path(entry) f.write("\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:]) - f.write(file_path.read_text()) + f.write(file_path.read_text()) # pylint: disable=no-member diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index c106f8370..670004ec7 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -43,7 +43,7 @@ def ensure_mq_enabled(): NoOptionError: Raises if an mq entry is not found in [extensions] """ user_hgrc = Path.home() / ".hgrc" - assert user_hgrc.is_file() + assert user_hgrc.is_file() # pylint: disable=no-member user_hgrc_cfg = configparser.SafeConfigParser() user_hgrc_cfg.read(str(user_hgrc)) diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index 3812a8db6..3de1a65b7 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -58,7 +58,7 @@ def make_cdb_cmd(prog_full_path, crashed_pid): # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. cdb_path = win64_debugging_folder / "cdb.exe" - if cdb_path.is_file(): + if cdb_path.is_file(): # pylint: disable=no-member print() print("WARNING: cdb.exe is not found - all crashes will be interesting.") print() @@ -70,7 +70,7 @@ def make_cdb_cmd(prog_full_path, crashed_pid): while True: if dump_name.is_file(): dbggr_cmd_path = Path(__file__).parent / "cdb_cmds.txt" - assert dbggr_cmd_path.is_file() + assert dbggr_cmd_path.is_file() # pylint: disable=no-member cdb_cmd_list = [] cdb_cmd_list.append("$<" + str(dbggr_cmd_path)) @@ -119,7 +119,7 @@ def make_gdb_cmd(prog_full_path, crashed_pid): if core_name and core_name_path.is_file(): dbggr_cmd_path = Path(__file__).parent / "gdb_cmds.txt" - assert dbggr_cmd_path.is_file() + assert dbggr_cmd_path.is_file() # pylint: disable=no-member # Run gdb and move the core file. Tip: gdb gives more info for: # (debug with intact build dir > debug > opt with frame pointers > opt) @@ -256,8 +256,8 @@ def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): reports_dir = base_dir / "Library" / "Logs" / "DiagnosticReports" # Find a crash log for the right process name and pid, preferring # newer crash logs (which sort last). - if reports_dir.is_dir(): - crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) + if reports_dir.is_dir(): # pylint: disable=no-member + crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) # pylint: disable=no-member else: crash_logs = [] diff --git a/tests/js/test_build_options.py b/tests/js/test_build_options.py index 08774c66a..33121a6c2 100644 --- a/tests/js/test_build_options.py +++ b/tests/js/test_build_options.py @@ -44,6 +44,7 @@ class BuildOptionsTests(unittest.TestCase): monkeypatch = MonkeyPatch() trees_location = Path.home() / "trees" + # pylint: disable=no-member @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), reason="requires a Mozilla Mercurial repository") def test_get_random_valid_repo(self): diff --git a/tests/util/test_hg_helpers.py b/tests/util/test_hg_helpers.py index 133bd9448..3d3228b0d 100644 --- a/tests/util/test_hg_helpers.py +++ b/tests/util/test_hg_helpers.py @@ -51,6 +51,7 @@ def test_get_cset_hash_in_bisectmsg(self): "The variable msg is:")): hg_helpers.get_cset_hash_from_bisect_msg("1a2345 - abababababab") + # pylint: disable=no-member @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), reason="requires a Mozilla Mercurial repository") def test_hgrc_repo_name(self): diff --git a/tests/util/test_os_ops.py b/tests/util/test_os_ops.py index f9426b794..929680c6c 100644 --- a/tests/util/test_os_ops.py +++ b/tests/util/test_os_ops.py @@ -34,17 +34,17 @@ def test_make_wtmp_dir(self): tmp_dir = Path(tmp_dir) wtmp_dir_1 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_1.is_dir()) + self.assertTrue(wtmp_dir_1.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_1.name.endswith("1")) wtmp_dir_2 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_2.is_dir()) + self.assertTrue(wtmp_dir_2.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_2.name.endswith("2")) wtmp_dir_3 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_3.is_dir()) + self.assertTrue(wtmp_dir_3.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_3.name.endswith("3")) wtmp_dir_4 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_4.is_dir()) + self.assertTrue(wtmp_dir_4.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_4.name.endswith("4")) From 19536cef5ff6c6ae3e295169c9dc9fc49ec6a7ea Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 11 May 2018 17:29:26 -0700 Subject: [PATCH 004/110] Fix #23 - Remove captureStdout and use subprocess32/subprocess from PyPI/stdlib throughout. --- src/funfuzz/autobisectjs/autobisectjs.py | 44 ++++++++-- .../known_broken_earliest_working.py | 3 +- src/funfuzz/bot.py | 26 ++++-- src/funfuzz/js/compare_jit.py | 12 ++- src/funfuzz/js/compile_shell.py | 38 +++++++-- src/funfuzz/js/inspect_shell.py | 33 +++++-- src/funfuzz/js/shell_flags.py | 4 +- src/funfuzz/util/fork_join.py | 4 +- src/funfuzz/util/hg_helpers.py | 82 +++++++++++++----- src/funfuzz/util/subprocesses.py | 85 ------------------- 10 files changed, 189 insertions(+), 142 deletions(-) diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index 32dd2b87a..2dad188b1 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -10,9 +10,9 @@ from __future__ import absolute_import, print_function # isort:skip from optparse import OptionParser # pylint: disable=deprecated-module +import os import re import shutil -import subprocess import sys import tempfile import time @@ -31,8 +31,11 @@ if sys.version_info.major == 2: from pathlib2 import Path + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error else: from pathlib import Path # pylint: disable=import-error + import subprocess def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -198,9 +201,15 @@ def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name, subprocess.check_call(hgPrefix + ["purge", "--all"]) # Reset bisect ranges and set skip ranges. - sps.captureStdout(hgPrefix + ["bisect", "-r"]) + subprocess.run(hgPrefix + ["bisect", "-r"], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + timeout=99) if options.skipRevs: - sps.captureStdout(hgPrefix + ["bisect", "--skip", options.skipRevs]) + subprocess.run(hgPrefix + ["bisect", "--skip", options.skipRevs], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + timeout=300) labels = {} # Specify `hg bisect` ranges. @@ -210,8 +219,14 @@ def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name, labels[sRepo] = ("good", "assumed start rev is good") labels[eRepo] = ("bad", "assumed end rev is bad") subprocess.check_call(hgPrefix + ["bisect", "-U", "-g", sRepo]) + mid_bisect_output = subprocess.run( + hgPrefix + ["bisect", "-U", "-b", eRepo], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=300).stdout.decode("utf-8", errors="replace") currRev = hg_helpers.get_cset_hash_from_bisect_msg( - sps.captureStdout(hgPrefix + ["bisect", "-U", "-b", eRepo])[0].split("\n")[0]) + mid_bisect_output.split("\n")) iterNum = 1 if options.testInitialRevs: @@ -261,7 +276,10 @@ def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name, subprocess.check_call(hgPrefix + ["bisect", "-U", "-r"]) sps.vdump("Resetting working directory") - sps.captureStdout(hgPrefix + ["update", "-C", "-r", "default"], ignoreStderr=True) + subprocess.run(hgPrefix + ["update", "-C", "-r", "default"], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + timeout=999) hg_helpers.destroyPyc(repo_dir) print_(time.asctime(), flush=True) @@ -340,8 +358,13 @@ def checkBlameParents(repo_dir, blamedRev, blamedGoodOrBad, labels, testRev, sta bisectLied = False missedCommonAncestor = False - parents = sps.captureStdout(["hg", "-R", repo_dir] + ["parent", "--template={node|short},", - "-r", blamedRev])[0].split(",")[:-1] + hg_parent_output = subprocess.run( + ["hg", "-R", repo_dir] + ["parent", "--template={node|short},", "-r", blamedRev], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=99).stdout.decode("utf-8", errors="replace") + parents = hg_parent_output.split(",")[:-1] if len(parents) == 1: return @@ -413,7 +436,12 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl # pylint: disable=too-many-arguments """Tell hg what we learned about the revision.""" assert hgLabel in ("good", "bad", "skip") - outputResult = sps.captureStdout(hgPrefix + ["bisect", "-U", "--" + hgLabel, currRev])[0] + outputResult = subprocess.run( + hgPrefix + ["bisect", "-U", "--" + hgLabel, currRev], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=999).stdout.decode("utf-8", errors="replace") outputLines = outputResult.split("\n") if options.build_options: diff --git a/src/funfuzz/autobisectjs/known_broken_earliest_working.py b/src/funfuzz/autobisectjs/known_broken_earliest_working.py index bd5d4c61e..04dfd5ed7 100644 --- a/src/funfuzz/autobisectjs/known_broken_earliest_working.py +++ b/src/funfuzz/autobisectjs/known_broken_earliest_working.py @@ -129,7 +129,8 @@ def earliest_known_working_rev(options, flags, skip_revs): # pylint: disable=mi # Note that the sed version check only works with GNU sed, not BSD sed found in macOS. if (platform.system() == "Linux" and parse_version(subprocess.run(["sed", "--version"], - stdout=subprocess.PIPE).stdout.split()[3]) >= parse_version("4.3")): + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace").split()[3]) + >= parse_version("4.3")): required.append("ebcbf47a83e7") # m-c 328765 Fx53, 1st w/ working builds using sed 4.3+ found on Ubuntu 17.04+ if options.disableProfiling: required.append("800a887c705e") # m-c 324836 Fx53, 1st w/ --disable-profiling, see bug 1321065 diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 40c5ac65d..97d0945b5 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -13,6 +13,7 @@ from builtins import object # pylint: disable=redefined-builtin import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module +import os import platform import shutil import sys @@ -31,7 +32,10 @@ if sys.version_info.major == 2: from pathlib2 import Path import psutil + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error else: + import subprocess from pathlib import Path # pylint: disable=import-error JS_SHELL_DEFAULT_TIMEOUT = 24 # see comments in loop for tradeoffs @@ -136,18 +140,24 @@ def main(): # pylint: disable=missing-docstring def printMachineInfo(): # pylint: disable=invalid-name """Log information about the machine.""" print("Platform details: %s" % " ".join(platform.uname())) - print("hg version: %s" % sps.captureStdout(["hg", "-q", "version"])[0]) - - # In here temporarily to see if mock Linux slaves on TBPL have gdb installed + print("hg version: %s" % + subprocess.run(["hg", "-q", "version"], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=9).stdout.rstrip()) try: - print("gdb version: %s" % sps.captureStdout(["gdb", "--version"], combineStderr=True, - ignoreStderr=True, ignoreExitCode=True)[0]) - except (KeyboardInterrupt, Exception) as ex: # pylint: disable=broad-except + print("gdb version: %s" % + subprocess.run( + ["gdb", "--version"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=9).stdout.decode("utf-8", errors="replace").split("\n")[0]) + except (KeyboardInterrupt, subprocess.CalledProcessError) as ex: print("Error involving gdb is: %r" % (ex,)) # FIXME: Should have if which(git) or something # pylint: disable=fixme - # print("git version: %s" % sps.captureStdout(["git", "--version"], combineStderr=True, - # ignoreStderr=True, ignoreExitCode=True)[0]) print("Python version: %s" % sys.version.split()[0]) print("Number of cores visible to OS: %d" % multiprocessing.cpu_count()) if sys.version_info.major == 2: diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 3a43081a5..ce9e0cbd5 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, print_function # isort:skip from optparse import OptionParser # pylint: disable=deprecated-module +import os import sys # These pylint errors exist because FuzzManager is not Python 3-compatible yet @@ -22,7 +23,11 @@ from . import shell_flags from ..util import create_collector from ..util import lithium_helpers -from ..util import subprocesses as sps + +if sys.version_info.major == 2 and os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + import subprocess if sys.version_info.major == 2: from pathlib2 import Path @@ -220,7 +225,10 @@ def diffFiles(f1, f2): # pylint: disable=invalid-name,missing-param-doc,missing """Return a command to diff two files, along with the diff output (if it's short).""" diffcmd = ["diff", "-u", f1, f2] s = " ".join(diffcmd) + "\n\n" # pylint: disable=invalid-name - diff = sps.captureStdout(diffcmd, ignoreExitCode=True)[0] + diff = subprocess.run(diffcmd, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=99).stdout.decode("utf-8", errors="replace") if len(diff) < 10000: s += diff + "\n\n" # pylint: disable=invalid-name else: diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 7fcff5d4f..3a9e175c4 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -521,9 +521,21 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ if "\\" in entry: entry = entry.replace("\\", "/") changed_cfg_cmds.append(entry) - sps.captureStdout(changed_cfg_cmds, ignoreStderr=True, currWorkingDir=str(shell.get_js_objdir()), env=cfg_env) + out = subprocess.run(changed_cfg_cmds, + check=True, + cwd=str(shell.get_js_objdir()), + env=cfg_env, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") else: - sps.captureStdout(cfg_cmds, ignoreStderr=True, currWorkingDir=str(shell.get_js_objdir()), env=cfg_env) + out = subprocess.run(cfg_cmds, + check=True, + cwd=str(shell.get_js_objdir()), + env=cfg_env, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + + # We could save the stdout here into a file if it throws shell.setEnvAdded(env_vars) shell.setEnvFull(cfg_env) @@ -534,16 +546,22 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- """Compile and copy a binary.""" try: cmd_list = [MAKE_BINARY, "-C", str(shell.get_js_objdir()), "-j" + str(COMPILATION_JOBS), "-s"] - out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, - currWorkingDir=str(shell.get_js_objdir()), env=shell.getEnvFull())[0] + out = subprocess.run(cmd_list, + cwd=str(shell.get_js_objdir()), + env=shell.getEnvFull(), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") except Exception as ex: # pylint: disable=broad-except # This exception message is returned from sps.captureStdout via cmd_list. if (platform.system() == "Linux" or platform.system() == "Darwin") and \ ("GCC running out of memory" in repr(ex) or "Clang running out of memory" in repr(ex)): # FIXME: Absolute hack to retry after hitting OOM. # pylint: disable=fixme print("Trying once more due to the compiler running out of memory...") - out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, - currWorkingDir=shell.get_js_objdir(), env=shell.getEnvFull())[0] + out = subprocess.run(cmd_list, + cwd=str(shell.get_js_objdir()), + env=shell.getEnvFull(), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") # A non-zero error can be returned during make, but eventually a shell still gets compiled. if shell.get_shell_compiled_path().is_file(): print("A shell was compiled even though there was a non-zero exit code. Continuing...") @@ -551,6 +569,8 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) raise + # We could save the stdout here into a file if it throws + if shell.get_shell_compiled_path().is_file(): shutil.copy2(str(shell.get_shell_compiled_path()), str(shell.get_shell_cache_js_bin_path())) for run_lib in shell.get_shell_compiled_runlibs_path(): @@ -766,7 +786,11 @@ def updateRepo(repo, rev): # pylint: disable=invalid-name,missing-param-doc,mis # Print *with* a trailing newline to avoid breaking other stuff print("Updating to rev %s in the %s repository..." % (rev.decode("utf-8", errors="replace"), repo.decode("utf-8", errors="replace"))) - sps.captureStdout(["hg", "-R", repo, "update", "-C", "-r", rev], ignoreStderr=True) + subprocess.run(["hg", "-R", repo, "update", "-C", "-r", rev], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.DEVNULL, + timeout=999) def verify_full_win_pageheap(shell_path): diff --git a/src/funfuzz/js/inspect_shell.py b/src/funfuzz/js/inspect_shell.py index 64d96eba0..61902f4be 100644 --- a/src/funfuzz/js/inspect_shell.py +++ b/src/funfuzz/js/inspect_shell.py @@ -9,13 +9,22 @@ from __future__ import absolute_import, print_function # isort:skip +import json +import os import platform +import sys from lithium.interestingness.utils import env_with_path from shellescape import quote from ..util import subprocesses as sps +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + import subprocess + RUN_NSPR_LIB = "" RUN_PLDS_LIB = "" RUN_PLC_LIB = "" @@ -77,8 +86,12 @@ def archOfBinary(binary): # pylint: disable=inconsistent-return-statements,inva # pylint: disable=missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc """Test if a binary is 32-bit or 64-bit.""" # We can possibly use the python-magic-bin PyPI library in the future - unsplit_file_type = sps.captureStdout(["file", str(binary)])[0] - filetype = unsplit_file_type.decode("utf-8", errors="replace").split(":", 1)[1] + unsplit_file_type = subprocess.run( + ["file", str(binary)], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=99).stdout.decode("utf-8", errors="replace") + filetype = unsplit_file_type.split(":", 1)[1] if platform.system() == "Windows": assert "MS Windows" in filetype return "32" if "Intel 80386 32-bit" in filetype else "64" @@ -134,8 +147,14 @@ def testBinary(shellPath, args, useValgrind): # pylint: disable=invalid-name,mi """Test the given shell with the given args.""" test_cmd = (constructVgCmdList() if useValgrind else []) + [str(shellPath)] + args sps.vdump("The testing command is: " + " ".join(quote(x) for x in test_cmd)) - out, return_code = sps.captureStdout(test_cmd, combineStderr=True, ignoreStderr=True, - ignoreExitCode=True, env=env_with_path(str(shellPath.parent))) + test_cmd_result = subprocess.run( + test_cmd, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + env=env_with_path(str(shellPath.parent)), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=999) + out, return_code = test_cmd_result.stdout.decode("utf-8", errors="replace"), test_cmd_result.returncode sps.vdump("The exit code is: " + str(return_code)) return out, return_code @@ -149,9 +168,9 @@ def testJsShellOrXpcshell(s): # pylint: disable=invalid-name,missing-param-doc, def queryBuildConfiguration(s, parameter): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Test if a binary is compiled with specified parameters, in getBuildConfiguration().""" - ans = testBinary(s, ["-e", 'print(getBuildConfiguration()["' + parameter + '"])'], - False)[0] - return ans.decode("utf-8", errors="replace").find("true") != -1 + return json.loads(testBinary(s, + ["-e", 'print(getBuildConfiguration()["' + parameter + '"])'], + False)[0].rstrip().lower()) def verifyBinary(sh): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc diff --git a/src/funfuzz/js/shell_flags.py b/src/funfuzz/js/shell_flags.py index 95580015b..538cf04ad 100644 --- a/src/funfuzz/js/shell_flags.py +++ b/src/funfuzz/js/shell_flags.py @@ -33,9 +33,7 @@ def shell_supports_flag(shell_path, flag): Returns: bool: True if the flag is supported, i.e. does not cause the shell to throw an error, False otherwise. """ - dummy_parameters = ["-e", "42"] - # This can be refactored when sps.captureStdout is gone - out = inspect_shell.shellSupports(shell_path, [flag] + dummy_parameters) + out = inspect_shell.shellSupports(shell_path, [flag, "-e", "42"]) return out diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index fa7d76631..0c4544538 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -70,8 +70,8 @@ def log_name(log_dir, i, log_type): def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring - sys.stdout = open(log_name(logDir, i, "out"), "w", buffering=0) - sys.stderr = open(log_name(logDir, i, "err"), "w", buffering=0) + sys.stdout = open(log_name(logDir, i, "out"), "w", buffering=0) # I WONDER .decode("utf-8", errors="replace") + sys.stderr = open(log_name(logDir, i, "err"), "w", buffering=0) # I WONDER .decode("utf-8", errors="replace") fun(*(someArgs + (i,))) diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index 670004ec7..b19041edb 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -13,15 +13,17 @@ import configparser import os import re -import subprocess import sys from . import subprocesses as sps if sys.version_info.major == 2: from pathlib2 import Path + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error else: from pathlib import Path # pylint: disable=import-error + import subprocess def destroyPyc(repo_dir): # pylint: disable=invalid-name,missing-docstring @@ -57,24 +59,39 @@ def ensure_mq_enabled(): def findCommonAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - return sps.captureStdout(["hg", "-R", str(repo_dir), "log", "-r", "ancestor(" + a + "," + b + ")", - "--template={node|short}"])[0] + return subprocess.run( + ["hg", "-R", str(repo_dir), "log", "-r", "ancestor(" + a + "," + b + ")", "--template={node|short}"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=999 + ).stdout.decode("utf-8", errors="replace") def isAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist.""" - return sps.captureStdout(["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", - "--template={node|short}"])[0] != "" + return subprocess.run( + ["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=999 + ).stdout.decode("utf-8", errors="replace") != "" def existsAndIsAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Return true iff |a| exists and is an ancestor of |b|.""" # Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error - out = sps.captureStdout(["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", - "--template={node|short}"], combineStderr=True, ignoreExitCode=True)[0] - return out != "" and out.decode("utf-8", errors="replace").find("abort: unknown revision") < 0 + out = subprocess.run( + ["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=999 + ).stdout.decode("utf-8", errors="replace") + return out != "" and out.find("abort: unknown revision") < 0 def get_cset_hash_from_bisect_msg(msg): @@ -114,7 +131,13 @@ def get_repo_hash_and_id(repo_dir, repo_rev="parents() and default"): # This returns null if the repository is not on default. hg_log_template_cmds = ["hg", "-R", str(repo_dir), "log", "-r", repo_rev, "--template", "{node|short} {rev}"] - hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] + hg_id_full = subprocess.run( + hg_log_template_cmds, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=99 + ).stdout.decode("utf-8", errors="replace") is_on_default = bool(hg_id_full) if not is_on_default: update_default = input("Not on default tip! " @@ -131,9 +154,15 @@ def get_repo_hash_and_id(repo_dir, repo_rev="parents() and default"): "{node|short} {rev}"] else: raise ValueError("Invalid choice.") - hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] + hg_id_full = subprocess.run( + hg_log_template_cmds, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=99 + ).stdout.decode("utf-8", errors="replace") assert hg_id_full != "" - (hg_id_hash, hg_id_local_num) = hg_id_full.decode("utf-8", errors="replace").split(" ") + (hg_id_hash, hg_id_local_num) = hg_id_full.split(" ") sps.vdump("Finished getting the hash and local id number of the repository.") return hg_id_hash, hg_id_local_num, is_on_default @@ -171,9 +200,14 @@ def patch_hg_repo_with_mq(patch_file, repo_dir=None): # We may have passed in the patch with or without the full directory. patch_abs_path = patch_file.resolve() pname = patch_abs_path.name - qimport_output, qimport_return_code = sps.captureStdout(["hg", "-R", str(repo_dir), "qimport", str(patch_abs_path)], - combineStderr=True, ignoreStderr=True, - ignoreExitCode=True) + qimport_result = subprocess.run( + ["hg", "-R", str(repo_dir), "qimport", patch_abs_path], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=99) + qimport_output, qimport_return_code = (qimport_result.stdout.decode("utf-8", errors="replace"), + qimport_result.returncode) if qimport_return_code != 0: if "already exists" in qimport_output: print("A patch with the same name has already been qpush'ed. Please qremove it first.") @@ -181,8 +215,14 @@ def patch_hg_repo_with_mq(patch_file, repo_dir=None): print("Patch qimport'ed...", end=" ") - qpush_output, qpush_return_code = sps.captureStdout(["hg", "-R", str(repo_dir), "qpush", pname], - combineStderr=True, ignoreStderr=True) + qpush_result = subprocess.run( + ["hg", "-R", str(repo_dir), "qpush", pname], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=99) + qpush_output, qpush_return_code = qpush_result.stdout.decode("utf-8", errors="replace"), qpush_result.returncode assert " is empty" not in qpush_output, "Patch to be qpush'ed should not be empty." if qpush_return_code != 0: @@ -207,9 +247,13 @@ def qpop_qrm_applied_patch(patch_file, repo_dir): Raises: OSError: Raises when `hg qpop` did not return a return code of 0 """ - qpop_output, qpop_return_code = sps.captureStdout(["hg", "-R", str(repo_dir), "qpop"], - combineStderr=True, ignoreStderr=True, - ignoreExitCode=True) + qpop_result = subprocess.run( + ["hg", "-R", str(repo_dir), "qpop"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=99) + qpop_output, qpop_return_code = qpop_result.stdout.decode("utf-8", errors="replace"), qpop_result.returncode if qpop_return_code != 0: print("`hg qpop` output is: " + qpop_output) raise OSError("Return code from `hg qpop` is: " + str(qpop_return_code)) diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index f20bbcd08..5b88b2f70 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -14,95 +14,10 @@ import platform import shutil import stat -import subprocess -import sys - -from shellescape import quote verbose = False # pylint: disable=invalid-name -# pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc -# pylint: disable=missing-type-doc,too-complex,too-many-arguments,too-many-branches,too-many-statements -def captureStdout(inputCmd, ignoreStderr=False, combineStderr=False, ignoreExitCode=False, currWorkingDir=None, - env="NOTSET", verbosity=False): - """Capture standard output, return the output as a string, along with the return value.""" - currWorkingDir = str(currWorkingDir) or ( - os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member - if env == "NOTSET": - vdump(" ".join(quote(x) for x in inputCmd)) - env = os.environ - else: - # There is no way yet to only print the environment variables that were added by the harness - # We could dump all of os.environ but it is too much verbose output. - vdump("ENV_VARIABLES_WERE_ADDED_HERE " + " ".join(quote(x) for x in inputCmd)) - cmd = [] - for el in inputCmd: - if el.startswith('"') and el.endswith('"'): - cmd.append(str(el[1:-1])) - else: - cmd.append(str(el)) - assert cmd != [] - try: - p = subprocess.Popen( - cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT if combineStderr else subprocess.PIPE, - cwd=currWorkingDir, - env=env) - (stdout, stderr) = p.communicate() - except OSError as e: - raise Exception(repr(e.strerror) + " error calling: " + " ".join(quote(x) for x in cmd)) - if p.returncode != 0: - oomErrorOutput = stdout if combineStderr else stderr - if (platform.system() == "Linux" or platform.system() == "Darwin") and oomErrorOutput: - if "internal compiler error: Killed (program cc1plus)" in oomErrorOutput: - raise Exception("GCC running out of memory") - elif "error: unable to execute command: Killed" in oomErrorOutput: - raise Exception("Clang running out of memory") - if not ignoreExitCode: - # Potential problem area: Note that having a non-zero exit code does not mean that the - # operation did not succeed, for example when compiling a shell. A non-zero exit code - # can appear even though a shell compiled successfully. - # Pymake in builds earlier than revision 232553f741a0 did not support the "-s" option. - if "no such option: -s" not in stdout: - print("Nonzero exit code from: ") - print(" %s" % " ".join(quote(x) for x in cmd)) - print("stdout is:") - print(stdout) - if stderr is not None: - print("stderr is:") - print(stderr) - # Pymake in builds earlier than revision 232553f741a0 did not support the "-s" option. - if "hg pull: option --rebase not recognized" not in stdout and "no such option: -s" not in stdout: - if platform.system() == "Windows" and stderr and "Permission denied" in stderr and \ - "configure: error: installation or configuration problem: " + \ - "C++ compiler cannot create executables." in stderr: - raise Exception("Windows conftest.exe configuration permission problem") - else: - raise Exception("Nonzero exit code") - if not combineStderr and not ignoreStderr and stderr: - # Ignore hg color mode throwing an error in console on Windows platforms. - if not (platform.system() == "Windows" and "warning: failed to set color mode to win32" in stderr): - print("Unexpected output on stderr from: ") - print(" %s" % " ".join(quote(x) for x in cmd)) - print("%s %s" % (stdout, stderr)) - raise Exception("Unexpected output on stderr") - if stderr and ignoreStderr and stderr and p.returncode != 0: - # During configure, there will always be stderr. Sometimes this stderr causes configure to - # stop the entire script, especially on Windows. - print("Return code not zero, and unexpected output on stderr from: ") - print(" %s" % " ".join(quote(x) for x in cmd)) - print("%s %s" % (stdout, stderr)) - raise Exception("Return code not zero, and unexpected output on stderr") - if verbose or verbosity: - print(stdout) - if stderr is not None: - print(stderr) - return stdout.rstrip(), p.returncode - - def getAbsPathForAdjacentFile(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Get the absolute path of a particular file, given its base directory and filename.""" From 0f3ada012e320218a3c0a8911b34612caaab1c1e Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 11 May 2018 18:34:30 -0700 Subject: [PATCH 005/110] Remove getAbsPathForAdjacentFile and normExpUserPath. --- src/funfuzz/util/subprocesses.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index 5b88b2f70..d63c9f1fc 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -18,12 +18,6 @@ verbose = False # pylint: disable=invalid-name -def getAbsPathForAdjacentFile(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Get the absolute path of a particular file, given its base directory and filename.""" - return os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) - - def rm_tree_incl_readonly(dir_tree): """Remove a directory tree including all read-only files. @@ -75,10 +69,6 @@ def handle_rm_readonly(func, path, exc): raise OSError("Unable to handle read-only files.") -def normExpUserPath(p): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return os.path.normpath(os.path.expanduser(p)) - - def vdump(inp): # pylint: disable=missing-param-doc,missing-type-doc """Append the word "DEBUG" to any verbose output.""" if verbose: From 38db227cbdc48feb44aaf51155823173ff63807e Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 11 May 2018 19:09:28 -0700 Subject: [PATCH 006/110] Remove unused imports. --- src/funfuzz/bot.py | 1 - src/funfuzz/util/repos_update.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 97d0945b5..d8b7a52dd 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -26,7 +26,6 @@ from .util import create_collector from .util import fork_join from .util import hg_helpers -from .util import subprocesses as sps from .util.lock_dir import LockDir if sys.version_info.major == 2: diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index a7c875bb6..c4f244bd3 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -19,8 +19,6 @@ import sys import time -from . import subprocesses as sps - if sys.version_info.major == 2: if os.name == "posix": import subprocess32 as subprocess # pylint: disable=import-error From c7da6926355c70a789b791feee76ca999d41307f Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 11 May 2018 19:09:49 -0700 Subject: [PATCH 007/110] subprocess-related follow-up fix. --- src/funfuzz/js/compile_shell.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 3a9e175c4..4d551cdd0 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -521,19 +521,19 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ if "\\" in entry: entry = entry.replace("\\", "/") changed_cfg_cmds.append(entry) - out = subprocess.run(changed_cfg_cmds, - check=True, - cwd=str(shell.get_js_objdir()), - env=cfg_env, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + subprocess.run(changed_cfg_cmds, + check=True, + cwd=str(shell.get_js_objdir()), + env=cfg_env, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") else: - out = subprocess.run(cfg_cmds, - check=True, - cwd=str(shell.get_js_objdir()), - env=cfg_env, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + subprocess.run(cfg_cmds, + check=True, + cwd=str(shell.get_js_objdir()), + env=cfg_env, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") # We could save the stdout here into a file if it throws From a48cfa5bcffadfe3b944cdb48bd5ee4542d36bd3 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 11 May 2018 19:10:00 -0700 Subject: [PATCH 008/110] pathlib-related follow-up fix. --- src/funfuzz/util/repos_update.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index c4f244bd3..363d57952 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -20,9 +20,11 @@ import time if sys.version_info.major == 2: + from pathlib2 import Path if os.name == "posix": import subprocess32 as subprocess # pylint: disable=import-error else: + from pathlib import Path # pylint: disable=import-error import subprocess @@ -38,9 +40,9 @@ git_64bit_path = Path(os.getenv("PROGRAMFILES")) / "Git" / "bin" / "git.exe" git_32bit_path = Path(os.getenv("PROGRAMFILES(X86)")) / "Git" / "bin" / "git.exe" if git_64bit_path.is_file(): - GITBINARY = git_64bit_path + GITBINARY = str(git_64bit_path) elif git_32bit_path.is_file(): - GITBINARY = git_32bit_path + GITBINARY = str(git_32bit_path) else: raise OSError("Git binary not found") else: From 7bf932fbc31fea5043436d7c936bd70cde19d621 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 12 May 2018 00:13:08 -0700 Subject: [PATCH 009/110] Ignore more pylint "no-member" false positives. --- src/funfuzz/util/create_collector.py | 2 +- src/funfuzz/util/repos_update.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/funfuzz/util/create_collector.py b/src/funfuzz/util/create_collector.py index 104954b4b..d6011c70d 100644 --- a/src/funfuzz/util/create_collector.py +++ b/src/funfuzz/util/create_collector.py @@ -26,7 +26,7 @@ def make_collector(): Collector: jsfunfuzz collector object """ sigcache_path = Path.home() / "sigcache" - sigcache_path.mkdir(exist_ok=True) + sigcache_path.mkdir(exist_ok=True) # pylint: disable=no-member return Collector(sigCacheDir=str(sigcache_path), tool="jsfunfuzz") diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index 363d57952..ed53379a3 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -39,9 +39,9 @@ # pylint: disable=invalid-name git_64bit_path = Path(os.getenv("PROGRAMFILES")) / "Git" / "bin" / "git.exe" git_32bit_path = Path(os.getenv("PROGRAMFILES(X86)")) / "Git" / "bin" / "git.exe" - if git_64bit_path.is_file(): + if git_64bit_path.is_file(): # pylint: disable=no-member GITBINARY = str(git_64bit_path) - elif git_32bit_path.is_file(): + elif git_32bit_path.is_file(): # pylint: disable=no-member GITBINARY = str(git_32bit_path) else: raise OSError("Git binary not found") From 934bdb4903d14f8c13f12d9e6fb49217e47dba29 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 25 May 2018 14:23:28 -0700 Subject: [PATCH 010/110] Add comment related to is_win_dumping_to_default raising WindowsError in some circumstances. --- src/funfuzz/util/os_ops.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index 3de1a65b7..3c9795846 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -291,6 +291,10 @@ def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branches """Check whether Windows minidumps are enabled and set to go to Windows' default location. + Raises: + WindowsError: Raises if querying for the DumpType key throws and it is unrelated to various issues, + e.g. the key not being present. + Returns: bool: Returns True when Windows has dumping enabled, and is dumping to the default location, otherwise False """ From e1f9bd93a497df3a193a17c1e24dc0ff99f47c52 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 25 May 2018 14:37:23 -0700 Subject: [PATCH 011/110] In Python 3, winreg functions usually throw OSError instead of WindowsError. --- src/funfuzz/util/os_ops.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index 3c9795846..9507e63ad 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -292,7 +292,9 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche """Check whether Windows minidumps are enabled and set to go to Windows' default location. Raises: - WindowsError: Raises if querying for the DumpType key throws and it is unrelated to various issues, + OSError: Raises if querying for the DumpType key throws and it is unrelated to various issues in Python 3, + e.g. the key not being present. + WindowsError: Raises if querying for the DumpType key throws and it is unrelated to various issues in Python 2, e.g. the key not being present. Returns: @@ -316,7 +318,7 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche if not (dump_type_reg_value[0] == 1 and dump_type_reg_value[1] == winreg.REG_DWORD): print(NO_DUMP_MSG) return False - except WindowsError as ex: # pylint: disable=undefined-variable + except (OSError, WindowsError) as ex: # pylint: disable=undefined-variable if ex.errno == 2: print(NO_DUMP_MSG) return False @@ -333,7 +335,7 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche "all crashes will be uninteresting." % dump_folder_reg_value[0]) print() return False - except WindowsError as ex: # pylint: disable=undefined-variable + except (OSError, WindowsError) as ex: # pylint: disable=undefined-variable # If the key value cannot be found, the dumps will be put in the default location if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": return True @@ -341,7 +343,7 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche raise return True - except WindowsError as ex: # pylint: disable=undefined-variable + except (OSError, WindowsError) as ex: # pylint: disable=undefined-variable # If the LocalDumps registry key cannot be found, dumps will be put in the default location. if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": print() From 9d49f0f486b6b910bcb1a715f0dc066f3fd39f15 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 25 May 2018 15:05:28 -0700 Subject: [PATCH 012/110] Oops, WindowsError is a subclass of OSError. --- src/funfuzz/util/os_ops.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index 9507e63ad..c0eb8587a 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -292,10 +292,8 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche """Check whether Windows minidumps are enabled and set to go to Windows' default location. Raises: - OSError: Raises if querying for the DumpType key throws and it is unrelated to various issues in Python 3, + OSError: Raises if querying for the DumpType key throws and it is unrelated to various issues, e.g. the key not being present. - WindowsError: Raises if querying for the DumpType key throws and it is unrelated to various issues in Python 2, - e.g. the key not being present. Returns: bool: Returns True when Windows has dumping enabled, and is dumping to the default location, otherwise False @@ -318,7 +316,7 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche if not (dump_type_reg_value[0] == 1 and dump_type_reg_value[1] == winreg.REG_DWORD): print(NO_DUMP_MSG) return False - except (OSError, WindowsError) as ex: # pylint: disable=undefined-variable + except OSError as ex: if ex.errno == 2: print(NO_DUMP_MSG) return False @@ -335,7 +333,7 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche "all crashes will be uninteresting." % dump_folder_reg_value[0]) print() return False - except (OSError, WindowsError) as ex: # pylint: disable=undefined-variable + except OSError as ex: # If the key value cannot be found, the dumps will be put in the default location if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": return True @@ -343,7 +341,7 @@ def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branche raise return True - except (OSError, WindowsError) as ex: # pylint: disable=undefined-variable + except OSError as ex: # If the LocalDumps registry key cannot be found, dumps will be put in the default location. if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": print() From de4d39816f27c805124bf4fcc39a16bd29caa787 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 25 May 2018 17:54:55 -0700 Subject: [PATCH 013/110] Revert "Ignore more pylint "no-member" false positives." This reverts commit 7bf932fbc31fea5043436d7c936bd70cde19d621. --- src/funfuzz/util/create_collector.py | 2 +- src/funfuzz/util/repos_update.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/funfuzz/util/create_collector.py b/src/funfuzz/util/create_collector.py index d6011c70d..104954b4b 100644 --- a/src/funfuzz/util/create_collector.py +++ b/src/funfuzz/util/create_collector.py @@ -26,7 +26,7 @@ def make_collector(): Collector: jsfunfuzz collector object """ sigcache_path = Path.home() / "sigcache" - sigcache_path.mkdir(exist_ok=True) # pylint: disable=no-member + sigcache_path.mkdir(exist_ok=True) return Collector(sigCacheDir=str(sigcache_path), tool="jsfunfuzz") diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index ed53379a3..363d57952 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -39,9 +39,9 @@ # pylint: disable=invalid-name git_64bit_path = Path(os.getenv("PROGRAMFILES")) / "Git" / "bin" / "git.exe" git_32bit_path = Path(os.getenv("PROGRAMFILES(X86)")) / "Git" / "bin" / "git.exe" - if git_64bit_path.is_file(): # pylint: disable=no-member + if git_64bit_path.is_file(): GITBINARY = str(git_64bit_path) - elif git_32bit_path.is_file(): # pylint: disable=no-member + elif git_32bit_path.is_file(): GITBINARY = str(git_32bit_path) else: raise OSError("Git binary not found") From 8e02d4e660a0780d7d6889765b0ba84ffa502719 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 25 May 2018 17:55:20 -0700 Subject: [PATCH 014/110] Revert "Ignore pylint "no-member" false positive error for now." This reverts commit 42865b604d750dd8140518751a56b9f3b834dbf5. --- src/funfuzz/bot.py | 1 - src/funfuzz/js/build_options.py | 2 +- src/funfuzz/js/compile_shell.py | 3 +-- src/funfuzz/js/link_fuzzer.py | 4 ++-- src/funfuzz/util/hg_helpers.py | 2 +- src/funfuzz/util/os_ops.py | 10 +++++----- tests/js/test_build_options.py | 1 - tests/util/test_hg_helpers.py | 1 - tests/util/test_os_ops.py | 8 ++++---- 9 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index d8b7a52dd..f50769fe2 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -88,7 +88,6 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur if args: print("Warning: bot does not use positional arguments") - # pylint: disable=no-member if not options.useTreeherderBuilds and not build_options.DEFAULT_TREES_LOCATION.is_dir(): # We don't have trees, so we must use treeherder builds. options.useTreeherderBuilds = True diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index dc76df261..99adec5df 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -188,7 +188,7 @@ def parse_shell_opts(args): print("WARNING: This set of build options is not tested well because: %s" % valid[1]) # Ensures releng machines do not enter the if block and assumes mozilla-central always exists - if DEFAULT_TREES_LOCATION.is_dir(): # pylint: disable=no-member + if DEFAULT_TREES_LOCATION.is_dir(): # Repositories do not get randomized if a repository is specified. if build_options.repo_dir is None: # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 4d551cdd0..43e7c5a4d 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -9,7 +9,6 @@ # reset ; rg -g '!*subprocesses.py' -g '!*crashesat.py' -g '!*s3cache.py' -g '!*test_shell_flags.py' # -g '!*test_compile_shell.py' -g '!*known_broken*.py' -t py "import os$" -# disable no-member pylint messages can be removed after https://github.com/PyCQA/pylint/issues/1660 lands on 1.8 from __future__ import absolute_import, print_function, unicode_literals # isort:skip @@ -802,7 +801,7 @@ def verify_full_win_pageheap(shell_path): # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" - if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member + if gflags_bin_path.is_file() and shell_path.is_file(): print(subprocess.check_output([str(gflags_bin_path).decode("utf-8", errors="replace"), "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"])) diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index d0497f7d9..e017f3666 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -30,9 +30,9 @@ def link_fuzzer(target_path, prologue=""): if prologue: f.write(prologue) - for entry in (base_dir / "files_to_link.txt").read_text().split(): # pylint: disable=no-member + for entry in (base_dir / "files_to_link.txt").read_text().split(): entry = entry.rstrip() if entry and not entry.startswith("#"): file_path = base_dir / Path(entry) f.write("\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:]) - f.write(file_path.read_text()) # pylint: disable=no-member + f.write(file_path.read_text()) diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index b19041edb..64459494b 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -45,7 +45,7 @@ def ensure_mq_enabled(): NoOptionError: Raises if an mq entry is not found in [extensions] """ user_hgrc = Path.home() / ".hgrc" - assert user_hgrc.is_file() # pylint: disable=no-member + assert user_hgrc.is_file() user_hgrc_cfg = configparser.SafeConfigParser() user_hgrc_cfg.read(str(user_hgrc)) diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index c0eb8587a..4bf0564b4 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -58,7 +58,7 @@ def make_cdb_cmd(prog_full_path, crashed_pid): # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. cdb_path = win64_debugging_folder / "cdb.exe" - if cdb_path.is_file(): # pylint: disable=no-member + if cdb_path.is_file(): print() print("WARNING: cdb.exe is not found - all crashes will be interesting.") print() @@ -70,7 +70,7 @@ def make_cdb_cmd(prog_full_path, crashed_pid): while True: if dump_name.is_file(): dbggr_cmd_path = Path(__file__).parent / "cdb_cmds.txt" - assert dbggr_cmd_path.is_file() # pylint: disable=no-member + assert dbggr_cmd_path.is_file() cdb_cmd_list = [] cdb_cmd_list.append("$<" + str(dbggr_cmd_path)) @@ -119,7 +119,7 @@ def make_gdb_cmd(prog_full_path, crashed_pid): if core_name and core_name_path.is_file(): dbggr_cmd_path = Path(__file__).parent / "gdb_cmds.txt" - assert dbggr_cmd_path.is_file() # pylint: disable=no-member + assert dbggr_cmd_path.is_file() # Run gdb and move the core file. Tip: gdb gives more info for: # (debug with intact build dir > debug > opt with frame pointers > opt) @@ -256,8 +256,8 @@ def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): reports_dir = base_dir / "Library" / "Logs" / "DiagnosticReports" # Find a crash log for the right process name and pid, preferring # newer crash logs (which sort last). - if reports_dir.is_dir(): # pylint: disable=no-member - crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) # pylint: disable=no-member + if reports_dir.is_dir(): + crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) else: crash_logs = [] diff --git a/tests/js/test_build_options.py b/tests/js/test_build_options.py index 33121a6c2..08774c66a 100644 --- a/tests/js/test_build_options.py +++ b/tests/js/test_build_options.py @@ -44,7 +44,6 @@ class BuildOptionsTests(unittest.TestCase): monkeypatch = MonkeyPatch() trees_location = Path.home() / "trees" - # pylint: disable=no-member @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), reason="requires a Mozilla Mercurial repository") def test_get_random_valid_repo(self): diff --git a/tests/util/test_hg_helpers.py b/tests/util/test_hg_helpers.py index 3d3228b0d..133bd9448 100644 --- a/tests/util/test_hg_helpers.py +++ b/tests/util/test_hg_helpers.py @@ -51,7 +51,6 @@ def test_get_cset_hash_in_bisectmsg(self): "The variable msg is:")): hg_helpers.get_cset_hash_from_bisect_msg("1a2345 - abababababab") - # pylint: disable=no-member @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), reason="requires a Mozilla Mercurial repository") def test_hgrc_repo_name(self): diff --git a/tests/util/test_os_ops.py b/tests/util/test_os_ops.py index 929680c6c..f9426b794 100644 --- a/tests/util/test_os_ops.py +++ b/tests/util/test_os_ops.py @@ -34,17 +34,17 @@ def test_make_wtmp_dir(self): tmp_dir = Path(tmp_dir) wtmp_dir_1 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_1.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_1.is_dir()) self.assertTrue(wtmp_dir_1.name.endswith("1")) wtmp_dir_2 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_2.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_2.is_dir()) self.assertTrue(wtmp_dir_2.name.endswith("2")) wtmp_dir_3 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_3.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_3.is_dir()) self.assertTrue(wtmp_dir_3.name.endswith("3")) wtmp_dir_4 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_4.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_4.is_dir()) self.assertTrue(wtmp_dir_4.name.endswith("4")) From f521964f665886dfc417708a11fc97b4d1613914 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 25 May 2018 17:58:06 -0700 Subject: [PATCH 015/110] Remove last no-member ignore line false-positive. --- tests/js/test_compile_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index a5caee9fa..90edc7b35 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -45,7 +45,7 @@ def test_shell_compile(self): Returns: Path: Path to the compiled shell. """ - self.assertTrue(self.mc_hg_repo.is_dir()) # pylint: disable=no-member + self.assertTrue(self.mc_hg_repo.is_dir()) # Change the repository location by uncommenting this line and specifying the right one # "-R ~/trees/mozilla-central/") From 7e9a7d9fd638492378971c79490e8609aae0457b Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 25 May 2018 22:32:41 -0700 Subject: [PATCH 016/110] Continue to ignore no-member pylint errors until newer linter libraries get released. --- src/funfuzz/bot.py | 1 + src/funfuzz/js/build_options.py | 2 +- src/funfuzz/js/compile_shell.py | 3 ++- src/funfuzz/js/link_fuzzer.py | 4 ++-- src/funfuzz/util/create_collector.py | 2 +- src/funfuzz/util/hg_helpers.py | 2 +- src/funfuzz/util/os_ops.py | 10 +++++----- src/funfuzz/util/repos_update.py | 4 ++-- tests/js/test_build_options.py | 1 + tests/js/test_compile_shell.py | 2 +- tests/util/test_hg_helpers.py | 1 + tests/util/test_os_ops.py | 8 ++++---- 12 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 41845e77c..133274293 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -88,6 +88,7 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur if args: print("Warning: bot does not use positional arguments") + # pylint: disable=no-member if not options.useTreeherderBuilds and not build_options.DEFAULT_TREES_LOCATION.is_dir(): # We don't have trees, so we must use treeherder builds. options.useTreeherderBuilds = True diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index d420b4f2c..9b15e8607 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -188,7 +188,7 @@ def parse_shell_opts(args): print("WARNING: This set of build options is not tested well because: %s" % valid[1]) # Ensures releng machines do not enter the if block and assumes mozilla-central always exists - if DEFAULT_TREES_LOCATION.is_dir(): + if DEFAULT_TREES_LOCATION.is_dir(): # pylint: disable=no-member # Repositories do not get randomized if a repository is specified. if build_options.repo_dir is None: # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 43e7c5a4d..4d551cdd0 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -9,6 +9,7 @@ # reset ; rg -g '!*subprocesses.py' -g '!*crashesat.py' -g '!*s3cache.py' -g '!*test_shell_flags.py' # -g '!*test_compile_shell.py' -g '!*known_broken*.py' -t py "import os$" +# disable no-member pylint messages can be removed after https://github.com/PyCQA/pylint/issues/1660 lands on 1.8 from __future__ import absolute_import, print_function, unicode_literals # isort:skip @@ -801,7 +802,7 @@ def verify_full_win_pageheap(shell_path): # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" - if gflags_bin_path.is_file() and shell_path.is_file(): + if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member print(subprocess.check_output([str(gflags_bin_path).decode("utf-8", errors="replace"), "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"])) diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index e017f3666..d0497f7d9 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -30,9 +30,9 @@ def link_fuzzer(target_path, prologue=""): if prologue: f.write(prologue) - for entry in (base_dir / "files_to_link.txt").read_text().split(): + for entry in (base_dir / "files_to_link.txt").read_text().split(): # pylint: disable=no-member entry = entry.rstrip() if entry and not entry.startswith("#"): file_path = base_dir / Path(entry) f.write("\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:]) - f.write(file_path.read_text()) + f.write(file_path.read_text()) # pylint: disable=no-member diff --git a/src/funfuzz/util/create_collector.py b/src/funfuzz/util/create_collector.py index 104954b4b..d6011c70d 100644 --- a/src/funfuzz/util/create_collector.py +++ b/src/funfuzz/util/create_collector.py @@ -26,7 +26,7 @@ def make_collector(): Collector: jsfunfuzz collector object """ sigcache_path = Path.home() / "sigcache" - sigcache_path.mkdir(exist_ok=True) + sigcache_path.mkdir(exist_ok=True) # pylint: disable=no-member return Collector(sigCacheDir=str(sigcache_path), tool="jsfunfuzz") diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index 64459494b..b19041edb 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -45,7 +45,7 @@ def ensure_mq_enabled(): NoOptionError: Raises if an mq entry is not found in [extensions] """ user_hgrc = Path.home() / ".hgrc" - assert user_hgrc.is_file() + assert user_hgrc.is_file() # pylint: disable=no-member user_hgrc_cfg = configparser.SafeConfigParser() user_hgrc_cfg.read(str(user_hgrc)) diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index 4bf0564b4..c0eb8587a 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -58,7 +58,7 @@ def make_cdb_cmd(prog_full_path, crashed_pid): # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. cdb_path = win64_debugging_folder / "cdb.exe" - if cdb_path.is_file(): + if cdb_path.is_file(): # pylint: disable=no-member print() print("WARNING: cdb.exe is not found - all crashes will be interesting.") print() @@ -70,7 +70,7 @@ def make_cdb_cmd(prog_full_path, crashed_pid): while True: if dump_name.is_file(): dbggr_cmd_path = Path(__file__).parent / "cdb_cmds.txt" - assert dbggr_cmd_path.is_file() + assert dbggr_cmd_path.is_file() # pylint: disable=no-member cdb_cmd_list = [] cdb_cmd_list.append("$<" + str(dbggr_cmd_path)) @@ -119,7 +119,7 @@ def make_gdb_cmd(prog_full_path, crashed_pid): if core_name and core_name_path.is_file(): dbggr_cmd_path = Path(__file__).parent / "gdb_cmds.txt" - assert dbggr_cmd_path.is_file() + assert dbggr_cmd_path.is_file() # pylint: disable=no-member # Run gdb and move the core file. Tip: gdb gives more info for: # (debug with intact build dir > debug > opt with frame pointers > opt) @@ -256,8 +256,8 @@ def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): reports_dir = base_dir / "Library" / "Logs" / "DiagnosticReports" # Find a crash log for the right process name and pid, preferring # newer crash logs (which sort last). - if reports_dir.is_dir(): - crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) + if reports_dir.is_dir(): # pylint: disable=no-member + crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) # pylint: disable=no-member else: crash_logs = [] diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index 363d57952..ed53379a3 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -39,9 +39,9 @@ # pylint: disable=invalid-name git_64bit_path = Path(os.getenv("PROGRAMFILES")) / "Git" / "bin" / "git.exe" git_32bit_path = Path(os.getenv("PROGRAMFILES(X86)")) / "Git" / "bin" / "git.exe" - if git_64bit_path.is_file(): + if git_64bit_path.is_file(): # pylint: disable=no-member GITBINARY = str(git_64bit_path) - elif git_32bit_path.is_file(): + elif git_32bit_path.is_file(): # pylint: disable=no-member GITBINARY = str(git_32bit_path) else: raise OSError("Git binary not found") diff --git a/tests/js/test_build_options.py b/tests/js/test_build_options.py index 08774c66a..33121a6c2 100644 --- a/tests/js/test_build_options.py +++ b/tests/js/test_build_options.py @@ -44,6 +44,7 @@ class BuildOptionsTests(unittest.TestCase): monkeypatch = MonkeyPatch() trees_location = Path.home() / "trees" + # pylint: disable=no-member @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), reason="requires a Mozilla Mercurial repository") def test_get_random_valid_repo(self): diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index 90edc7b35..a5caee9fa 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -45,7 +45,7 @@ def test_shell_compile(self): Returns: Path: Path to the compiled shell. """ - self.assertTrue(self.mc_hg_repo.is_dir()) + self.assertTrue(self.mc_hg_repo.is_dir()) # pylint: disable=no-member # Change the repository location by uncommenting this line and specifying the right one # "-R ~/trees/mozilla-central/") diff --git a/tests/util/test_hg_helpers.py b/tests/util/test_hg_helpers.py index 133bd9448..3d3228b0d 100644 --- a/tests/util/test_hg_helpers.py +++ b/tests/util/test_hg_helpers.py @@ -51,6 +51,7 @@ def test_get_cset_hash_in_bisectmsg(self): "The variable msg is:")): hg_helpers.get_cset_hash_from_bisect_msg("1a2345 - abababababab") + # pylint: disable=no-member @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), reason="requires a Mozilla Mercurial repository") def test_hgrc_repo_name(self): diff --git a/tests/util/test_os_ops.py b/tests/util/test_os_ops.py index f9426b794..929680c6c 100644 --- a/tests/util/test_os_ops.py +++ b/tests/util/test_os_ops.py @@ -34,17 +34,17 @@ def test_make_wtmp_dir(self): tmp_dir = Path(tmp_dir) wtmp_dir_1 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_1.is_dir()) + self.assertTrue(wtmp_dir_1.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_1.name.endswith("1")) wtmp_dir_2 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_2.is_dir()) + self.assertTrue(wtmp_dir_2.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_2.name.endswith("2")) wtmp_dir_3 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_3.is_dir()) + self.assertTrue(wtmp_dir_3.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_3.name.endswith("3")) wtmp_dir_4 = os_ops.make_wtmp_dir(tmp_dir) - self.assertTrue(wtmp_dir_4.is_dir()) + self.assertTrue(wtmp_dir_4.is_dir()) # pylint: disable=no-member self.assertTrue(wtmp_dir_4.name.endswith("4")) From 9ccc030c5bf4de381bd2e27adb48f8a73435929f Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Tue, 10 Apr 2018 16:17:41 -0700 Subject: [PATCH 017/110] Inline scanLine function, then move last detect_malloc_errors function "amiss" into util/file_manipulation.py --- src/funfuzz/js/js_interesting.py | 4 +- src/funfuzz/util/__init__.py | 1 - src/funfuzz/util/detect_malloc_errors.py | 50 ------------------------ src/funfuzz/util/file_manipulation.py | 19 +++++++++ 4 files changed, 21 insertions(+), 53 deletions(-) delete mode 100644 src/funfuzz/util/detect_malloc_errors.py diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 2c3f27708..1cc87a8fd 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -24,7 +24,7 @@ from . import inspect_shell from ..util import create_collector -from ..util import detect_malloc_errors +from ..util import file_manipulation from ..util import os_ops if sys.version_info.major == 2: @@ -107,7 +107,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if os_ops.grab_crash_log(runthis[0], runinfo.pid, logPrefix, True): with open(str(logPrefix + "-crash.txt")) as f: auxCrashData = [line.strip() for line in f.readlines()] - elif detect_malloc_errors.amiss(logPrefix): + elif file_manipulation.amiss(logPrefix): issues.append("malloc error") lev = max(lev, JS_NEW_ASSERT_OR_CRASH) elif runinfo.return_code == 0 and not in_compare_jit: diff --git a/src/funfuzz/util/__init__.py b/src/funfuzz/util/__init__.py index cf7a0f18e..9134f4ac6 100644 --- a/src/funfuzz/util/__init__.py +++ b/src/funfuzz/util/__init__.py @@ -10,7 +10,6 @@ from . import crashesat from . import create_collector -from . import detect_malloc_errors from . import file_manipulation from . import fork_join from . import hg_helpers diff --git a/src/funfuzz/util/detect_malloc_errors.py b/src/funfuzz/util/detect_malloc_errors.py deleted file mode 100644 index c97101711..000000000 --- a/src/funfuzz/util/detect_malloc_errors.py +++ /dev/null @@ -1,50 +0,0 @@ -# coding=utf-8 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -"""Look for "szone_error" (Tiger), "malloc_error_break" (Leopard), "MallocHelp" (?) -which are signs of malloc being unhappy (double free, out-of-memory, etc). -""" - -from __future__ import absolute_import, print_function # isort:skip - -PLINE = "" -PPLINE = "" - - -def amiss(log_prefix): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc - found_something = False - global PLINE, PPLINE # pylint: disable=global-statement - - PLINE = "" - PPLINE = "" - - with open(str(log_prefix + "-err.txt")) as f: - for line in f: - if scanLine(line): - found_something = True - break # Don't flood the log with repeated malloc failures - - return found_something - - -def scanLine(line): # pylint: disable=inconsistent-return-statements,invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - global PPLINE, PLINE # pylint: disable=global-statement - - line = line.strip("\x07").rstrip("\n") - - if (line.find("szone_error") != -1 or - line.find("malloc_error_break") != -1 or - line.find("MallocHelp") != -1): - if PLINE.find("can't allocate region") == -1: - print() - print(PPLINE) - print(PLINE) - print(line) - return True - - PPLINE = PLINE - PLINE = line diff --git a/src/funfuzz/util/file_manipulation.py b/src/funfuzz/util/file_manipulation.py index 6ad8232df..8947ef0aa 100644 --- a/src/funfuzz/util/file_manipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -10,6 +10,25 @@ from __future__ import absolute_import, print_function # isort:skip +def amiss(log_prefix): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc + """Look for "szone_error" (Tiger), "malloc_error_break" (Leopard), "MallocHelp" (?) + which are signs of malloc being unhappy (double free, out-of-memory, etc). + """ + found_something = False + with open(log_prefix + "-err.txt") as f: + for line in f: + line = line.strip("\x07").rstrip("\n") + if (line.find("szone_error") != -1 or + line.find("malloc_error_break") != -1 or + line.find("MallocHelp") != -1): + print() + print(line) + found_something = True + break # Don't flood the log with repeated malloc failures + + return found_something + + def fuzzSplice(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc # pylint: disable=missing-type-doc """Return the lines of a file, minus the ones between the two lines containing SPLICE.""" From 9a1c70bd60a6eae41c719d4c219a76a1befb1662 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:01:24 -0700 Subject: [PATCH 018/110] Replace logPrefix file-renaming and suffix-changing with the proper pathlib way of doing so. --- src/funfuzz/js/compare_jit.py | 14 +++++++---- src/funfuzz/js/js_interesting.py | 35 ++++++++++++++++----------- src/funfuzz/js/loop.py | 26 +++++++++++++------- src/funfuzz/util/file_manipulation.py | 3 ++- src/funfuzz/util/lithium_helpers.py | 7 +++--- 5 files changed, 53 insertions(+), 32 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index d7c012cf6..934254e26 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -59,8 +59,9 @@ def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc # pylint: disable=too-many-arguments,too-many-locals + initialdir_name = (logPrefix.parent / (logPrefix.stem + "-initial")) # pylint: disable=invalid-name - cl = compareLevel(jsEngine, flags, infilename, logPrefix + "-initial", options, False, True) + cl = compareLevel(jsEngine, flags, infilename, initialdir_name, options, False, True) lev = cl[0] if lev != js_interesting.JS_FINE: @@ -70,7 +71,8 @@ def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, itest, logPrefix, jsEngine, [], infilename, repo, build_options_str, targetTime, lev) if lithResult == lithium_helpers.LITH_FINISHED: print("Retesting %s after running Lithium:" % infilename) - retest_cl = compareLevel(jsEngine, flags, infilename, logPrefix + "-final", options, True, False) + finaldir_name = (logPrefix.parent / (logPrefix.stem + "-final")) + retest_cl = compareLevel(jsEngine, flags, infilename, finaldir_name, options, True, False) if retest_cl[0] != js_interesting.JS_FINE: cl = retest_cl quality = 0 @@ -111,7 +113,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi commands = [[jsEngine] + combo + [str(infilename)] for combo in combos] for i in range(0, len(commands)): - prefix = logPrefix + "-r" + str(i) + prefix = (logPrefix.parent / ("%s-r%s" % (logPrefix.stem, str(i)))) command = commands[i] r = js_interesting.ShellResult(options, command, prefix, True) # pylint: disable=invalid-name @@ -130,7 +132,8 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi js_interesting.summaryString(r.issues + ["compare_jit found a more serious bug"], r.lev, r.runinfo.elapsedtime))) - with open(str(logPrefix + "-summary.txt"), "w") as f: + summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") + with open(str(summary_log), "w") as f: f.write("\n".join(r.issues + [" ".join(quote(x) for x in command), "compare_jit found a more serious bug"]) + "\n") print(" %s" % " ".join(quote(x) for x in command)) @@ -183,7 +186,8 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = (" " + " ".join(quote(x) for x in commands[0]) + "\n " + " ".join(quote(x) for x in command) + "\n\n" + summary) - with open(str(logPrefix + "-summary.txt"), "w") as f: + summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") + with open(str(summary_log), "w") as f: f.write(rerunCommand + "\n\n" + summary) print("%s | %s" % (str(infilename), js_interesting.summaryString( issues, js_interesting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime))) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 1cc87a8fd..7fc524b1e 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -73,7 +73,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like "./js" # pylint: disable=invalid-name assert os.path.isfile(os.path.abspath(pathToBinary + ".fuzzmanagerconf")) - pc = ProgramConfiguration.fromBinary(os.path.abspath(pathToBinary).split(".")[0]) + pc = ProgramConfiguration.fromBinary(str(pathToBinary.parent / pathToBinary.stem)) pc.addProgramArguments(runthis[1:-1]) if options.valgrind: @@ -91,9 +91,11 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and "readlines" everything into memory. - with open(str(logPrefix + "-out.txt")) as f: + out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") + with open(str(out_log)) as f: out = f.readlines() - with open(str(logPrefix + "-err.txt")) as f: + err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt") + with open(str(err_log)) as f: err = f.readlines() if options.valgrind and runinfo.return_code == VALGRIND_ERROR_EXIT_CODE: @@ -105,7 +107,8 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa issues.append(line.rstrip()) elif runinfo.sta == timed_run.CRASHED: if os_ops.grab_crash_log(runthis[0], runinfo.pid, logPrefix, True): - with open(str(logPrefix + "-crash.txt")) as f: + crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") + with open(str(crash_log)) as f: auxCrashData = [line.strip() for line in f.readlines()] elif file_manipulation.amiss(logPrefix): issues.append("malloc error") @@ -169,8 +172,9 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa print("%s | %s" % (logPrefix, summaryString(issues, lev, runinfo.elapsedtime))) if lev != JS_FINE: - with open(str(logPrefix + "-summary.txt"), "w") as f: - f.writelines(["Number: " + logPrefix + "\n", + summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") + with open(str(summary_log), "w") as f: + f.writelines(["Number: " + str(logPrefix) + "\n", "Command: " + " ".join(quote(x) for x in runthis) + "\n"] + ["Status: " + i + "\n" for i in issues]) @@ -249,16 +253,19 @@ def deleteLogs(logPrefix): # pylint: disable=invalid-name,missing-param-doc,mis """Whoever might call baseLevel should eventually call this function (unless a bug was found).""" # If this turns up a WindowsError on Windows, remember to have excluded fuzzing locations in # the search indexer, anti-virus realtime protection and backup applications. - os.remove(logPrefix + "-out.txt") - os.remove(logPrefix + "-err.txt") - if os.path.exists(logPrefix + "-crash.txt"): - os.remove(logPrefix + "-crash.txt") - if os.path.exists(logPrefix + "-vg.xml"): - os.remove(logPrefix + "-vg.xml") + (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt").unlink() + (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt").unlink() + crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") + if crash_log.is_file(): + crash_log.unlink() + valgrind_xml = (logPrefix.parent / (logPrefix.stem + "-vg")).with_suffix(".xml") + if valgrind_xml.is_file(): + valgrind_xml.unlink() # pylint: disable=fixme # FIXME: in some cases, subprocesses gzips a core file only for us to delete it immediately. - if os.path.exists(logPrefix + "-core.gz"): - os.remove(logPrefix + "-core.gz") + core_gzip = (logPrefix.parent / (logPrefix.stem + "-core")).with_suffix(".gz") + if core_gzip.is_file(): + core_gzip.unlink() def set_ulimit(): diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index c90d3ca1f..2f2a4af1f 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -167,17 +167,21 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in js_interesting_options.jsengineWithArgs, logPrefix, False) if res.lev != js_interesting.JS_FINE: - showtail(logPrefix + "-out.txt") - showtail(logPrefix + "-err.txt") + out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") + showtail(out_log) + err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt") + showtail(err_log) # splice jsfunfuzz.js with `grep "/*FRC-" wN-out` - filenameToReduce = logPrefix + "-reduced.js" # pylint: disable=invalid-name + reduced_log = (logPrefix.parent / (logPrefix.stem + "-reduced")).with_suffix(".js") + filenameToReduce = reduced_log # pylint: disable=invalid-name [before, after] = file_manipulation.fuzzSplice(fuzzjs) - with open(str(logPrefix + "-out.txt"), "r") as f: + with open(str(out_log), "r") as f: newfileLines = before + [ # pylint: disable=invalid-name l.replace("/*FRC-", "/*") for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after - with open(str(logPrefix + "-orig.js"), "w") as f: + orig_log = (logPrefix.parent / (logPrefix.stem + "-orig")).with_suffix(".js") + with open(str(orig_log), "w") as f: f.writelines(newfileLines) with open(str(filenameToReduce), "w") as f: f.writelines(newfileLines) @@ -199,7 +203,10 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in # pylint: disable=no-member fargs = js_interesting_options.jsengineWithArgs[:-1] + [filenameToReduce] # pylint: disable=invalid-name - retestResult = js_interesting.ShellResult(js_interesting_options, fargs, logPrefix + "-final", False) + retestResult = js_interesting.ShellResult(js_interesting_options, + fargs, + logPrefix.parent / (logPrefix.stem + "-final"), + False) if retestResult.lev > js_interesting.JS_FINE: res = retestResult quality = 0 @@ -221,13 +228,14 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in # pylint: disable=no-member if options.use_compare_jit and res.lev == js_interesting.JS_FINE and \ js_interesting_options.shellIsDeterministic and are_flags_deterministic: - linesToCompare = jitCompareLines(logPrefix + "-out.txt", "/*FCM*/") # pylint: disable=invalid-name - jitcomparefilename = logPrefix + "-cj-in.js" + out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") + linesToCompare = jitCompareLines(out_log, "/*FCM*/") # pylint: disable=invalid-name + jitcomparefilename = (logPrefix.parent / (logPrefix.stem + "-cj-in")).with_suffix(".js") with open(str(jitcomparefilename), "w") as f: f.writelines(linesToCompare) # pylint: disable=invalid-name anyBug = compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename, - logPrefix + "-cj", options.repo, + logPrefix.parent / (logPrefix.stem + "-cj"), options.repo, options.build_options_str, targetTime, js_interesting_options) if not anyBug: jitcomparefilename.unlink() diff --git a/src/funfuzz/util/file_manipulation.py b/src/funfuzz/util/file_manipulation.py index 8947ef0aa..4d242dddc 100644 --- a/src/funfuzz/util/file_manipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -15,7 +15,8 @@ def amiss(log_prefix): # pylint: disable=missing-param-doc,missing-return-doc,m which are signs of malloc being unhappy (double free, out-of-memory, etc). """ found_something = False - with open(log_prefix + "-err.txt") as f: + err_log = (log_prefix.parent / (log_prefix.stem + "-err")).with_suffix(".txt") + with open(str(err_log)) as f: for line in f: line = line.strip("\x07").rstrip("\n") if (line.find("szone_error") != -1 or diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 3c32883de..2e94008ec 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -65,7 +65,8 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ["-i"] + itest ) print(" ".join(quote(x) for x in autobisectCmd)) - autoBisectLogFilename = logPrefix + "-autobisect.txt" # pylint: disable=invalid-name + # pylint: disable=invalid-name + autoBisectLogFilename = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") subprocess.call(autobisectCmd, stdout=open(str(autoBisectLogFilename), "w"), stderr=subprocess.STDOUT) print("Done running autobisectjs. Log: %s" % autoBisectLogFilename) @@ -92,10 +93,10 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam lithArgs = ["--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp] + lithArgs else: # loop is being run standalone - lithtmp = logPrefix + "-lith-tmp" + lithtmp = logPrefix.parent / (logPrefix.stem + "-lith-tmp") Path.mkdir(lithtmp) lithArgs = ["--tempdir=" + lithtmp] + lithArgs - lithlogfn = logPrefix + "-lith-out.txt" + lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) print(" ".join(quote(x) for x in runlithiumpy + lithArgs)) subprocess.call(runlithiumpy + lithArgs, stdout=open(str(lithlogfn), "w"), stderr=subprocess.STDOUT) From 3340b2c5aeafd2fce690d13f4a1a14c4a7ef14fc Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:03:53 -0700 Subject: [PATCH 019/110] Deal with .fuzzmanagerconf files using pathlib. --- src/funfuzz/js/js_interesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 7fc524b1e..f8e82fb09 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -68,11 +68,11 @@ class ShellResult(object): # pylint: disable=missing-docstring,too-many-instanc # options dict should include: timeout, knownPath, collector, valgrind, shellIsDeterministic def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disable=too-complex,too-many-branches # pylint: disable=too-many-locals,too-many-statements - pathToBinary = runthis[0] # pylint: disable=invalid-name + pathToBinary = runthis[0].resolve() # pylint: disable=invalid-name # This relies on the shell being a local one from compile_shell: # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like "./js" # pylint: disable=invalid-name - assert os.path.isfile(os.path.abspath(pathToBinary + ".fuzzmanagerconf")) + assert pathToBinary.with_suffix(".fuzzmanagerconf").is_file() pc = ProgramConfiguration.fromBinary(str(pathToBinary.parent / pathToBinary.stem)) pc.addProgramArguments(runthis[1:-1]) From c113499095234675e223ae42ad1735936348ce03 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:04:28 -0700 Subject: [PATCH 020/110] Convert Paths to make them Lithium-friendly. --- src/funfuzz/js/js_interesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index f8e82fb09..eaa6f4878 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -83,7 +83,8 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa runthis) # logPrefix should be a string for timed_run in Lithium version 0.2.1 to work properly, apparently - runinfo = timed_run.timed_run(runthis, options.timeout, logPrefix.encode("utf-8"), preexec_fn=set_ulimit) + runthis = [str(x) if isinstance(x, Path) else x for x in runthis] # Convert all Paths to strings for Lithium + runinfo = timed_run.timed_run(runthis, options.timeout, str(logPrefix).encode("utf-8"), preexec_fn=set_ulimit) lev = JS_FINE issues = [] From 840c7bbdfca3ee691c29045f233ca13fc409e719 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:05:35 -0700 Subject: [PATCH 021/110] Check file sizes in truncateFile using the pathlib way. --- src/funfuzz/js/js_interesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index eaa6f4878..16a9a18d2 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -240,7 +240,7 @@ def summaryString(issues, level, elapsedtime): # pylint: disable=invalid-name,m def truncateFile(fn, maxSize): # pylint: disable=invalid-name,missing-docstring - if os.path.exists(fn) and os.path.getsize(fn) > maxSize: + if fn.is_file() and fn.stat().st_size > maxSize: with open(str(fn), "r+") as f: f.truncate(maxSize) From 578a0e6742d461fe8eafdc033e4ff7554601f853 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:06:01 -0700 Subject: [PATCH 022/110] More pathlib fixes. --- src/funfuzz/js/js_interesting.py | 2 +- src/funfuzz/js/loop.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 16a9a18d2..5d918cfae 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -311,7 +311,7 @@ def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missin options.knownPath = args[0] options.jsengineWithArgs = args[1:] options.collector = create_collector.make_collector() - if not os.path.exists(options.jsengineWithArgs[0]): + if not options.jsengineWithArgs[0].is_file(): raise Exception("js shell does not exist: " + options.jsengineWithArgs[0]) options.shellIsDeterministic = inspect_shell.queryBuildConfiguration( options.jsengineWithArgs[0], "more-deterministic") diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 2f2a4af1f..3968dfc77 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -85,7 +85,7 @@ def showtail(filename): # pylint: disable=missing-docstring # FIXME: Get jsfunfuzz to output start & end of interesting result boundaries instead of this. cmd = [] cmd.extend(["tail", "-n", "20"]) - cmd.append(filename) + cmd.append(str(filename)) print(" ".join(cmd)) print() subprocess.check_call(cmd) From bde71ebd0589e7e382f0bbc71429a63804f7f331 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:06:35 -0700 Subject: [PATCH 023/110] We should assert the existence of the concatenated jsfunfuzz file a little later. --- src/funfuzz/js/loop.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 3968dfc77..8e9ae77a6 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -133,8 +133,9 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in regressionTestPrologue = "" # pylint: disable=invalid-name fuzzjs = wtmpDir / "jsfunfuzz.js" - assert fuzzjs.is_file() + link_fuzzer.link_fuzzer(fuzzjs, regressionTestPrologue) + assert fuzzjs.is_file() iteration = 0 while True: From a9533be58839a8ac7f678b93283d7ca9345f539b Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:07:08 -0700 Subject: [PATCH 024/110] Rename the autoBisectLogFilename and autoBisectLog variables. --- src/funfuzz/util/lithium_helpers.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 2e94008ec..da1a6d47c 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -65,18 +65,17 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ["-i"] + itest ) print(" ".join(quote(x) for x in autobisectCmd)) - # pylint: disable=invalid-name - autoBisectLogFilename = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") - subprocess.call(autobisectCmd, stdout=open(str(autoBisectLogFilename), "w"), stderr=subprocess.STDOUT) - print("Done running autobisectjs. Log: %s" % autoBisectLogFilename) + autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") + subprocess.call(autobisectCmd, stdout=open(str(autobisect_log), "w"), stderr=subprocess.STDOUT) + print("Done running autobisectjs. Log: %s" % autobisect_log) - with open(str(autoBisectLogFilename), "r") as f: + with open(str(autobisect_log), "r") as f: lines = f.readlines() - autoBisectLog = file_manipulation.truncateMid(lines, 50, ["..."]) # pylint: disable=invalid-name + autobisect_log_trunc = file_manipulation.truncateMid(lines, 50, ["..."]) else: - autoBisectLog = [] # pylint: disable=invalid-name + autobisect_log_trunc = [] # pylint: disable=invalid-name - return (lithResult, lithDetails, autoBisectLog) + return (lithResult, lithDetails, autobisect_log_trunc) def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc From 60199afac1d6f30604cd66644f16b28cda0a3a68 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:07:44 -0700 Subject: [PATCH 025/110] Make handle_rm_readonly Windows-only for simplification since we only use it on Windows. --- src/funfuzz/util/subprocesses.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index d63c9f1fc..e15447d15 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -45,7 +45,7 @@ def rm_tree_incl_readonly(dir_tree): def handle_rm_readonly(func, path, exc): - """Handle read-only files. Adapted from http://stackoverflow.com/q/1213706 and some docs below adapted from + """Handle read-only files on Windows. Adapted from http://stackoverflow.com/q/1213706 and some docs adapted from Python 2.7 official docs. Args: @@ -56,14 +56,9 @@ def handle_rm_readonly(func, path, exc): Raises: OSError: Raised if the read-only files are unable to be handled """ + assert platform.system() == "Windows" if func in (os.rmdir, os.remove) and exc[1].errno == errno.EACCES: - if os.name == "posix": - # Ensure parent directory is also writeable. - pardir = os.path.abspath(os.path.join(path, os.path.pardir)) - if not os.access(pardir, os.W_OK): - os.chmod(pardir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - elif os.name == "nt": - os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 func(path) else: raise OSError("Unable to handle read-only files.") From 5d0fc3497fde2fc63ba113aab31439e47c0078fc Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:19:58 -0700 Subject: [PATCH 026/110] Remove superfluous __name__/__main__ line in bot.py --- src/funfuzz/bot.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 133274293..ecf4b2398 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -252,7 +252,3 @@ def mtrArgsCreation(options, cshell): # pylint: disable=invalid-name,missing-pa manyTimedRunArgs.append(cshell.getRepoName()) # known bugs' directory manyTimedRunArgs.append(cshell.get_shell_cache_js_bin_path()) return manyTimedRunArgs - - -if __name__ == "__main__": - main() From cbf2ba6d1ab9a117a042e6ab862f2c848df5cbda Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 03:20:41 -0700 Subject: [PATCH 027/110] Python 3 only allows binary mode when using open() with buffering=0 set. --- src/funfuzz/util/fork_join.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index 669c3b789..a8c4dab59 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -69,8 +69,8 @@ def log_name(log_dir, i, log_type): def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring - sys.stdout = open(log_name(logDir, i, "out"), "w", buffering=0) # I WONDER .decode("utf-8", errors="replace") - sys.stderr = open(log_name(logDir, i, "err"), "w", buffering=0) # I WONDER .decode("utf-8", errors="replace") + sys.stdout = open(log_name(logDir, i, "out"), "wb", buffering=0) # I WONDER .decode("utf-8", errors="replace") + sys.stderr = open(log_name(logDir, i, "err"), "wb", buffering=0) # I WONDER .decode("utf-8", errors="replace") fun(*(someArgs + (i,))) From 2043c32d2760229168a05d65d1eaeea22498b34c Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 04:19:20 -0700 Subject: [PATCH 028/110] Use the parent/stem/suffix model of pathlib in more places. --- src/funfuzz/js/compare_jit.py | 12 ++++++++---- src/funfuzz/js/js_interesting.py | 6 ++++-- src/funfuzz/util/lithium_helpers.py | 2 +- src/funfuzz/util/os_ops.py | 29 +++++++++++++++-------------- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 934254e26..550c30720 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -211,23 +211,27 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc -def summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix): +def summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix1): issues = [] summary = "" if mismatchErr: issues.append("[Non-crash bug] Mismatch on stderr") summary += "[Non-crash bug] Mismatch on stderr\n" - summary += diffFiles(prefix0 + "-err.txt", prefix + "-err.txt") + err0_log = (prefix0.parent / (prefix0.stem + "-err")).with_suffix(".txt") + err1_log = (prefix1.parent / (prefix1.stem + "-err")).with_suffix(".txt") + summary += diffFiles(err0_log, err1_log) if mismatchOut: issues.append("[Non-crash bug] Mismatch on stdout") summary += "[Non-crash bug] Mismatch on stdout\n" - summary += diffFiles(prefix0 + "-out.txt", prefix + "-out.txt") + out0_log = (prefix0.parent / (prefix0.stem + "-out")).with_suffix(".txt") + out1_log = (prefix1.parent / (prefix1.stem + "-out")).with_suffix(".txt") + summary += diffFiles(out0_log, out1_log) return (summary, issues) def diffFiles(f1, f2): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Return a command to diff two files, along with the diff output (if it's short).""" - diffcmd = ["diff", "-u", f1, f2] + diffcmd = ["diff", "-u", str(f1), str(f2)] s = " ".join(diffcmd) + "\n\n" # pylint: disable=invalid-name diff = subprocess.run(diffcmd, cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 5d918cfae..717a32114 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -334,8 +334,10 @@ def interesting(_args, tempPrefix): # pylint: disable=invalid-name,missing-docs options = gOptions # options, runthis, logPrefix, in_compare_jit res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) - truncateFile(tempPrefix + "-out.txt", 1000000) - truncateFile(tempPrefix + "-err.txt", 1000000) + out_log = (tempPrefix.parent / (tempPrefix.stem + "-out")).with_suffix(".txt") + err_log = (tempPrefix.parent / (tempPrefix.stem + "-err")).with_suffix(".txt") + truncateFile(out_log, 1000000) + truncateFile(err_log, 1000000) return res.lev >= gOptions.minimumInterestingLevel diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index da1a6d47c..a89a9bf6f 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -137,7 +137,7 @@ def reduction_strat(logPrefix, infilename, lithArgs, targetTime, lev): # pylint # This is an array because Python does not like assigning to upvars. reductionCount = [0] # pylint: disable=invalid-name - backup_file = infilename + "-backup" + backup_file = (logPrefix.parent / (logPrefix.stem + "-backup")) def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index c0eb8587a..a9d2df79a 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -160,14 +160,14 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): progname = prog_full_path.name use_logfiles = isinstance(log_prefix, ("".__class__, u"".__class__)) - crash_log_path = Path(log_prefix + "-crash.txt") - core_path = Path(log_prefix + "-core") + crash_log = (log_prefix.parent / (log_prefix.stem + "-crash")).with_suffix(".txt") + core_file = (log_prefix.parent / (log_prefix.stem + "-core")) if use_logfiles: - if crash_log_path.is_file(): - crash_log_path.unlink() - if core_path.is_file(): - core_path.unlink() + if crash_log.is_file(): + crash_log.unlink() + if core_file.is_file(): + core_file.unlink() if not want_stack or progname == "valgrind": return "" @@ -188,7 +188,7 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): dbggr_cmd, stdin=None, stderr=subprocess.STDOUT, - stdout=open(str(crash_log_path), "w") if use_logfiles else None, + stdout=open(str(crash_log), "w") if use_logfiles else None, # It would be nice to use this everywhere, but it seems to be broken on Windows # (http://docs.python.org/library/subprocess.html) close_fds=(os.name == "posix"), @@ -199,11 +199,11 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): print("Debugger exited with code %d : %s" % (dbbgr_exit_code, " ".join(quote(x) for x in dbggr_cmd))) if use_logfiles: if core_file.is_file(): - shutil.move(str(core_file), str(core_path)) - subprocess.call(["gzip", "-f", str(core_path)]) + shutil.move(str(core_file), str(core_file)) + subprocess.call(["gzip", "-f", str(core_file)]) # chmod here, else the uploaded -core.gz files do not have sufficient permissions. - subprocess.check_call(["chmod", "og+r", "%s.gz" % core_path]) - return str(crash_log_path) + subprocess.check_call(["chmod", "og+r", "%s.gz" % core_file]) + return str(crash_log) else: print("I don't know what to do with a core file when log_prefix is null") @@ -271,13 +271,14 @@ def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): # Copy, don't rename, because we might not have permissions # (especially for the system rather than user crash log directory) # Use copyfile, as we do not want to copy the permissions metadata over - shutil.copyfile(str(full_report_path), log_prefix + "-crash.txt") - subprocess.run(["chmod", "og+r", log_prefix + "-crash.txt"], + crash_log = (log_prefix.parent / (log_prefix.stem + "-crash")).with_suffix(".txt") + shutil.copyfile(str(full_report_path), str(crash_log)) + subprocess.run(["chmod", "og+r", str(crash_log)], # pylint: disable=no-member cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), check=True, timeout=9) - return log_prefix + "-crash.txt" + return str(crash_log) return str(full_report_path) except OSError: From ae2ec0aaf555ca795665537e69fa2ca8784fe4de Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 04:19:53 -0700 Subject: [PATCH 029/110] Make runthis retain its Path components and only convert to string when calling Lithium functions. --- src/funfuzz/js/js_interesting.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 717a32114..98d127e44 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -83,8 +83,11 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa runthis) # logPrefix should be a string for timed_run in Lithium version 0.2.1 to work properly, apparently - runthis = [str(x) if isinstance(x, Path) else x for x in runthis] # Convert all Paths to strings for Lithium - runinfo = timed_run.timed_run(runthis, options.timeout, str(logPrefix).encode("utf-8"), preexec_fn=set_ulimit) + runinfo = timed_run.timed_run( + [str(x) if isinstance(x, Path) else x for x in runthis], # Convert all Paths to strings for Lithium + options.timeout, + str(logPrefix).encode("utf-8"), + preexec_fn=set_ulimit) lev = JS_FINE issues = [] From 69f2ce8a540d5da4a122e513a97356af66980086 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 04:23:16 -0700 Subject: [PATCH 030/110] Make quote usage first convert items to strings. --- src/funfuzz/js/compare_jit.py | 14 +++++++------- src/funfuzz/js/compile_shell.py | 8 ++++---- src/funfuzz/js/inspect_shell.py | 2 +- src/funfuzz/js/js_interesting.py | 2 +- src/funfuzz/util/lithium_helpers.py | 8 ++++---- src/funfuzz/util/os_ops.py | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 550c30720..e932df27e 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -123,7 +123,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi if (r.return_code == 1 or r.return_code == 2) and (anyLineContains(r.out, "[[script] scriptArgs*]") or ( anyLineContains(r.err, "[scriptfile] [scriptarg...]"))): print("Got usage error from:") - print(" %s" % " ".join(quote(x) for x in command)) + print(" %s" % " ".join(quote(str(x)) for x in command)) assert i js_interesting.deleteLogs(prefix) elif r.lev > js_interesting.JS_OVERALL_MISMATCH: @@ -134,15 +134,15 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi r.runinfo.elapsedtime))) summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") with open(str(summary_log), "w") as f: - f.write("\n".join(r.issues + [" ".join(quote(x) for x in command), + f.write("\n".join(r.issues + [" ".join(quote(str(x)) for x in command), "compare_jit found a more serious bug"]) + "\n") - print(" %s" % " ".join(quote(x) for x in command)) + print(" %s" % " ".join(quote(str(x)) for x in command)) return (r.lev, r.crashInfo) elif r.lev != js_interesting.JS_FINE or r.return_code != 0: print("%s | %s" % (str(infilename), js_interesting.summaryString( r.issues + ["compare_jit is not comparing output, because the shell exited strangely"], r.lev, r.runinfo.elapsedtime))) - print(" %s" % " ".join(quote(x) for x in command)) + print(" %s" % " ".join(quote(str(x)) for x in command)) js_interesting.deleteLogs(prefix) if not i: return (js_interesting.JS_FINE, None) @@ -177,15 +177,15 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr if mismatchErr or mismatchOut: # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. # pylint: disable=invalid-name - rerunCommand = " ".join(quote(x) for x in ["python -m funfuzz.js.compare_jit", + rerunCommand = " ".join(quote(str(x)) for x in ["python -m funfuzz.js.compare_jit", "--flags=" + " ".join(flags), "--timeout=" + str(options.timeout), options.knownPath, jsEngine, str(infilename.name)]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) - summary = (" " + " ".join(quote(x) for x in commands[0]) + "\n " + - " ".join(quote(x) for x in command) + "\n\n" + summary) + summary = (" " + " ".join(quote(str(x)) for x in commands[0]) + "\n " + + " ".join(quote(str(x)) for x in command) + "\n\n" + summary) summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") with open(str(summary_log), "w") as f: f.write(rerunCommand + "\n\n" + summary) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 4d551cdd0..a67872ebf 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -507,8 +507,8 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ '"' if " " in cfg_env[str(env_var)] else env_var + "=" + cfg_env[str(env_var)]) env_vars.append(str_to_be_appended) - sps.vdump("Command to be run is: " + " ".join(quote(x) for x in env_vars) + " " + - " ".join(quote(x) for x in cfg_cmds)) + sps.vdump("Command to be run is: " + " ".join(quote(str(x)) for x in env_vars) + " " + + " ".join(quote(str(x)) for x in cfg_cmds)) assert shell.get_js_objdir().is_dir() @@ -624,8 +624,8 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi f.write("# %s\n# \n" % str(shell.getEnvFull())) f.write("# Full configuration command with needed environment variables is:\n") - f.write("# %s %s\n# \n" % (" ".join(quote(x) for x in shell.getEnvAdded()), - " ".join(quote(x) for x in shell.getCfgCmdExclEnv()))) + f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.getEnvAdded()), + " ".join(quote(str(x)) for x in shell.getCfgCmdExclEnv()))) # .fuzzmanagerconf details f.write("\n") diff --git a/src/funfuzz/js/inspect_shell.py b/src/funfuzz/js/inspect_shell.py index 37575c286..e5670b2aa 100644 --- a/src/funfuzz/js/inspect_shell.py +++ b/src/funfuzz/js/inspect_shell.py @@ -146,7 +146,7 @@ def testBinary(shellPath, args, useValgrind): # pylint: disable=invalid-name,mi # pylint: disable=missing-return-type-doc,missing-type-doc """Test the given shell with the given args.""" test_cmd = (constructVgCmdList() if useValgrind else []) + [str(shellPath)] + args - sps.vdump("The testing command is: " + " ".join(quote(x) for x in test_cmd)) + sps.vdump("The testing command is: " + " ".join(quote(str(x)) for x in test_cmd)) test_cmd_result = subprocess.run( test_cmd, cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 98d127e44..2ae95ce94 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -179,7 +179,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") with open(str(summary_log), "w") as f: f.writelines(["Number: " + str(logPrefix) + "\n", - "Command: " + " ".join(quote(x) for x in runthis) + "\n"] + + "Command: " + " ".join(quote(str(x)) for x in runthis) + "\n"] + ["Status: " + i + "\n" for i in issues]) self.lev = lev diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index a89a9bf6f..8da7afc94 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -52,7 +52,7 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis print() print("Done running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:") - print(" ".join(quote(x) for x in [sys.executable, "-u", "-m", "lithium", "--strategy=check-only"] + lithArgs)) + print(" ".join(quote(str(x)) for x in [sys.executable, "-u", "-m", "lithium", "--strategy=check-only"] + lithArgs)) print() # pylint: disable=literal-comparison @@ -64,7 +64,7 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ["-p", " ".join(engineFlags + [infilename])] + ["-i"] + itest ) - print(" ".join(quote(x) for x in autobisectCmd)) + print(" ".join(quote(str(x)) for x in autobisectCmd)) autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") subprocess.call(autobisectCmd, stdout=open(str(autobisect_log), "w"), stderr=subprocess.STDOUT) print("Done running autobisectjs. Log: %s" % autobisect_log) @@ -97,7 +97,7 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam lithArgs = ["--tempdir=" + lithtmp] + lithArgs lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) - print(" ".join(quote(x) for x in runlithiumpy + lithArgs)) + print(" ".join(quote(str(x)) for x in runlithiumpy + lithArgs)) subprocess.call(runlithiumpy + lithArgs, stdout=open(str(lithlogfn), "w"), stderr=subprocess.STDOUT) print("Done running Lithium") if deletableLithTemp: @@ -152,7 +152,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis reductionCount[0] += 1 # Remove empty elements full_lith_args = [x for x in (strategy + lithArgs) if x] - print(" ".join(quote(x) for x in [sys.executable, "-u", "-m", "lithium"] + full_lith_args)) + print(" ".join(quote(str(x)) for x in [sys.executable, "-u", "-m", "lithium"] + full_lith_args)) desc = "-chars" if strategy == "--char" else "-lines" (lith_result, lith_details) = run_lithium( # pylint: disable=invalid-name diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index a9d2df79a..a6cfe4c3f 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -196,7 +196,7 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): preexec_fn=(disable_corefile if platform.system() == "Linux" else None) ) if dbbgr_exit_code != 0: - print("Debugger exited with code %d : %s" % (dbbgr_exit_code, " ".join(quote(x) for x in dbggr_cmd))) + print("Debugger exited with code %d : %s" % (dbbgr_exit_code, " ".join(quote(str(x)) for x in dbggr_cmd))) if use_logfiles: if core_file.is_file(): shutil.move(str(core_file), str(core_file)) From dfdf7cd282d4723ef7708444f944cc655aac411e Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 04:30:19 -0700 Subject: [PATCH 031/110] Replace the filenameToReduce variable name with reduced_log. --- src/funfuzz/js/loop.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 8e9ae77a6..067372179 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -175,7 +175,6 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in # splice jsfunfuzz.js with `grep "/*FRC-" wN-out` reduced_log = (logPrefix.parent / (logPrefix.stem + "-reduced")).with_suffix(".js") - filenameToReduce = reduced_log # pylint: disable=invalid-name [before, after] = file_manipulation.fuzzSplice(fuzzjs) with open(str(out_log), "r") as f: @@ -184,7 +183,7 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in orig_log = (logPrefix.parent / (logPrefix.stem + "-orig")).with_suffix(".js") with open(str(orig_log), "w") as f: f.writelines(newfileLines) - with open(str(filenameToReduce), "w") as f: + with open(str(reduced_log), "w") as f: f.writelines(newfileLines) # Run Lithium and autobisectjs (make a reduced testcase and find a regression window) @@ -196,13 +195,13 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in itest.append("--timeout=" + str(options.timeout)) itest.append(options.knownPath) (lithResult, _lithDetails, autoBisectLog) = lithium_helpers.pinpoint( # pylint: disable=invalid-name - itest, logPrefix, options.jsEngine, engineFlags, filenameToReduce, options.repo, + itest, logPrefix, options.jsEngine, engineFlags, reduced_log, options.repo, options.build_options_str, targetTime, res.lev) # Upload with final output if lithResult == lithium_helpers.LITH_FINISHED: # pylint: disable=no-member - fargs = js_interesting_options.jsengineWithArgs[:-1] + [filenameToReduce] + fargs = js_interesting_options.jsengineWithArgs[:-1] + [reduced_log] # pylint: disable=invalid-name retestResult = js_interesting.ShellResult(js_interesting_options, fargs, @@ -216,13 +215,13 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in else: quality = 10 - print("Submitting %s (quality=%s) at %s" % (filenameToReduce, quality, time.asctime())) + print("Submitting %s (quality=%s) at %s" % (reduced_log, quality, time.asctime())) metadata = {} if autoBisectLog: metadata = {"autoBisectLog": "".join(autoBisectLog)} - collector.submit(res.crashInfo, filenameToReduce, quality, metaData=metadata) - print("Submitted %s" % filenameToReduce) + collector.submit(res.crashInfo, reduced_log, quality, metaData=metadata) + print("Submitted %s" % reduced_log) else: are_flags_deterministic = "--dump-bytecode" not in engineFlags and "-D" not in engineFlags From 6a152a0d7231e45b8cd50d2c27fb649db90fbf44 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 26 May 2018 04:31:30 -0700 Subject: [PATCH 032/110] Force shutil usages to operate on strings. --- src/funfuzz/util/lithium_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 8da7afc94..1d8978177 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -158,7 +158,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis (lith_result, lith_details) = run_lithium( # pylint: disable=invalid-name full_lith_args, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) if lith_result == LITH_FINISHED: - shutil.copy2(infilename, backup_file) + shutil.copy2(str(infilename), str(backup_file)) return lith_result, lith_details @@ -268,7 +268,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis if lith_result != LITH_FINISHED and lith_result != LITH_PLEASE_CONTINUE: # Probably can move instead of copy the backup, once this has stabilised. if backup_file.is_file(): - shutil.copy2(str(backup_file), infilename) + shutil.copy2(str(backup_file), str(infilename)) else: print("DEBUG! backup_file is supposed to be: %s" % backup_file) From f9ae532aee28ba8d34a8a671ed78aedc53c3fd77 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Tue, 29 May 2018 15:38:12 -0700 Subject: [PATCH 033/110] Rename _ to branch. --- src/funfuzz/js/build_options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 9b15e8607..981faf20f 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -354,8 +354,8 @@ def get_random_valid_repo(tree): tree = tree.resolve() valid_repos = [] - for _ in ["mozilla-central", "mozilla-beta"]: - if (tree / _ / ".hg" / "hgrc").is_file(): + for branch in ["mozilla-central", "mozilla-beta"]: + if (tree / branch / ".hg" / "hgrc").is_file(): valid_repos.append(_) # After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen From 4565450cc61eabdb4768bb726b7dd65150fc6f19 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 00:14:04 -0700 Subject: [PATCH 034/110] Follow-up fix. --- src/funfuzz/js/build_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 981faf20f..410123550 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -356,7 +356,7 @@ def get_random_valid_repo(tree): valid_repos = [] for branch in ["mozilla-central", "mozilla-beta"]: if (tree / branch / ".hg" / "hgrc").is_file(): - valid_repos.append(_) + valid_repos.append(branch) # After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen if "mozilla-beta" in valid_repos and chance(0.5): From 983794298520ea7f9a330392d07fcd2bdcfc0ef9 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 00:15:08 -0700 Subject: [PATCH 035/110] Whitespace fixes. --- src/funfuzz/js/compare_jit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index e932df27e..17433eacb 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -178,11 +178,11 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. # pylint: disable=invalid-name rerunCommand = " ".join(quote(str(x)) for x in ["python -m funfuzz.js.compare_jit", - "--flags=" + " ".join(flags), - "--timeout=" + str(options.timeout), - options.knownPath, - jsEngine, - str(infilename.name)]) + "--flags=" + " ".join(flags), + "--timeout=" + str(options.timeout), + options.knownPath, + jsEngine, + str(infilename.name)]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = (" " + " ".join(quote(str(x)) for x in commands[0]) + "\n " + " ".join(quote(str(x)) for x in command) + "\n\n" + summary) From 6e3de9fd1029b2afe5efa6e50926e6d6c936d810 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 22:36:55 -0700 Subject: [PATCH 036/110] Fix/remove invalid comments as indicated during review. --- src/funfuzz/js/build_options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 410123550..e977abde2 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -345,12 +345,12 @@ def get_random_valid_repo(tree): """Given a path to Mozilla Mercurial repositories, return a randomly chosen valid one. Args: - tree (str): Intended location of Mozilla Mercurial repositories + tree (Path): Intended location of Mozilla Mercurial repositories Returns: Path: Location of a valid Mozilla repository """ - assert isinstance(tree, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + assert isinstance(tree, Path) tree = tree.resolve() valid_repos = [] From a45f78e9a8d2eacd1cde35efe4bb0530e3f533a2 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 22:39:55 -0700 Subject: [PATCH 037/110] Remove more comments. --- src/funfuzz/autobisectjs/autobisectjs.py | 2 +- src/funfuzz/js/compare_jit.py | 2 +- src/funfuzz/util/lithium_helpers.py | 1 - src/funfuzz/util/os_ops.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index 2dad188b1..ff2e77dc8 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -498,7 +498,7 @@ def rm_old_local_cached_dirs(cache_dir): Args: cache_dir (Path): Full path to the cache directory """ - assert isinstance(cache_dir, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + assert isinstance(cache_dir, Path) cache_dir = cache_dir.expanduser() # This is in autobisectjs because it has a lock so we do not race while removing directories diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 17433eacb..38c32cc37 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -99,7 +99,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi # we also use it directly for knownPath, timeout, and collector # Return: (lev, crashInfo) or (js_interesting.JS_FINE, None) - assert isinstance(infilename, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + assert isinstance(infilename, Path) combos = shell_flags.basic_flag_sets(jsEngine) diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 1d8978177..090dbe055 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -133,7 +133,6 @@ def reduction_strat(logPrefix, infilename, lithArgs, targetTime, lev): # pylint # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc,too-complex # pylint: disable=too-many-branches,too-many-locals,too-many-statements """Reduce jsfunfuzz output files using Lithium by using various strategies.""" - # RMassert isinstance(cacheDir, Path) # We can remove casting Path to str after moving to Python 3.6+ completely # This is an array because Python does not like assigning to upvars. reductionCount = [0] # pylint: disable=invalid-name diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index a6cfe4c3f..440898890 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -363,7 +363,7 @@ def make_wtmp_dir(base_dir): Returns: Path: Full path to the numbered wtmp directory """ - assert isinstance(base_dir, Path) # We can remove casting Path to str after moving to Python 3.6+ completely + assert isinstance(base_dir, Path) i = 1 while True: From cadb5e6524596151a324d3cf1bc77eb889bd0d18 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 22:43:32 -0700 Subject: [PATCH 038/110] Stop using range and use enumerate instead as suggested in review. --- src/funfuzz/js/compare_jit.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 38c32cc37..2eaad9a27 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -16,7 +16,6 @@ # These pylint errors exist because FuzzManager is not Python 3-compatible yet from FTB.ProgramConfiguration import ProgramConfiguration # pylint: disable=import-error import FTB.Signatures.CrashInfo as CrashInfo # pylint: disable=import-error,no-name-in-module -from past.builtins import range from shellescape import quote from . import js_interesting @@ -112,7 +111,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi commands = [[jsEngine] + combo + [str(infilename)] for combo in combos] - for i in range(0, len(commands)): + for i, command in enumerate(commands): prefix = (logPrefix.parent / ("%s-r%s" % (logPrefix.stem, str(i)))) command = commands[i] r = js_interesting.ShellResult(options, command, prefix, True) # pylint: disable=invalid-name From 999f7024600608c0172201a4009dd0d05dc722c9 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 22:49:47 -0700 Subject: [PATCH 039/110] More pathlib fixes. --- src/funfuzz/js/compare_jit.py | 2 +- src/funfuzz/js/js_interesting.py | 7 ++++--- src/funfuzz/js/loop.py | 2 +- src/funfuzz/util/lithium_helpers.py | 8 ++++---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 2eaad9a27..54329a5a4 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -84,7 +84,7 @@ def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, metadata = {} if autoBisectLog: metadata = {"autoBisectLog": "".join(autoBisectLog)} - options.collector.submit(cl[1], infilename, quality, metaData=metadata) + options.collector.submit(cl[1], str(infilename), quality, metaData=metadata) return True return False diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 2ae95ce94..0fbbd66e7 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -312,10 +312,10 @@ def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missin if len(args) < 2: raise Exception("Not enough positional arguments") options.knownPath = args[0] - options.jsengineWithArgs = args[1:] + options.jsengineWithArgs = [Path(args[1]).resolve()] + args[1:-1] + [Path(args[-1]).resolve()] + assert options.jsengineWithArgs[0].is_file() # js shell + assert options.jsengineWithArgs[-1].is_file() # testcase options.collector = create_collector.make_collector() - if not options.jsengineWithArgs[0].is_file(): - raise Exception("js shell does not exist: " + options.jsengineWithArgs[0]) options.shellIsDeterministic = inspect_shell.queryBuildConfiguration( options.jsengineWithArgs[0], "more-deterministic") @@ -347,6 +347,7 @@ def interesting(_args, tempPrefix): # pylint: disable=invalid-name,missing-docs # For direct, manual use def main(): # pylint: disable=missing-docstring options = parseOptions(sys.argv[1:]) + tempPrefixISNOTAPATH tempPrefix = "m" # pylint: disable=invalid-name res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) # pylint: disable=no-member print(res.lev) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 067372179..3dd1d23cb 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -220,7 +220,7 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in metadata = {} if autoBisectLog: metadata = {"autoBisectLog": "".join(autoBisectLog)} - collector.submit(res.crashInfo, reduced_log, quality, metaData=metadata) + collector.submit(res.crashInfo, str(reduced_log), quality, metaData=metadata) print("Submitted %s" % reduced_log) else: diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 090dbe055..7fe87942c 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -45,7 +45,7 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename (If it's not prepared to accept engineFlags, engineFlags must be empty.) """ - lithArgs = itest + [jsEngine] + engineFlags + [infilename] # pylint: disable=invalid-name + lithArgs = itest + [str(jsEngine)] + engineFlags + [str(infilename)] # pylint: disable=invalid-name (lithResult, lithDetails) = reduction_strat( # pylint: disable=invalid-name logPrefix, infilename, lithArgs, targetTime, suspiciousLevel) @@ -94,7 +94,7 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam # loop is being run standalone lithtmp = logPrefix.parent / (logPrefix.stem + "-lith-tmp") Path.mkdir(lithtmp) - lithArgs = ["--tempdir=" + lithtmp] + lithArgs + lithArgs = ["--tempdir=" + str(lithtmp)] + lithArgs lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) print(" ".join(quote(str(x)) for x in runlithiumpy + lithArgs)) @@ -103,7 +103,7 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam if deletableLithTemp: shutil.rmtree(deletableLithTemp) r = readLithiumResult(lithlogfn) # pylint: disable=invalid-name - subprocess.call(["gzip", "-f", lithlogfn]) + subprocess.call(["gzip", "-f", str(lithlogfn)]) return r @@ -155,7 +155,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis desc = "-chars" if strategy == "--char" else "-lines" (lith_result, lith_details) = run_lithium( # pylint: disable=invalid-name - full_lith_args, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) + full_lith_args, (logPrefix.parent / ("%s-%s%s" % (logPrefix.stem, reductionCount[0], desc))), targetTime) if lith_result == LITH_FINISHED: shutil.copy2(str(infilename), str(backup_file)) From e926b7ba15941b41e9ea291de85b759f0c29d216 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 23:51:12 -0700 Subject: [PATCH 040/110] Remove bad line. --- src/funfuzz/js/js_interesting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 0fbbd66e7..e12972b5d 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -347,7 +347,6 @@ def interesting(_args, tempPrefix): # pylint: disable=invalid-name,missing-docs # For direct, manual use def main(): # pylint: disable=missing-docstring options = parseOptions(sys.argv[1:]) - tempPrefixISNOTAPATH tempPrefix = "m" # pylint: disable=invalid-name res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) # pylint: disable=no-member print(res.lev) From 241f703edeed327996ebcf9a6dd566fb10b86b56 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 30 May 2018 23:57:16 -0700 Subject: [PATCH 041/110] Fix pathlib issue related to get_shell_cache_js_bin_path. --- src/funfuzz/js/compile_shell.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index a67872ebf..cc2beb849 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -732,7 +732,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa if use_s3cache: if s3cache_obj.downloadFile(str(shell.get_shell_name_without_ext() + ".busted"), - str(shell.get_shell_cache_js_bin_path() + ".busted")): + str(shell.get_shell_cache_js_bin_path()) + ".busted"): raise Exception("Found a .busted file for rev " + shell.get_hg_hash()) if s3cache_obj.downloadFile(str(shell.get_shell_name_without_ext() + ".tar.bz2"), @@ -764,7 +764,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa shell.get_shell_cache_dir().mkdir() createBustedFile(cached_no_shell, ex) if use_s3cache: - s3cache_obj.uploadFileToS3(str(shell.get_shell_cache_js_bin_path() + ".busted")) + s3cache_obj.uploadFileToS3(str(shell.get_shell_cache_js_bin_path()) + ".busted") raise finally: if shell.build_opts.patch_file: From 0adf8073021903c5e7c4cb1824e261c75ef79ecc Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:08:34 -0700 Subject: [PATCH 042/110] Yet more pathlib fixes. --- src/funfuzz/js/js_interesting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index e12972b5d..2248a5dcc 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -110,7 +110,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if valgrindErrorPrefix and line.startswith(valgrindErrorPrefix): issues.append(line.rstrip()) elif runinfo.sta == timed_run.CRASHED: - if os_ops.grab_crash_log(runthis[0], runinfo.pid, logPrefix, True): + if os_ops.grab_crash_log(str(runthis[0]), runinfo.pid, logPrefix, True): crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") with open(str(crash_log)) as f: auxCrashData = [line.strip() for line in f.readlines()] @@ -145,7 +145,8 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa extracted_gdb_cmds.append("-ex") extracted_gdb_cmds.append("%s" % line.rstrip()) no_main_log_gdb_log = subprocess.run( - ["gdb", "-n", "-batch"] + extracted_gdb_cmds + ["--args"] + runthis, + (["gdb", "-n", "-batch"] + extracted_gdb_cmds + ["--args"] + + [str(x) if isinstance(x, Path) else x for x in runthis]), check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE From 3d2417587f7ea10111e37b6bf146e73f359befd7 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:29:57 -0700 Subject: [PATCH 043/110] Comment and whitespace changes. --- src/funfuzz/js/compile_shell.py | 10 +++------- src/funfuzz/js/js_interesting.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index cc2beb849..3676c3037 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -7,10 +7,6 @@ """Compiles SpiderMonkey shells on different platforms using various specified configuration parameters. """ -# reset ; rg -g '!*subprocesses.py' -g '!*crashesat.py' -g '!*s3cache.py' -g '!*test_shell_flags.py' -# -g '!*test_compile_shell.py' -g '!*known_broken*.py' -t py "import os$" -# disable no-member pylint messages can be removed after https://github.com/PyCQA/pylint/issues/1660 lands on 1.8 - from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin @@ -76,10 +72,10 @@ class CompiledShellError(Exception): class CompiledShell(object): # pylint: disable=missing-docstring,too-many-instance-attributes,too-many-public-methods def __init__(self, build_opts, hg_hash): self.shell_name_without_ext = build_options.computeShellName(build_opts, hg_hash) - self.hg_hash = hg_hash # pylint: disable=invalid-name + self.hg_hash = hg_hash self.build_opts = build_opts - self.js_objdir = "" # pylint: disable=invalid-name + self.js_objdir = "" self.cfg = "" self.destDir = "" # pylint: disable=invalid-name @@ -158,7 +154,7 @@ def getEnvFull(self): # pylint: disable=invalid-name,missing-docstring,missing- return self.fullEnv def get_hg_hash(self): - """Retrieve the hash of the current changeset of the repository + """Retrieve the hash of the current changeset of the repository. Returns: str: Changeset hash diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 2248a5dcc..3f7b072c5 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -146,7 +146,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa extracted_gdb_cmds.append("%s" % line.rstrip()) no_main_log_gdb_log = subprocess.run( (["gdb", "-n", "-batch"] + extracted_gdb_cmds + ["--args"] + - [str(x) if isinstance(x, Path) else x for x in runthis]), + [str(x) if isinstance(x, Path) else x for x in runthis]), check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE From 9f87bb9cf601d912074b7001277745865f482491 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:31:03 -0700 Subject: [PATCH 044/110] Refactor getCfgCmdExclEnv to get_cfg_cmd_excl_env and setCfgCmdExclEnv to set_cfg_cmd_excl_env. --- src/funfuzz/js/compile_shell.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 3676c3037..af00526b4 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -134,11 +134,20 @@ def run(argv=None): # pylint: disable=missing-param-doc,missing-return-doc,miss return 0 - def getCfgCmdExclEnv(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc + def get_cfg_cmd_excl_env(self): + """Retrieve the configure command excluding the enviroment variables. + + Returns: + list: Configure command + """ return self.cfg - def setCfgCmdExclEnv(self, cfg): # pylint: disable=invalid-name,missing-docstring + def set_cfg_cmd_excl_env(self, cfg): + """Sets the configure command excluding the enviroment variables. + + Args: + cfg (list): Configure command + """ self.cfg = cfg def setEnvAdded(self, addedEnv): # pylint: disable=invalid-name,missing-docstring @@ -535,7 +544,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ shell.setEnvAdded(env_vars) shell.setEnvFull(cfg_env) - shell.setCfgCmdExclEnv(cfg_cmds) + shell.set_cfg_cmd_excl_env(cfg_cmds) def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc @@ -621,7 +630,7 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi f.write("# Full configuration command with needed environment variables is:\n") f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.getEnvAdded()), - " ".join(quote(str(x)) for x in shell.getCfgCmdExclEnv()))) + " ".join(quote(str(x)) for x in shell.get_cfg_cmd_excl_env()))) # .fuzzmanagerconf details f.write("\n") From 32e23c83f64960ae351468c4443c77bab82d3d74 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:31:33 -0700 Subject: [PATCH 045/110] self.cfg should be instantiated as a list in compile_shell. --- src/funfuzz/js/compile_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index af00526b4..ec08282d6 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -77,7 +77,7 @@ def __init__(self, build_opts, hg_hash): self.js_objdir = "" - self.cfg = "" + self.cfg = [] self.destDir = "" # pylint: disable=invalid-name self.addedEnv = "" # pylint: disable=invalid-name self.fullEnv = "" # pylint: disable=invalid-name From 7d4e0d37e7e55462a33a48a658cf134572d1d508 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:36:53 -0700 Subject: [PATCH 046/110] Add a docstring for CompiledShell class. --- src/funfuzz/js/compile_shell.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index ec08282d6..1ed7406ee 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -69,7 +69,13 @@ class CompiledShellError(Exception): pass -class CompiledShell(object): # pylint: disable=missing-docstring,too-many-instance-attributes,too-many-public-methods +class CompiledShell(object): # pylint: disable=too-many-instance-attributes,too-many-public-methods + """A CompiledShell object represents an actual compiled shell binary. + + Args: + build_opts (object): Object containing the build options defined in build_options.py + hg_hash (str): Changeset hash + """ def __init__(self, build_opts, hg_hash): self.shell_name_without_ext = build_options.computeShellName(build_opts, hg_hash) self.hg_hash = hg_hash From e6aca2d9d9d452ea4d836296936ceb975a076a14 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:43:49 -0700 Subject: [PATCH 047/110] Inline updateRepo and remove the function. --- src/funfuzz/js/compile_shell.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 1ed7406ee..62ecc5237 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -759,7 +759,16 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa try: if updateToRev: - updateRepo(shell.build_opts.repo_dir, updateToRev) + # Print *with* a trailing newline to avoid breaking other stuff + print("Updating to rev %s in the %s repository..." % ( + updateToRev.decode("utf-8", errors="replace"), + shell.build_opts.repo_dir.decode("utf-8", errors="replace"))) + subprocess.run(["hg", "-R", shell.build_opts.repo_dir, "update", "-C", "-r", updateToRev], + check=True, + # pylint: disable=no-member + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), + stderr=subprocess.DEVNULL, + timeout=999) if shell.build_opts.patch_file: hg_helpers.patch_hg_repo_with_mq(shell.build_opts.patch_file, shell.get_repo_dir()) @@ -792,18 +801,6 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa shell.get_s3_tar_with_ext_full_path().unlink() -def updateRepo(repo, rev): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Update repository to the specific revision.""" - # Print *with* a trailing newline to avoid breaking other stuff - print("Updating to rev %s in the %s repository..." % (rev.decode("utf-8", errors="replace"), - repo.decode("utf-8", errors="replace"))) - subprocess.run(["hg", "-R", repo, "update", "-C", "-r", rev], - check=True, - cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member - stderr=subprocess.DEVNULL, - timeout=999) - - def verify_full_win_pageheap(shell_path): """Turn on full page heap verification on Windows. From a8aaf2e5f814487670f13f69c9694c1e2ea3e588 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:50:17 -0700 Subject: [PATCH 048/110] fork_join tests should become actual tests for pytest, commenting them out for now. --- src/funfuzz/util/fork_join.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index a8c4dab59..dd651aca8 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -78,19 +78,19 @@ def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=inval # * "Green Chairs" from the first few processes # * A pause and error (with stack trace) from process 5 # * "Green Chairs" again from the rest. -def test_forkJoin(): # pylint: disable=invalid-name,missing-docstring - forkJoin(".", 8, test_forkJoin_inner, "Green", "Chairs") +# def test_forkJoin(): # pylint: disable=invalid-name,missing-docstring +# forkJoin(".", 8, test_forkJoin_inner, "Green", "Chairs") -def test_forkJoin_inner(adj, noun, forkjoin_id): # pylint: disable=invalid-name,missing-docstring - import time - print("%s %s" % (adj, noun)) - print(forkjoin_id) - if forkjoin_id == 5: - time.sleep(1) - raise NameError() +# def test_forkJoin_inner(adj, noun, forkjoin_id): # pylint: disable=invalid-name,missing-docstring +# import time +# print("%s %s" % (adj, noun)) +# print(forkjoin_id) +# if forkjoin_id == 5: +# time.sleep(1) +# raise NameError() -if __name__ == "__main__": - print("test_forkJoin():") - test_forkJoin() +# if __name__ == "__main__": +# print("test_forkJoin():") +# test_forkJoin() From b017d9d4233b46de9c7b79b873c045c5c9a1e0dd Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:51:24 -0700 Subject: [PATCH 049/110] Make a tempfile import top-level and use the backports.tempfile version from PyPI for Python 2. --- src/funfuzz/js/compare_jit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 54329a5a4..38cccf115 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -29,9 +29,11 @@ import subprocess if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module from pathlib2 import Path else: from pathlib import Path # pylint: disable=import-error + import tempfile gOptions = "" # pylint: disable=invalid-name lengthLimit = 1000000 # pylint: disable=invalid-name @@ -299,7 +301,6 @@ def interesting(_args, tempPrefix): # pylint: disable=invalid-name def main(): - import tempfile options = parseOptions(sys.argv[1:]) print(compareLevel( options.jsengine, options.flags, options.infilename, # pylint: disable=no-member From 1eddcfc9d1c7f58777eec9fea57e70c62ea5c3e7 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:52:15 -0700 Subject: [PATCH 050/110] Remove unneeded 'if __name__ == "__main__":' lines. --- src/funfuzz/js/build_options.py | 4 ---- src/funfuzz/js/compare_jit.py | 4 ---- src/funfuzz/js/compile_shell.py | 4 ---- src/funfuzz/js/js_interesting.py | 4 ---- src/funfuzz/js/loop.py | 6 ------ src/funfuzz/loop_bot.py | 4 ---- src/funfuzz/util/repos_update.py | 4 ---- 7 files changed, 30 deletions(-) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index e977abde2..065267c8c 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -381,7 +381,3 @@ def main(): # pylint: disable=missing-docstring print("Running this file directly doesn't do anything, but here's our subparser help:") print() parse_shell_opts("--help") - - -if __name__ == "__main__": - main() diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 38cccf115..477db5b72 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -305,7 +305,3 @@ def main(): print(compareLevel( options.jsengine, options.flags, options.infilename, # pylint: disable=no-member tempfile.mkdtemp("compare_jitmain"), options, True, False)[0]) - - -if __name__ == "__main__": - main() diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 62ecc5237..8492a5368 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -818,7 +818,3 @@ def verify_full_win_pageheap(shell_path): def main(): """Execute main() function in CompiledShell class.""" exit(CompiledShell.main()) - - -if __name__ == "__main__": - main() diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 3f7b072c5..03ac1f4c8 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -359,7 +359,3 @@ def main(): # pylint: disable=missing-docstring options.collector.submit(res.crashInfo, testcaseFilename, quality) # pylint: disable=no-member else: print("Not submitting (not interesting)") - - -if __name__ == "__main__": - main() diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 3dd1d23cb..ecd0b97bd 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -277,9 +277,3 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid "// DDEND\n" ] return lines - - -if __name__ == "__main__": - # pylint: disable=no-member - many_timed_runs(None, sps.make_wtmp_dir(Path(os.getcwdu() if sys.version_info.major == 2 else os.getcwd())), - sys.argv[1:], create_collector.make_collector()) diff --git a/src/funfuzz/loop_bot.py b/src/funfuzz/loop_bot.py index c3980bf1c..75ad41d4e 100644 --- a/src/funfuzz/loop_bot.py +++ b/src/funfuzz/loop_bot.py @@ -45,7 +45,3 @@ def main(): # pylint: disable=missing-docstring [sys.executable, "-u", "-m", "funfuzz.util.repos_update"], [sys.executable, "-u", "-m", "funfuzz.bot"] + sys.argv[1:] ], 60) - - -if __name__ == "__main__": - main() diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index ed53379a3..5aa6bc39e 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -138,7 +138,3 @@ def main(): # pylint: disable=missing-docstring logger.info("WARNING: OSError hit:") logger.info(ex) logger.info(time.asctime()) - - -if __name__ == "__main__": - main() From 58347b516754178141bddf9097671615535e88e8 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:56:34 -0700 Subject: [PATCH 051/110] Remove unneeded imports from loop.py --- src/funfuzz/js/loop.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index ecd0b97bd..4ee01585f 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -20,10 +20,8 @@ from . import js_interesting from . import link_fuzzer from . import shell_flags -from ..util import create_collector from ..util import file_manipulation from ..util import lithium_helpers -from ..util import subprocesses as sps if sys.version_info.major == 2: from pathlib2 import Path From d19d28391154b1e752f0816e1203e59d89a540c2 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 00:56:56 -0700 Subject: [PATCH 052/110] Consolidate subprocess32 import with others in compare_jit.py --- src/funfuzz/js/compare_jit.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 477db5b72..864b0e766 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -23,16 +23,14 @@ from ..util import create_collector from ..util import lithium_helpers -if sys.version_info.major == 2 and os.name == "posix": - import subprocess32 as subprocess # pylint: disable=import-error -else: - import subprocess - if sys.version_info.major == 2: import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module from pathlib2 import Path + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error else: from pathlib import Path # pylint: disable=import-error + import subprocess import tempfile gOptions = "" # pylint: disable=invalid-name From b916d99105b2d9767d39da05f81dcad4ebd526b2 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Thu, 31 May 2018 01:13:16 -0700 Subject: [PATCH 053/110] Rename printMachineInfo to print_machine_info and refactor it a little. --- src/funfuzz/bot.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index ecf4b2398..4c0f2a5fc 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -20,6 +20,8 @@ import tempfile import time +from whichcraft import which + from .js import build_options from .js import compile_shell from .js import loop @@ -107,7 +109,7 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur def main(): # pylint: disable=missing-docstring - printMachineInfo() + print_machine_info() options = parseOpts() @@ -136,28 +138,19 @@ def main(): # pylint: disable=missing-docstring shutil.rmtree(options.tempDir) -def printMachineInfo(): # pylint: disable=invalid-name +def print_machine_info(): """Log information about the machine.""" print("Platform details: %s" % " ".join(platform.uname())) - print("hg version: %s" % - subprocess.run(["hg", "-q", "version"], - check=True, - cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member - stdout=subprocess.PIPE, - timeout=9).stdout.rstrip()) - try: - print("gdb version: %s" % - subprocess.run( - ["gdb", "--version"], - cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - timeout=9).stdout.decode("utf-8", errors="replace").split("\n")[0]) - except (KeyboardInterrupt, subprocess.CalledProcessError) as ex: - print("Error involving gdb is: %r" % (ex,)) - - # FIXME: Should have if which(git) or something # pylint: disable=fixme + + print("hg info: %s" % subprocess.run(["hg", "-q", "version"], check=True, stdout=subprocess.PIPE).stdout.rstrip()) + if which("gdb"): + gdb_version = subprocess.run(["gdb", "--version"], + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + print("gdb info: %s" % gdb_version.split("\n")[0]) + if which("git"): + print("git info: %s" % subprocess.run(["git", "version"], check=True, stdout=subprocess.PIPE).stdout.rstrip()) print("Python version: %s" % sys.version.split()[0]) + print("Number of cores visible to OS: %d" % multiprocessing.cpu_count()) if sys.version_info.major == 2: rootdir_free_space = psutil.disk_usage("/").free / (1024 ** 3) From 598aa3e360c4a31c7dfb663798dec060473c7313 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 14:34:17 -0700 Subject: [PATCH 054/110] Defend against logPrefix not being a Path, e.g. when Lithium uses it, it is likely a string. --- src/funfuzz/js/compare_jit.py | 3 +++ src/funfuzz/js/js_interesting.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 864b0e766..1ac90ebcf 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -58,6 +58,9 @@ def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc # pylint: disable=too-many-arguments,too-many-locals + # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. + if not isinstance(logPrefix, Path): + logPrefix = Path(logPrefix) initialdir_name = (logPrefix.parent / (logPrefix.stem + "-initial")) # pylint: disable=invalid-name cl = compareLevel(jsEngine, flags, infilename, initialdir_name, options, False, True) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 03ac1f4c8..d2ff280a2 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -68,6 +68,10 @@ class ShellResult(object): # pylint: disable=missing-docstring,too-many-instanc # options dict should include: timeout, knownPath, collector, valgrind, shellIsDeterministic def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disable=too-complex,too-many-branches # pylint: disable=too-many-locals,too-many-statements + + # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. + if not isinstance(logPrefix, Path): + logPrefix = Path(logPrefix) pathToBinary = runthis[0].resolve() # pylint: disable=invalid-name # This relies on the shell being a local one from compile_shell: # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like "./js" From a6e704e352e5c46838003838880f6c004aab47a7 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 14:36:23 -0700 Subject: [PATCH 055/110] Rename tempPrefix to cwd_prefix. --- src/funfuzz/js/compare_jit.py | 4 ++-- src/funfuzz/js/js_interesting.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 1ac90ebcf..d82d0a254 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -295,9 +295,9 @@ def init(args): # FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme -def interesting(_args, tempPrefix): # pylint: disable=invalid-name +def interesting(_args, cwd_prefix): actualLevel = compareLevel( # pylint: disable=invalid-name - gOptions.jsengine, gOptions.flags, gOptions.infilename, tempPrefix, gOptions, False, False)[0] + gOptions.jsengine, gOptions.flags, gOptions.infilename, cwd_prefix, gOptions, False, False)[0] return actualLevel >= gOptions.minimumInterestingLevel diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index d2ff280a2..fab46bc92 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -337,13 +337,13 @@ def init(args): # pylint: disable=missing-docstring # FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme -def interesting(_args, tempPrefix): # pylint: disable=invalid-name,missing-docstring,missing-return-doc +def interesting(_args, cwd_prefix): # pylint: disable=missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc options = gOptions # options, runthis, logPrefix, in_compare_jit - res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) - out_log = (tempPrefix.parent / (tempPrefix.stem + "-out")).with_suffix(".txt") - err_log = (tempPrefix.parent / (tempPrefix.stem + "-err")).with_suffix(".txt") + res = ShellResult(options, options.jsengineWithArgs, cwd_prefix, False) + out_log = (cwd_prefix.parent / (cwd_prefix.stem + "-out")).with_suffix(".txt") + err_log = (cwd_prefix.parent / (cwd_prefix.stem + "-err")).with_suffix(".txt") truncateFile(out_log, 1000000) truncateFile(err_log, 1000000) return res.lev >= gOptions.minimumInterestingLevel @@ -352,8 +352,8 @@ def interesting(_args, tempPrefix): # pylint: disable=invalid-name,missing-docs # For direct, manual use def main(): # pylint: disable=missing-docstring options = parseOptions(sys.argv[1:]) - tempPrefix = "m" # pylint: disable=invalid-name - res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) # pylint: disable=no-member + cwd_prefix = "m" + res = ShellResult(options, options.jsengineWithArgs, cwd_prefix, False) # pylint: disable=no-member print(res.lev) if options.submit: # pylint: disable=no-member if res.lev >= options.minimumInterestingLevel: # pylint: disable=no-member From dc5781474972fb9559bb8bfc497839cc770050cd Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 14:36:47 -0700 Subject: [PATCH 056/110] Make cwd_prefix explicit in using Path.cwd() when it is used. --- src/funfuzz/js/js_interesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index fab46bc92..3ab609e7e 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -352,7 +352,7 @@ def interesting(_args, cwd_prefix): # pylint: disable=missing-docstring,missing # For direct, manual use def main(): # pylint: disable=missing-docstring options = parseOptions(sys.argv[1:]) - cwd_prefix = "m" + cwd_prefix = Path.cwd() / "m" res = ShellResult(options, options.jsengineWithArgs, cwd_prefix, False) # pylint: disable=no-member print(res.lev) if options.submit: # pylint: disable=no-member From c96dba7d95cb6f11e422fbeda6e3692aca55f97c Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 14:53:28 -0700 Subject: [PATCH 057/110] Convert remaining subprocess calls to use subprocess.run() instead. --- src/funfuzz/autobisectjs/autobisectjs.py | 10 +++++----- src/funfuzz/js/compile_shell.py | 11 +++++++---- src/funfuzz/js/loop.py | 6 ++++-- src/funfuzz/loop_bot.py | 14 ++++++++++---- src/funfuzz/util/hg_helpers.py | 8 ++++---- src/funfuzz/util/lithium_helpers.py | 13 +++++++++---- src/funfuzz/util/os_ops.py | 6 +++--- 7 files changed, 42 insertions(+), 26 deletions(-) diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index ff2e77dc8..0787937f8 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -196,9 +196,9 @@ def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name, # Refresh source directory (overwrite all local changes) to default tip if required. if options.resetRepoFirst: - subprocess.check_call(hgPrefix + ["update", "-C", "default"]) + subprocess.run(hgPrefix + ["update", "-C", "default"], check=True) # Throws exit code 255 if purge extension is not enabled in .hgrc: - subprocess.check_call(hgPrefix + ["purge", "--all"]) + subprocess.run(hgPrefix + ["purge", "--all"], check=True) # Reset bisect ranges and set skip ranges. subprocess.run(hgPrefix + ["bisect", "-r"], @@ -218,7 +218,7 @@ def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name, else: labels[sRepo] = ("good", "assumed start rev is good") labels[eRepo] = ("bad", "assumed end rev is bad") - subprocess.check_call(hgPrefix + ["bisect", "-U", "-g", sRepo]) + subprocess.run(hgPrefix + ["bisect", "-U", "-g", sRepo], check=True) mid_bisect_output = subprocess.run( hgPrefix + ["bisect", "-U", "-b", eRepo], check=True, @@ -273,7 +273,7 @@ def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name, realEndRepo) sps.vdump("Resetting bisect") - subprocess.check_call(hgPrefix + ["bisect", "-U", "-r"]) + subprocess.run(hgPrefix + ["bisect", "-U", "-r"], check=True) sps.vdump("Resetting working directory") subprocess.run(hgPrefix + ["update", "-C", "-r", "default"], @@ -475,7 +475,7 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl currRev = hg_helpers.get_cset_hash_from_bisect_msg(outputLines[0]) if currRev is None: print_("Resetting to default revision...", flush=True) - subprocess.check_call(hgPrefix + ["update", "-C", "default"]) + subprocess.run(hgPrefix + ["update", "-C", "default"], check=True) hg_helpers.destroyPyc(repo_dir) raise Exception("hg did not suggest a changeset to test!") diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 8492a5368..019d4e3c8 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -317,7 +317,7 @@ def autoconf_run(working_dir): # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 if not Path(autoconf213_mac_bin).is_file(): autoconf213_mac_bin = "autoconf213" - subprocess.check_call([autoconf213_mac_bin], cwd=str(working_dir)) + subprocess.run([autoconf213_mac_bin], check=True, cwd=str(working_dir)) elif platform.system() == "Linux": if which("autoconf2.13"): subprocess.run(["autoconf2.13"], check=True, cwd=str(working_dir)) @@ -327,7 +327,7 @@ def autoconf_run(working_dir): subprocess.run(["autoconf213"], check=True, cwd=str(working_dir)) elif platform.system() == "Windows": # Windows needs to call sh to be able to find autoconf. - subprocess.check_call(["sh", "autoconf-2.13"], cwd=str(working_dir)) + subprocess.run(["sh", "autoconf-2.13"], check=True, cwd=str(working_dir)) def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc @@ -811,8 +811,11 @@ def verify_full_win_pageheap(shell_path): # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member - print(subprocess.check_output([str(gflags_bin_path).decode("utf-8", errors="replace"), - "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"])) + print(subprocess.run([str(gflags_bin_path).decode("utf-8", errors="replace"), + "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"], + check=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout) def main(): diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 4ee01585f..5bc2c48c2 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -12,7 +12,6 @@ import json from optparse import OptionParser # pylint: disable=deprecated-module import os -import subprocess import sys import time @@ -24,9 +23,12 @@ from ..util import lithium_helpers if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error from pathlib2 import Path else: from pathlib import Path # pylint: disable=import-error + import subprocess def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -86,7 +88,7 @@ def showtail(filename): # pylint: disable=missing-docstring cmd.append(str(filename)) print(" ".join(cmd)) print() - subprocess.check_call(cmd) + subprocess.run(cmd, check=True) print() print() diff --git a/src/funfuzz/loop_bot.py b/src/funfuzz/loop_bot.py index 75ad41d4e..ebfd5f0ab 100644 --- a/src/funfuzz/loop_bot.py +++ b/src/funfuzz/loop_bot.py @@ -7,18 +7,24 @@ """Loop of { update repos, call bot } to allow things to run unattended All command-line options are passed through to bot -Since this script updates the fuzzing repo, it should be very simple, and use subprocess.call() rather than import +This script used to update funfuzz itself (when run as scripts, no longer supported) +so it uses subprocess.run() rather than import Config-ish bits should move to bot, OR move into a config file, -OR this file should subprocess-call ITSELF rather than using a while loop. +OR this file should subprocess-run ITSELF rather than using a while loop. """ from __future__ import absolute_import, print_function # isort:skip -import subprocess +import os import sys import time +if sys.version_info.major == 2 and os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + import subprocess + def loop_seq(cmd_seq, wait_time): # pylint: disable=missing-param-doc,missing-type-doc """Call a sequence of commands in a loop. @@ -29,7 +35,7 @@ def loop_seq(cmd_seq, wait_time): # pylint: disable=missing-param-doc,missing-t print("localLoop #%d!" % i) for cmd in cmd_seq: try: - subprocess.check_call(cmd) + subprocess.run(cmd, check=True) except subprocess.CalledProcessError as ex: print("Something went wrong when calling: %r" % (cmd,)) print("%r" % (ex,)) diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index b19041edb..f678bf787 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -147,7 +147,7 @@ def get_repo_hash_and_id(repo_dir, repo_rev="parents() and default"): print("Aborting...") sys.exit(0) elif update_default == "d": - subprocess.check_call(["hg", "-R", str(repo_dir), "update", "default"]) + subprocess.run(["hg", "-R", str(repo_dir), "update", "default"], check=True) is_on_default = True elif update_default == "u": hg_log_template_cmds = ["hg", "-R", str(repo_dir), "log", "-r", "parents()", "--template", @@ -229,8 +229,8 @@ def patch_hg_repo_with_mq(patch_file, repo_dir=None): qpop_qrm_applied_patch(patch_file, repo_dir) print("You may have untracked .rej or .orig files in the repository.") print("`hg status` output of the repository of interesting files in %s :" % repo_dir) - subprocess.check_call(["hg", "-R", str(repo_dir), "status", "--modified", "--added", - "--removed", "--deleted"]) + subprocess.run(["hg", "-R", str(repo_dir), "status", "--modified", "--added", + "--removed", "--deleted"], check=True) raise OSError("Return code from `hg qpush` is: " + str(qpush_return_code)) print("Patch qpush'ed. Continuing...", end=" ") @@ -259,5 +259,5 @@ def qpop_qrm_applied_patch(patch_file, repo_dir): raise OSError("Return code from `hg qpop` is: " + str(qpop_return_code)) print("Patch qpop'ed...", end=" ") - subprocess.check_call(["hg", "-R", str(repo_dir), "qdelete", patch_file.name]) + subprocess.run(["hg", "-R", str(repo_dir), "qdelete", patch_file.name], check=True) print("Patch qdelete'd.") diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 7fe87942c..a964d2831 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -9,9 +9,9 @@ from __future__ import absolute_import, print_function # isort:skip +import os import re import shutil -import subprocess import sys import tempfile @@ -25,9 +25,12 @@ from ..js.js_interesting import JS_VG_AMISS if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error from pathlib2 import Path else: from pathlib import Path # pylint: disable=import-error + import subprocess runlithiumpy = [sys.executable, "-u", "-m", "lithium"] # pylint: disable=invalid-name @@ -66,7 +69,8 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ) print(" ".join(quote(str(x)) for x in autobisectCmd)) autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") - subprocess.call(autobisectCmd, stdout=open(str(autobisect_log), "w"), stderr=subprocess.STDOUT) + with open(str(autobisect_log), "w") as f: + subprocess.run(autobisectCmd, stderr=subprocess.STDOUT, stdout=f) print("Done running autobisectjs. Log: %s" % autobisect_log) with open(str(autobisect_log), "r") as f: @@ -98,12 +102,13 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) print(" ".join(quote(str(x)) for x in runlithiumpy + lithArgs)) - subprocess.call(runlithiumpy + lithArgs, stdout=open(str(lithlogfn), "w"), stderr=subprocess.STDOUT) + with open(str(lithlogfn), "w") as f: + subprocess.run(runlithiumpy + lithArgs, stderr=subprocess.STDOUT, stdout=f) print("Done running Lithium") if deletableLithTemp: shutil.rmtree(deletableLithTemp) r = readLithiumResult(lithlogfn) # pylint: disable=invalid-name - subprocess.call(["gzip", "-f", str(lithlogfn)]) + subprocess.run(["gzip", "-f", str(lithlogfn)], check=True) return r diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index 440898890..b4b546ac2 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -184,7 +184,7 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): sps.vdump(" ".join(dbggr_cmd)) core_file = Path(dbggr_cmd[-1]) assert core_file.is_file() - dbbgr_exit_code = subprocess.call( + dbbgr_exit_code = subprocess.run( dbggr_cmd, stdin=None, stderr=subprocess.STDOUT, @@ -200,9 +200,9 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): if use_logfiles: if core_file.is_file(): shutil.move(str(core_file), str(core_file)) - subprocess.call(["gzip", "-f", str(core_file)]) + subprocess.run(["gzip", "-f", str(core_file)], check=True) # chmod here, else the uploaded -core.gz files do not have sufficient permissions. - subprocess.check_call(["chmod", "og+r", "%s.gz" % core_file]) + subprocess.run(["chmod", "og+r", "%s.gz" % core_file], check=True) return str(crash_log) else: print("I don't know what to do with a core file when log_prefix is null") From 851227e942d5eae423673fe9ac36db669c593db5 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 15:08:02 -0700 Subject: [PATCH 058/110] Refactor getEnvAdded to get_env_added and setEnvAdded to set_env_added. --- src/funfuzz/js/compile_shell.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 019d4e3c8..9fa55026e 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -85,7 +85,7 @@ def __init__(self, build_opts, hg_hash): self.cfg = [] self.destDir = "" # pylint: disable=invalid-name - self.addedEnv = "" # pylint: disable=invalid-name + self.added_env = "" self.fullEnv = "" # pylint: disable=invalid-name self.js_cfg_file = "" # pylint: disable=invalid-name @@ -156,11 +156,21 @@ def set_cfg_cmd_excl_env(self, cfg): """ self.cfg = cfg - def setEnvAdded(self, addedEnv): # pylint: disable=invalid-name,missing-docstring - self.addedEnv = addedEnv + def set_env_added(self, added_env): + """Set environment variables that were added. - def getEnvAdded(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.addedEnv + Args: + added_env (list): Added environment variables + """ + self.added_env = added_env + + def get_env_added(self): + """Retrieve environment variables that were added. + + Returns: + list: Added environment variables + """ + return self.added_env def setEnvFull(self, fullEnv): # pylint: disable=invalid-name,missing-docstring self.fullEnv = fullEnv @@ -548,7 +558,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ # We could save the stdout here into a file if it throws - shell.setEnvAdded(env_vars) + shell.set_env_added(env_vars) shell.setEnvFull(cfg_env) shell.set_cfg_cmd_excl_env(cfg_cmds) @@ -635,7 +645,7 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi f.write("# %s\n# \n" % str(shell.getEnvFull())) f.write("# Full configuration command with needed environment variables is:\n") - f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.getEnvAdded()), + f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.get_env_added()), " ".join(quote(str(x)) for x in shell.get_cfg_cmd_excl_env()))) # .fuzzmanagerconf details From e35ba2d6a0d703eec7fb5bb68005f1375004b993 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 15:32:38 -0700 Subject: [PATCH 059/110] Fix potential issue in get_shell_compiled_runlibs_path. --- src/funfuzz/js/compile_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 9fa55026e..32652122d 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -266,7 +266,7 @@ def get_shell_compiled_runlibs_path(self): Path: Full path to the original location of the libraries of js binary compiled in the shell cache """ return [ - self.get_js_objdir() / "dist" / "bin" / runlib for runlib in inspect_shell.ALL_RUN_LIBS + self.get_js_objdir() / "dist" / "bin" / (runlib for runlib in inspect_shell.ALL_RUN_LIBS) ] def get_shell_name_with_ext(self): From 341a7d01d8e78c651099f1867ec15ca0f852b247 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 15:33:34 -0700 Subject: [PATCH 060/110] We must call Path on a path, not potentially a symlink. --- src/funfuzz/js/compile_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 32652122d..c27336e8e 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -325,7 +325,7 @@ def autoconf_run(working_dir): if platform.system() == "Darwin": autoconf213_mac_bin = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" if which("brew") else "autoconf213" # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 - if not Path(autoconf213_mac_bin).is_file(): + if not Path(which(autoconf213_mac_bin)).is_file(): autoconf213_mac_bin = "autoconf213" subprocess.run([autoconf213_mac_bin], check=True, cwd=str(working_dir)) elif platform.system() == "Linux": From 6174041852aa2d0d1bf52b15aae10d4ce35aecaf Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 15:34:12 -0700 Subject: [PATCH 061/110] Add tests for autoconf_run and ensure_cache_dir. --- tests/js/test_compile_shell.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index a5caee9fa..71e9ab02d 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -37,6 +37,18 @@ class CompileShellTests(unittest.TestCase): mc_hg_repo = Path.home() / "trees" / "mozilla-central" shell_cache = Path.home() / "shell-cache" + def test_autoconf_run(): # pylint: disable=no-method-argument + """Test the autoconf runs properly.""" + with tempfile.TemporaryDirectory(suffix="autoconf_run_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + + (tmp_dir / "configure.in").touch() # configure.in is required by autoconf2.13 + js.compile_shell.autoconf_run(tmp_dir) + + def test_ensure_cache_dir(self): + """Test the shell-cache dir is created properly if it does not exist.""" + self.assertTrue(js.compile_shell.ensure_cache_dir().is_dir()) + @pytest.mark.slow @lru_cache(maxsize=None) def test_shell_compile(self): From 31ca67c75ea1f22236bfe817fb457024e8c8df4b Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 15:44:17 -0700 Subject: [PATCH 062/110] More documentation for functions in the CompiledShell class. --- src/funfuzz/js/compile_shell.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index c27336e8e..fe708269e 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -93,7 +93,15 @@ def __init__(self, build_opts, hg_hash): self.jsVersion = "" # pylint: disable=invalid-name @classmethod - def main(cls, args=None): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc + def main(cls, args=None): + """Main function of CompiledShell class. + + Args: + args (object): Additional parameters + + Returns: + int: 0, to denote a successful compile and 1, to denote a failed compile + """ # logging.basicConfig(format="%(message)s", level=logging.INFO) try: @@ -105,8 +113,15 @@ def main(cls, args=None): # pylint: disable=missing-docstring,missing-return-do return 1 @staticmethod - def run(argv=None): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc - """Build a shell and place it in the autobisectjs cache.""" + def run(argv=None): + """Build a shell and place it in the autobisectjs cache. + + Args: + argv (object): Additional parameters + + Returns: + int: 0, to denote a successful compile + """ usage = "Usage: %prog [options]" parser = OptionParser(usage) parser.disable_interspersed_args() From 55243dd999da44975a68e35e99ece92778cfcdf6 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 15:55:45 -0700 Subject: [PATCH 063/110] Add more documentation for compare_jit function in compare_jit. --- src/funfuzz/js/compare_jit.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index d82d0a254..eb97b3f8c 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -52,12 +52,13 @@ def ignoreSomeOfStderr(e): # pylint: disable=invalid-name,missing-docstring,mis return lines -# For use by loop -# Returns True if any kind of bug is found -def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, targetTime, options): - # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - # pylint: disable=too-many-arguments,too-many-locals +def compare_jit(jsEngine, # pylint: disable=invalid-name,too-many-arguments,too-many-locals + flags, infilename, logPrefix, repo, build_options_str, targetTime, options): + """For use in loop.py + Returns: + bool: True if any kind of bug is found, otherwise False + """ # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. if not isinstance(logPrefix, Path): logPrefix = Path(logPrefix) From cf5230a3503ea1638618889e9a5eb79a55e727fc Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:05:01 -0700 Subject: [PATCH 064/110] Fix missing imports. --- tests/js/test_compile_shell.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index 71e9ab02d..0a69ae3d8 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -20,11 +20,13 @@ from funfuzz import util if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module from functools32 import lru_cache # pylint: disable=import-error from pathlib2 import Path else: from functools import lru_cache # pylint: disable=no-name-in-module from pathlib import Path # pylint: disable=import-error + import tempfile FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") logging.basicConfig(level=logging.DEBUG) From 067849d0cc7152ef4fd4cfeb890846d68ea463ec Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:42:57 -0700 Subject: [PATCH 065/110] Remove obsolete pylint ignore line. --- src/funfuzz/js/compile_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index fe708269e..abfab4f99 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -87,7 +87,7 @@ def __init__(self, build_opts, hg_hash): self.destDir = "" # pylint: disable=invalid-name self.added_env = "" self.fullEnv = "" # pylint: disable=invalid-name - self.js_cfg_file = "" # pylint: disable=invalid-name + self.js_cfg_file = "" self.jsMajorVersion = "" # pylint: disable=invalid-name self.jsVersion = "" # pylint: disable=invalid-name From bec04610afbbb6d3c1bf0140431ec6f0b4b8d9a6 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:12:46 -0700 Subject: [PATCH 066/110] Re-add some pylint ignores for the compare_jit function. --- src/funfuzz/js/compare_jit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index eb97b3f8c..4d4abe1dc 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -52,13 +52,14 @@ def ignoreSomeOfStderr(e): # pylint: disable=invalid-name,missing-docstring,mis return lines -def compare_jit(jsEngine, # pylint: disable=invalid-name,too-many-arguments,too-many-locals +def compare_jit(jsEngine, # pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-many-arguments flags, infilename, logPrefix, repo, build_options_str, targetTime, options): """For use in loop.py Returns: bool: True if any kind of bug is found, otherwise False """ + # pylint: disable=too-many-locals # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. if not isinstance(logPrefix, Path): logPrefix = Path(logPrefix) From 271eef884d2884b018b8affac1c31c1956cf34aa Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:09:05 -0700 Subject: [PATCH 067/110] Refactor ignoreSomeOfStderr to ignore_some_stderr and fix pylint issues. --- src/funfuzz/js/compare_jit.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 4d4abe1dc..367e8c610 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -37,9 +37,17 @@ lengthLimit = 1000000 # pylint: disable=invalid-name -def ignoreSomeOfStderr(e): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc +def ignore_some_stderr(err_inp): + """Ignores parts of a list depending on whether they are needed. + + Args: + err_inp (list): Stderr + + Returns: + list: Stderr with potentially some lines removed + """ lines = [] - for line in e: + for line in err_inp: if line.endswith("malloc: enabling scribbling to detect mods to free blocks"): # MallocScribble prints a line that includes the process's pid. # We don't want to include that pid in the comparison! @@ -122,7 +130,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi r = js_interesting.ShellResult(options, command, prefix, True) # pylint: disable=invalid-name oom = js_interesting.oomed(r.err) - r.err = ignoreSomeOfStderr(r.err) + r.err = ignore_some_stderr(r.err) if (r.return_code == 1 or r.return_code == 2) and (anyLineContains(r.out, "[[script] scriptArgs*]") or ( anyLineContains(r.err, "[scriptfile] [scriptarg...]"))): From 580ef83328856ee55239f7039f191b2a5d0c1839 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 18 Apr 2018 18:48:21 -0700 Subject: [PATCH 068/110] Remove print_function from files that do not have any print functions. --- src/funfuzz/autobisectjs/autobisectjs.py | 2 +- src/funfuzz/autobisectjs/known_broken_earliest_working.py | 2 +- src/funfuzz/js/inspect_shell.py | 2 +- src/funfuzz/js/link_fuzzer.py | 2 +- src/funfuzz/js/shell_flags.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index 0787937f8..1089a7cad 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -7,7 +7,7 @@ """autobisectjs, for bisecting changeset regression windows. Supports Mercurial repositories and SpiderMonkey only. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import from optparse import OptionParser # pylint: disable=deprecated-module import os diff --git a/src/funfuzz/autobisectjs/known_broken_earliest_working.py b/src/funfuzz/autobisectjs/known_broken_earliest_working.py index d78703840..3fe8acd24 100644 --- a/src/funfuzz/autobisectjs/known_broken_earliest_working.py +++ b/src/funfuzz/autobisectjs/known_broken_earliest_working.py @@ -7,7 +7,7 @@ """Known broken changeset ranges of SpiderMonkey are specified in this file. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import import os import platform diff --git a/src/funfuzz/js/inspect_shell.py b/src/funfuzz/js/inspect_shell.py index e5670b2aa..e147dfbaf 100644 --- a/src/funfuzz/js/inspect_shell.py +++ b/src/funfuzz/js/inspect_shell.py @@ -7,7 +7,7 @@ """Allows inspection of the SpiderMonkey shell to ensure that it is compiled as intended with specified configurations. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import import json import os diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index d0497f7d9..c9941412f 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -7,7 +7,7 @@ """Concatenate js files to create jsfunfuzz. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import import sys diff --git a/src/funfuzz/js/shell_flags.py b/src/funfuzz/js/shell_flags.py index bacff1b43..c1b9c3e51 100644 --- a/src/funfuzz/js/shell_flags.py +++ b/src/funfuzz/js/shell_flags.py @@ -7,7 +7,7 @@ """Allows detection of support for various command-line flags. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import import multiprocessing import random From f8ddf1480717e6408ea8dcfaf9090c08eda0ef4b Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Wed, 18 Apr 2018 18:50:45 -0700 Subject: [PATCH 069/110] Add unicode_literals to the future import line of all files. --- src/funfuzz/autobisectjs/autobisectjs.py | 2 +- src/funfuzz/autobisectjs/known_broken_earliest_working.py | 2 +- src/funfuzz/bot.py | 2 +- src/funfuzz/js/build_options.py | 2 +- src/funfuzz/js/compare_jit.py | 2 +- src/funfuzz/js/inspect_shell.py | 2 +- src/funfuzz/js/js_interesting.py | 2 +- src/funfuzz/js/loop.py | 2 +- src/funfuzz/js/shell_flags.py | 2 +- src/funfuzz/loop_bot.py | 2 +- src/funfuzz/util/crashesat.py | 2 +- src/funfuzz/util/create_collector.py | 2 +- src/funfuzz/util/file_manipulation.py | 2 +- src/funfuzz/util/fork_join.py | 2 +- src/funfuzz/util/hg_helpers.py | 2 +- src/funfuzz/util/lithium_helpers.py | 2 +- src/funfuzz/util/lock_dir.py | 2 +- src/funfuzz/util/repos_update.py | 2 +- src/funfuzz/util/s3cache.py | 2 +- src/funfuzz/util/subprocesses.py | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index 1089a7cad..0b4effb20 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -7,7 +7,7 @@ """autobisectjs, for bisecting changeset regression windows. Supports Mercurial repositories and SpiderMonkey only. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip from optparse import OptionParser # pylint: disable=deprecated-module import os diff --git a/src/funfuzz/autobisectjs/known_broken_earliest_working.py b/src/funfuzz/autobisectjs/known_broken_earliest_working.py index 3fe8acd24..01c7bb97b 100644 --- a/src/funfuzz/autobisectjs/known_broken_earliest_working.py +++ b/src/funfuzz/autobisectjs/known_broken_earliest_working.py @@ -7,7 +7,7 @@ """Known broken changeset ranges of SpiderMonkey are specified in this file. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip import os import platform diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 4c0f2a5fc..194f61bab 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -8,7 +8,7 @@ """ -from __future__ import absolute_import, division, print_function # isort:skip +from __future__ import absolute_import, division, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin import multiprocessing diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 065267c8c..97b4c2858 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -7,7 +7,7 @@ """Allows specification of build configuration parameters. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import argparse from builtins import object # pylint: disable=redefined-builtin diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 367e8c610..a57469610 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -7,7 +7,7 @@ """Test comparing the output of SpiderMonkey using various flags (usually JIT-related). """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from optparse import OptionParser # pylint: disable=deprecated-module import os diff --git a/src/funfuzz/js/inspect_shell.py b/src/funfuzz/js/inspect_shell.py index e147dfbaf..ea393fad6 100644 --- a/src/funfuzz/js/inspect_shell.py +++ b/src/funfuzz/js/inspect_shell.py @@ -7,7 +7,7 @@ """Allows inspection of the SpiderMonkey shell to ensure that it is compiled as intended with specified configurations. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip import json import os diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 3ab609e7e..c7d6a9e97 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -7,7 +7,7 @@ """Check whether a testcase causes an interesting result in a shell. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin from optparse import OptionParser # pylint: disable=deprecated-module diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 5bc2c48c2..fa5a33ad2 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -7,7 +7,7 @@ """Allows the funfuzz harness to run continuously. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import json from optparse import OptionParser # pylint: disable=deprecated-module diff --git a/src/funfuzz/js/shell_flags.py b/src/funfuzz/js/shell_flags.py index c1b9c3e51..2d70d96b5 100644 --- a/src/funfuzz/js/shell_flags.py +++ b/src/funfuzz/js/shell_flags.py @@ -7,7 +7,7 @@ """Allows detection of support for various command-line flags. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip import multiprocessing import random diff --git a/src/funfuzz/loop_bot.py b/src/funfuzz/loop_bot.py index ebfd5f0ab..3431773cc 100644 --- a/src/funfuzz/loop_bot.py +++ b/src/funfuzz/loop_bot.py @@ -14,7 +14,7 @@ OR this file should subprocess-run ITSELF rather than using a while loop. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import os import sys diff --git a/src/funfuzz/util/crashesat.py b/src/funfuzz/util/crashesat.py index 3ce8e9bb4..bde87f9fd 100644 --- a/src/funfuzz/util/crashesat.py +++ b/src/funfuzz/util/crashesat.py @@ -10,7 +10,7 @@ Not merged into Lithium as it still relies on grab_crash_log. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip import argparse import logging diff --git a/src/funfuzz/util/create_collector.py b/src/funfuzz/util/create_collector.py index d6011c70d..11953d35c 100644 --- a/src/funfuzz/util/create_collector.py +++ b/src/funfuzz/util/create_collector.py @@ -7,7 +7,7 @@ """Functions here make use of a Collector created from FuzzManager. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import sys diff --git a/src/funfuzz/util/file_manipulation.py b/src/funfuzz/util/file_manipulation.py index 4d242dddc..95fd16263 100644 --- a/src/funfuzz/util/file_manipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -7,7 +7,7 @@ """Functions dealing with files and their contents. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip def amiss(log_prefix): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index dd651aca8..38d11d5c0 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -7,7 +7,7 @@ """Functions dealing with multiple processes. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import multiprocessing import sys diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index f678bf787..24400e53b 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -7,7 +7,7 @@ """Helper functions involving Mercurial (hg). """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import input # pylint: disable=redefined-builtin import configparser diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index a964d2831..8905e5537 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -7,7 +7,7 @@ """Helper functions to use the Lithium reducer. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import os import re diff --git a/src/funfuzz/util/lock_dir.py b/src/funfuzz/util/lock_dir.py index 9b0297e77..405e155b4 100644 --- a/src/funfuzz/util/lock_dir.py +++ b/src/funfuzz/util/lock_dir.py @@ -8,7 +8,7 @@ released. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index 5aa6bc39e..2126c2009 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -10,7 +10,7 @@ Assumes that the repositories are located in ../../trees/*. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip from copy import deepcopy import logging diff --git a/src/funfuzz/util/s3cache.py b/src/funfuzz/util/s3cache.py index ce3d12124..bc7c1e25a 100644 --- a/src/funfuzz/util/s3cache.py +++ b/src/funfuzz/util/s3cache.py @@ -7,7 +7,7 @@ """Functions here interact with Amazon EC2 using boto. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin import os diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index e15447d15..35cac7640 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -7,7 +7,7 @@ """Miscellaneous helper functions. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import errno import os From cc5611a4fa6d9ed7d112bfd793dcf0f084d9fa1d Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:21:45 -0700 Subject: [PATCH 070/110] Refactor getEnvFull to get_env_full and setEnvFull to set_env_full. --- src/funfuzz/js/compile_shell.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index abfab4f99..996bd9cba 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -86,7 +86,7 @@ def __init__(self, build_opts, hg_hash): self.cfg = [] self.destDir = "" # pylint: disable=invalid-name self.added_env = "" - self.fullEnv = "" # pylint: disable=invalid-name + self.full_env = "" self.js_cfg_file = "" self.jsMajorVersion = "" # pylint: disable=invalid-name @@ -187,11 +187,21 @@ def get_env_added(self): """ return self.added_env - def setEnvFull(self, fullEnv): # pylint: disable=invalid-name,missing-docstring - self.fullEnv = fullEnv + def set_env_full(self, full_env): + """Set the full environment including the newly added variables. - def getEnvFull(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.fullEnv + Args: + full_env (list): Full environment + """ + self.full_env = full_env + + def get_env_full(self): + """Retrieve the full environment including the newly added variables. + + Returns: + list: Full environment + """ + return self.full_env def get_hg_hash(self): """Retrieve the hash of the current changeset of the repository. @@ -574,7 +584,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ # We could save the stdout here into a file if it throws shell.set_env_added(env_vars) - shell.setEnvFull(cfg_env) + shell.set_env_full(cfg_env) shell.set_cfg_cmd_excl_env(cfg_cmds) @@ -584,7 +594,7 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- cmd_list = [MAKE_BINARY, "-C", str(shell.get_js_objdir()), "-j" + str(COMPILATION_JOBS), "-s"] out = subprocess.run(cmd_list, cwd=str(shell.get_js_objdir()), - env=shell.getEnvFull(), + env=shell.get_env_full(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") except Exception as ex: # pylint: disable=broad-except @@ -595,7 +605,7 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- print("Trying once more due to the compiler running out of memory...") out = subprocess.run(cmd_list, cwd=str(shell.get_js_objdir()), - env=shell.getEnvFull(), + env=shell.get_env_full(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") # A non-zero error can be returned during make, but eventually a shell still gets compiled. @@ -657,7 +667,7 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi shell.build_opts.build_options_str, shell.get_hg_hash())) f.write("# Full environment is:\n") - f.write("# %s\n# \n" % str(shell.getEnvFull())) + f.write("# %s\n# \n" % str(shell.get_env_full())) f.write("# Full configuration command with needed environment variables is:\n") f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.get_env_added()), From 999b741e61fddd07f7fff94707a29a3e719a6db0 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:27:09 -0700 Subject: [PATCH 071/110] Refactor getRepoName to get_repo_name. --- src/funfuzz/bot.py | 4 ++-- src/funfuzz/js/compile_shell.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 194f61bab..d110e4796 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -207,7 +207,7 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss options.build_options.build_options_str, options.build_options.repo_dir, bRev, - cshell.getRepoName(), + cshell.get_repo_name(), time.asctime() )) @@ -242,6 +242,6 @@ def mtrArgsCreation(options, cshell): # pylint: disable=invalid-name,missing-pa # Ordering of elements in manyTimedRunArgs is important. manyTimedRunArgs.append(str(options.timeout)) - manyTimedRunArgs.append(cshell.getRepoName()) # known bugs' directory + manyTimedRunArgs.append(cshell.get_repo_name()) # known bugs' directory manyTimedRunArgs.append(cshell.get_shell_cache_js_bin_path()) return manyTimedRunArgs diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 996bd9cba..619fee961 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -244,7 +244,12 @@ def get_repo_dir(self): """ return self.build_opts.repo_dir - def getRepoName(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + def get_repo_name(self): + """Retrieve the name of a Mercurial repository. + + Returns: + str: Name of the repository + """ return hg_helpers.hgrc_repo_name(self.build_opts.repo_dir) def getS3TarballWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc @@ -677,7 +682,7 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi f.write("\n") f.write("[Main]\n") f.write("platform = %s\n" % fmconf_platform) - f.write("product = %s\n" % shell.getRepoName()) + f.write("product = %s\n" % shell.get_repo_name()) f.write("product_version = %s\n" % shell.get_hg_hash()) f.write("os = %s\n" % fmconf_os) From 5a39f38ae8df74d72f07170d0928722803a1714b Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:31:53 -0700 Subject: [PATCH 072/110] Refactor getS3TarballWithExt to get_s3_tar_name_with_ext. --- src/funfuzz/js/compile_shell.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 619fee961..50b33d350 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -252,8 +252,12 @@ def get_repo_name(self): """ return hg_helpers.hgrc_repo_name(self.build_opts.repo_dir) - def getS3TarballWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc + def get_s3_tar_name_with_ext(self): + """Retrieve the name of the compressed shell tarball to be obtained from/sent to S3. + + Returns: + str: Name of the tarball + """ return self.get_shell_name_without_ext() + ".tar.bz2" def get_s3_tar_with_ext_full_path(self): @@ -262,7 +266,7 @@ def get_s3_tar_with_ext_full_path(self): Returns: Path: Full path to the tarball in the local shell cache directory """ - return ensure_cache_dir(Path.home()) / self.getS3TarballWithExt() + return ensure_cache_dir(Path.home()) / self.get_s3_tar_name_with_ext() def get_shell_cache_dir(self): """Retrieve the shell cache directory of the intended js binary. @@ -836,8 +840,8 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa if updateLatestTxt: # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. - txt_info = "-".join(str(shell.getS3TarballWithExt()).split("-")[:-1] + ["latest"]) + ".txt" - s3cache_obj.uploadStrToS3("", txt_info, str(shell.getS3TarballWithExt())) + txt_info = "-".join(str(shell.get_s3_tar_name_with_ext()).split("-")[:-1] + ["latest"]) + ".txt" + s3cache_obj.uploadStrToS3("", txt_info, str(shell.get_s3_tar_name_with_ext())) shell.get_s3_tar_with_ext_full_path().unlink() From 931a3aed3103e61ad41def42b14eb2cc82a69777 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:41:33 -0700 Subject: [PATCH 073/110] getMajorVersion and setMajorVersion seem unnecessary since they are just used/fixated off set/getVersion and both are set at the same time. --- src/funfuzz/js/compile_shell.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 50b33d350..6c555b9e1 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -89,7 +89,6 @@ def __init__(self, build_opts, hg_hash): self.full_env = "" self.js_cfg_file = "" - self.jsMajorVersion = "" # pylint: disable=invalid-name self.jsVersion = "" # pylint: disable=invalid-name @classmethod @@ -319,14 +318,6 @@ def get_shell_name_without_ext(self): """ return self.shell_name_without_ext - # Version numbers - def getMajorVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.jsMajorVersion - - def setMajorVersion(self, jsMajorVersion): # pylint: disable=invalid-name,missing-docstring - self.jsMajorVersion = jsMajorVersion - def getVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc return self.jsVersion @@ -633,7 +624,6 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- shutil.copy2(str(run_lib), str(shell.get_shell_cache_dir())) version = extract_vers(shell.get_js_objdir()) - shell.setMajorVersion(version.split(".")[0]) shell.setVersion(version) if platform.system() == "Linux": @@ -693,7 +683,7 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi f.write("\n") f.write("[Metadata]\n") f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) - f.write("majorVersion = %s\n" % shell.getMajorVersion()) + f.write("majorVersion = %s\n" % shell.getVersion().split(".")[0]) f.write("pathPrefix = %s/\n" % shell.get_repo_dir()) f.write("version = %s\n" % shell.getVersion()) From eed1e6348b3d3a359ff9b234f00be465e81ee766 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:47:42 -0700 Subject: [PATCH 074/110] Refactor getVersion to get_version and setVersion to set_version. --- src/funfuzz/js/compile_shell.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 6c555b9e1..656aebdd8 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -89,7 +89,7 @@ def __init__(self, build_opts, hg_hash): self.full_env = "" self.js_cfg_file = "" - self.jsVersion = "" # pylint: disable=invalid-name + self.js_version = "" # pylint: disable=invalid-name @classmethod def main(cls, args=None): @@ -318,11 +318,21 @@ def get_shell_name_without_ext(self): """ return self.shell_name_without_ext - def getVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.jsVersion + def get_version(self): + """Retrieve the version number of the js shell as extracted from js.pc - def setVersion(self, jsVersion): # pylint: disable=invalid-name,missing-docstring - self.jsVersion = jsVersion + Returns: + str: Version number of the js shell + """ + return self.js_version + + def set_version(self, js_version): + """Set the version number of the js shell as extracted from js.pc + + Args: + js_version (str): Version number of the js shell + """ + self.js_version = js_version def ensure_cache_dir(base_dir): @@ -624,7 +634,7 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- shutil.copy2(str(run_lib), str(shell.get_shell_cache_dir())) version = extract_vers(shell.get_js_objdir()) - shell.setVersion(version) + shell.set_version(version) if platform.system() == "Linux": # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) @@ -683,9 +693,9 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi f.write("\n") f.write("[Metadata]\n") f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) - f.write("majorVersion = %s\n" % shell.getVersion().split(".")[0]) + f.write("majorVersion = %s\n" % shell.get_version().split(".")[0]) f.write("pathPrefix = %s/\n" % shell.get_repo_dir()) - f.write("version = %s\n" % shell.getVersion()) + f.write("version = %s\n" % shell.get_version()) def extract_vers(objdir): # pylint: disable=inconsistent-return-statements From 7e1eaafc302f47b82cae0744c9da5b51ca0064ba Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 16:51:33 -0700 Subject: [PATCH 075/110] Fix str/unicode confusion in repos_update. --- src/funfuzz/util/repos_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index 2126c2009..19f74e39d 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -46,7 +46,7 @@ else: raise OSError("Git binary not found") else: - GITBINARY = "git" + GITBINARY = str("git") def time_cmd(cmd, cwd=None, env=None, timeout=None): From a76035233e26e7925c7f12fa445d48300fb9ab6e Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 16 Mar 2018 20:10:22 -0700 Subject: [PATCH 076/110] WIP: Add some FIXME comments, they really should be fixed. --- src/funfuzz/js/compile_shell.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 656aebdd8..a95347d65 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -598,6 +598,36 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ shell.set_cfg_cmd_excl_env(cfg_cmds) +# FIXME: Potential problem area: Note that having a non-zero exit code does not mean that the # pylint: disable=fixme +# operation did not succeed, for example when compiling a shell. A non-zero exit code +# can appear even though a shell compiled successfully. +# except OSError as e: +# raise Exception(repr(e.strerror) + ' error calling: ' + shellify(cmd)) +# if p.returncode != 0: +# oomErrorOutput = stdout if combineStderr else stderr +# if (isLinux or isMac) and oomErrorOutput: +# if 'internal compiler error: Killed (program cc1plus)' in oomErrorOutput: +# raise Exception('GCC running out of memory') +# elif 'error: unable to execute command: Killed' in oomErrorOutput: +# raise Exception('Clang running out of memory') +# if not ignoreExitCode: +# # Potential problem area: Note that having a non-zero exit code does not mean that the +# # operation did not succeed, for example when compiling a shell. A non-zero exit code +# # can appear even though a shell compiled successfully. +# print("Nonzero exit code from: ") +# print(" %s" % shellify(cmd)) +# print("stdout is:") +# print(stdout) +# if stderr is not None: +# print("stderr is:") +# print(stderr) +# if stderr and ignoreStderr: +# # During configure, there will always be stderr. Sometimes this stderr causes configure to +# # stop the entire script, especially on Windows. +# print("Return code not zero, and unexpected output on stderr from: ") +# print(" %s" % shellify(cmd)) +# print("%s %s" % (stdout, stderr)) +# raise Exception('Return code not zero, and unexpected output on stderr') def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc """Compile and copy a binary.""" try: @@ -609,6 +639,7 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") except Exception as ex: # pylint: disable=broad-except # This exception message is returned from sps.captureStdout via cmd_list. + # FIXME: raise the Exception here after captureStdout removal? # pylint: disable=fixme if (platform.system() == "Linux" or platform.system() == "Darwin") and \ ("GCC running out of memory" in repr(ex) or "Clang running out of memory" in repr(ex)): # FIXME: Absolute hack to retry after hitting OOM. # pylint: disable=fixme From 214340ccc687a9f7bfea087e2a35c2a87db4318c Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 17:08:15 -0700 Subject: [PATCH 077/110] Fix pylint errors in test_autoconf_run. --- tests/js/test_compile_shell.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index 0a69ae3d8..a9d6105d7 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -39,12 +39,13 @@ class CompileShellTests(unittest.TestCase): mc_hg_repo = Path.home() / "trees" / "mozilla-central" shell_cache = Path.home() / "shell-cache" - def test_autoconf_run(): # pylint: disable=no-method-argument + def test_autoconf_run(self): # pylint: disable=no-self-use """Test the autoconf runs properly.""" with tempfile.TemporaryDirectory(suffix="autoconf_run_test") as tmp_dir: tmp_dir = Path(tmp_dir) - (tmp_dir / "configure.in").touch() # configure.in is required by autoconf2.13 + # configure.in is required by autoconf2.13 + (tmp_dir / "configure.in").touch() # pylint: disable=no-member js.compile_shell.autoconf_run(tmp_dir) def test_ensure_cache_dir(self): From b8ea6489f5a75161d5ffedf81b941d33e48930a4 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 17:11:07 -0700 Subject: [PATCH 078/110] Fix the test_ensure_cache_dir test. --- tests/js/test_compile_shell.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index a9d6105d7..c0f768f1e 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -49,8 +49,9 @@ def test_autoconf_run(self): # pylint: disable=no-self-use js.compile_shell.autoconf_run(tmp_dir) def test_ensure_cache_dir(self): - """Test the shell-cache dir is created properly if it does not exist.""" - self.assertTrue(js.compile_shell.ensure_cache_dir().is_dir()) + """Test the shell-cache dir is created properly if it does not exist, and things work even though it does.""" + self.assertTrue(js.compile_shell.ensure_cache_dir(None).is_dir()) + self.assertTrue(js.compile_shell.ensure_cache_dir(Path.home()).is_dir()) # pylint: disable=no-member @pytest.mark.slow @lru_cache(maxsize=None) From 7cf377c8112b9af1ccf7962ec6881fec79f5c210 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 17:16:18 -0700 Subject: [PATCH 079/110] Remove extra whitespace. --- src/funfuzz/js/compile_shell.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index a95347d65..c843ab88a 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -102,10 +102,8 @@ def main(cls, args=None): int: 0, to denote a successful compile and 1, to denote a failed compile """ # logging.basicConfig(format="%(message)s", level=logging.INFO) - try: return cls.run(args) - except CompiledShellError as ex: print(repr(ex)) # log.error(ex) From 8c418a4008584abe9f8382104283e2db4e2c2f19 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 17:59:37 -0700 Subject: [PATCH 080/110] Split off some helper functions from compile_shell.py into sm_compile_helpers.py --- src/funfuzz/js/compile_shell.py | 184 ++----------------------- src/funfuzz/util/__init__.py | 1 + src/funfuzz/util/sm_compile_helpers.py | 183 ++++++++++++++++++++++++ tests/js/test_compile_shell.py | 16 --- tests/util/test_sm_compile_helpers.py | 43 ++++++ 5 files changed, 241 insertions(+), 186 deletions(-) create mode 100644 src/funfuzz/util/sm_compile_helpers.py create mode 100644 tests/util/test_sm_compile_helpers.py diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index c843ab88a..cae8d2c88 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -11,7 +11,6 @@ from builtins import object # pylint: disable=redefined-builtin import copy -import io import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module import os @@ -19,7 +18,6 @@ import shutil import sys import tarfile -import traceback from pkg_resources import parse_version from shellescape import quote @@ -29,6 +27,7 @@ from . import inspect_shell from ..util import hg_helpers from ..util import s3cache +from ..util import sm_compile_helpers from ..util import subprocesses as sps from ..util.lock_dir import LockDir @@ -140,7 +139,7 @@ def run(argv=None): options = parser.parse_args(argv)[0] options.build_opts = build_options.parse_shell_opts(options.build_opts) - with LockDir(get_lock_dir_path(Path.home(), options.build_opts.repo_dir)): + with LockDir(sm_compile_helpers.get_lock_dir_path(Path.home(), options.build_opts.repo_dir)): if options.revision: shell = CompiledShell(options.build_opts, options.revision) else: @@ -263,7 +262,7 @@ def get_s3_tar_with_ext_full_path(self): Returns: Path: Full path to the tarball in the local shell cache directory """ - return ensure_cache_dir(Path.home()) / self.get_s3_tar_name_with_ext() + return sm_compile_helpers.ensure_cache_dir(Path.home()) / self.get_s3_tar_name_with_ext() def get_shell_cache_dir(self): """Retrieve the shell cache directory of the intended js binary. @@ -271,7 +270,7 @@ def get_shell_cache_dir(self): Returns: Path: Full path to the shell cache directory of the intended js binary """ - return ensure_cache_dir(Path.home()) / self.get_shell_name_without_ext() + return sm_compile_helpers.ensure_cache_dir(Path.home()) / self.get_shell_name_without_ext() def get_shell_cache_js_bin_path(self): """Retrieve the full path to the js binary located in the shell cache. @@ -279,7 +278,8 @@ def get_shell_cache_js_bin_path(self): Returns: Path: Full path to the js binary in the shell cache """ - return ensure_cache_dir(Path.home()) / self.get_shell_name_without_ext() / self.get_shell_name_with_ext() + return (sm_compile_helpers.ensure_cache_dir(Path.home()) / + self.get_shell_name_without_ext() / self.get_shell_name_with_ext()) def get_shell_compiled_path(self): """Retrieve the full path to the original location of js binary compiled in the shell cache. @@ -333,46 +333,6 @@ def set_version(self, js_version): self.js_version = js_version -def ensure_cache_dir(base_dir): - """Retrieve a cache directory for compiled shells to live in, and create one if needed. - - Args: - base_dir (Path): Base directory to create the cache directory in - - Returns: - Path: Returns the full shell-cache path - """ - if not base_dir: - base_dir = Path.home() - cache_dir = base_dir / "shell-cache" - cache_dir.mkdir(exist_ok=True) - return cache_dir - - -def autoconf_run(working_dir): - """Run autoconf binaries corresponding to the platform. - - Args: - working_dir (Path): Directory to be set as the current working directory - """ - if platform.system() == "Darwin": - autoconf213_mac_bin = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" if which("brew") else "autoconf213" - # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 - if not Path(which(autoconf213_mac_bin)).is_file(): - autoconf213_mac_bin = "autoconf213" - subprocess.run([autoconf213_mac_bin], check=True, cwd=str(working_dir)) - elif platform.system() == "Linux": - if which("autoconf2.13"): - subprocess.run(["autoconf2.13"], check=True, cwd=str(working_dir)) - elif which("autoconf-2.13"): - subprocess.run(["autoconf-2.13"], check=True, cwd=str(working_dir)) - elif which("autoconf213"): - subprocess.run(["autoconf213"], check=True, cwd=str(working_dir)) - elif platform.system() == "Windows": - # Windows needs to call sh to be able to find autoconf. - subprocess.run(["sh", "autoconf-2.13"], check=True, cwd=str(working_dir)) - - def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc """Configures, compiles and copies a js shell according to required parameters.""" print("Compiling...") # Print *with* a trailing newline to avoid breaking other stuff @@ -380,7 +340,7 @@ def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missi js_objdir_path.mkdir() shell.set_js_objdir(js_objdir_path) - autoconf_run(shell.get_repo_dir() / "js" / "src") + sm_compile_helpers.autoconf_run(shell.get_repo_dir() / "js" / "src") configure_try_count = 0 while True: try: @@ -402,7 +362,7 @@ def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missi compile_log = shell.get_shell_cache_dir() / (shell.get_shell_name_without_ext() + ".fuzzmanagerconf") if not compile_log.is_file(): - envDump(shell, compile_log) + sm_compile_helpers.envDump(shell, compile_log) def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-complex,too-many-branches @@ -662,7 +622,7 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- if run_lib.is_file(): shutil.copy2(str(run_lib), str(shell.get_shell_cache_dir())) - version = extract_vers(shell.get_js_objdir()) + version = sm_compile_helpers.extract_vers(shell.get_js_objdir()) shell.set_version(version) if platform.system() == "Linux": @@ -674,105 +634,6 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.") -def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Create a .busted file with the exception message and backtrace included.""" - with open(str(filename), "w") as f: - f.write("Caught exception %r (%s)\n" % (e, e)) - f.write("Backtrace:\n") - f.write(traceback.format_exc() + "\n") - - print("Compilation failed (%s) (details in %s)" % (e, filename)) - - -def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Dump environment to a .fuzzmanagerconf file.""" - # Platform and OS detection for the spec, part of which is in: - # https://wiki.mozilla.org/Security/CrashSignatures - fmconf_platform = "x86" if shell.build_opts.enable32 else "x86-64" - - if platform.system() == "Linux": - fmconf_os = "linux" - elif platform.system() == "Darwin": - fmconf_os = "macosx" - elif platform.system() == "Windows": - fmconf_os = "windows" - - with open(str(log), "a") as f: - f.write("# Information about shell:\n# \n") - - f.write("# Create another shell in shell-cache like this one:\n") - f.write('# python -u -m %s -b "%s" -r %s\n# \n' % ("funfuzz.js.compile_shell", - shell.build_opts.build_options_str, shell.get_hg_hash())) - - f.write("# Full environment is:\n") - f.write("# %s\n# \n" % str(shell.get_env_full())) - - f.write("# Full configuration command with needed environment variables is:\n") - f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.get_env_added()), - " ".join(quote(str(x)) for x in shell.get_cfg_cmd_excl_env()))) - - # .fuzzmanagerconf details - f.write("\n") - f.write("[Main]\n") - f.write("platform = %s\n" % fmconf_platform) - f.write("product = %s\n" % shell.get_repo_name()) - f.write("product_version = %s\n" % shell.get_hg_hash()) - f.write("os = %s\n" % fmconf_os) - - f.write("\n") - f.write("[Metadata]\n") - f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) - f.write("majorVersion = %s\n" % shell.get_version().split(".")[0]) - f.write("pathPrefix = %s/\n" % shell.get_repo_dir()) - f.write("version = %s\n" % shell.get_version()) - - -def extract_vers(objdir): # pylint: disable=inconsistent-return-statements - """Extract the version from js.pc and put it into *.fuzzmanagerconf. - - Args: - objdir (Path): Full path to the objdir - - Raises: - OSError: Raises when js.pc is not found - - Returns: - str: Version number of the compiled js shell - """ - jspc_file_path = objdir / "js" / "src" / "js.pc" - # Moved to /js/src/build/, see bug 1262241, Fx55 m-c rev 351194:2159959522f4 - jspc_new_file_path = objdir / "js" / "src" / "build" / "js.pc" - - if jspc_file_path.is_file(): - actual_path = jspc_file_path - elif jspc_new_file_path.is_file(): - actual_path = jspc_new_file_path - else: - raise OSError("js.pc file not found - needed to extract the version number") - - with io.open(str(actual_path), mode="r", encoding="utf-8", errors="replace") as f: - for line in f: - if line.startswith("Version: "): # Sample line: "Version: 47.0a2" - return line.split(": ")[1].rstrip() - - -def get_lock_dir_path(cache_dir_base, repo_dir, tbox_id=""): - """Return the name of the lock directory. - - Args: - cache_dir_base (Path): Base directory where the cache directory is located - repo_dir (Path): Full path to the repository - tbox_id (str): Tinderbox entry id - - Returns: - Path: Full path to the shell cache lock directory - """ - lockdir_name = "shell-%s-lock" % repo_dir.name - if tbox_id: - lockdir_name += "-%s" % tbox_id - return ensure_cache_dir(cache_dir_base) / lockdir_name - - def makeTestRev(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc def testRev(rev): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc shell = CompiledShell(options.build_options, rev) @@ -791,7 +652,7 @@ def testRev(rev): # pylint: disable=invalid-name,missing-docstring,missing-retu def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disable=invalid-name,missing-param-doc # pylint: disable=missing-raises-doc,missing-type-doc,too-many-branches,too-complex,too-many-statements """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" - assert get_lock_dir_path(Path.home(), shell.build_opts.repo_dir).is_dir() + assert sm_compile_helpers.get_lock_dir_path(Path.home(), shell.build_opts.repo_dir).is_dir() cached_no_shell = shell.get_shell_cache_js_bin_path().with_suffix(".busted") if shell.get_shell_cache_js_bin_path().is_file(): @@ -800,7 +661,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa print("Found cached shell...") # Assuming that since the binary is present, everything else (e.g. symbols) is also present if platform.system() == "Windows": - verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) + sm_compile_helpers.verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) return elif cached_no_shell.is_file(): raise Exception("Found a cached shell that failed compilation...") @@ -827,7 +688,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa # Delete tarball after downloading from S3 shell.get_s3_tar_with_ext_full_path().unlink() if platform.system() == "Windows": - verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) + sm_compile_helpers.verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) return try: @@ -847,7 +708,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa cfgJsCompile(shell) if platform.system() == "Windows": - verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) + sm_compile_helpers.verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) except KeyboardInterrupt: sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) raise @@ -855,7 +716,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa # Remove the cache dir, but recreate it with only the .busted file. sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) shell.get_shell_cache_dir().mkdir() - createBustedFile(cached_no_shell, ex) + sm_compile_helpers.createBustedFile(cached_no_shell, ex) if use_s3cache: s3cache_obj.uploadFileToS3(str(shell.get_shell_cache_js_bin_path()) + ".busted") raise @@ -874,23 +735,6 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa shell.get_s3_tar_with_ext_full_path().unlink() -def verify_full_win_pageheap(shell_path): - """Turn on full page heap verification on Windows. - - Args: - shell_path (Path): Path to the compiled js shell - """ - # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx - # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ - gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" - if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member - print(subprocess.run([str(gflags_bin_path).decode("utf-8", errors="replace"), - "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"], - check=True, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE).stdout) - - def main(): """Execute main() function in CompiledShell class.""" exit(CompiledShell.main()) diff --git a/src/funfuzz/util/__init__.py b/src/funfuzz/util/__init__.py index 9134f4ac6..1cd755106 100644 --- a/src/funfuzz/util/__init__.py +++ b/src/funfuzz/util/__init__.py @@ -18,4 +18,5 @@ from . import os_ops from . import repos_update from . import s3cache +from . import sm_compile_helpers from . import subprocesses diff --git a/src/funfuzz/util/sm_compile_helpers.py b/src/funfuzz/util/sm_compile_helpers.py new file mode 100644 index 000000000..90674deca --- /dev/null +++ b/src/funfuzz/util/sm_compile_helpers.py @@ -0,0 +1,183 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Helper functions to compile SpiderMonkey shells. +""" + +from __future__ import absolute_import, print_function, unicode_literals # isort:skip + +import io +import os +import platform +import sys +import traceback + +from shellescape import quote +from whichcraft import which # Once we are fully on Python 3.5+, whichcraft can be removed in favour of shutil.which + +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + + +def ensure_cache_dir(base_dir): + """Retrieve a cache directory for compiled shells to live in, and create one if needed. + + Args: + base_dir (Path): Base directory to create the cache directory in + + Returns: + Path: Returns the full shell-cache path + """ + if not base_dir: + base_dir = Path.home() + cache_dir = base_dir / "shell-cache" + cache_dir.mkdir(exist_ok=True) + return cache_dir + + +def autoconf_run(working_dir): + """Run autoconf binaries corresponding to the platform. + + Args: + working_dir (Path): Directory to be set as the current working directory + """ + if platform.system() == "Darwin": + autoconf213_mac_bin = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" if which("brew") else "autoconf213" + # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 + if not Path(which(autoconf213_mac_bin)).is_file(): + autoconf213_mac_bin = "autoconf213" + subprocess.run([autoconf213_mac_bin], check=True, cwd=str(working_dir)) + elif platform.system() == "Linux": + if which("autoconf2.13"): + subprocess.run(["autoconf2.13"], check=True, cwd=str(working_dir)) + elif which("autoconf-2.13"): + subprocess.run(["autoconf-2.13"], check=True, cwd=str(working_dir)) + elif which("autoconf213"): + subprocess.run(["autoconf213"], check=True, cwd=str(working_dir)) + elif platform.system() == "Windows": + # Windows needs to call sh to be able to find autoconf. + subprocess.run(["sh", "autoconf-2.13"], check=True, cwd=str(working_dir)) + + +def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Create a .busted file with the exception message and backtrace included.""" + with open(str(filename), "w") as f: + f.write("Caught exception %r (%s)\n" % (e, e)) + f.write("Backtrace:\n") + f.write(traceback.format_exc() + "\n") + + print("Compilation failed (%s) (details in %s)" % (e, filename)) + + +def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Dump environment to a .fuzzmanagerconf file.""" + # Platform and OS detection for the spec, part of which is in: + # https://wiki.mozilla.org/Security/CrashSignatures + fmconf_platform = "x86" if shell.build_opts.enable32 else "x86-64" + + if platform.system() == "Linux": + fmconf_os = "linux" + elif platform.system() == "Darwin": + fmconf_os = "macosx" + elif platform.system() == "Windows": + fmconf_os = "windows" + + with open(str(log), "a") as f: + f.write("# Information about shell:\n# \n") + + f.write("# Create another shell in shell-cache like this one:\n") + f.write('# python -u -m %s -b "%s" -r %s\n# \n' % ("funfuzz.js.compile_shell", + shell.build_opts.build_options_str, shell.get_hg_hash())) + + f.write("# Full environment is:\n") + f.write("# %s\n# \n" % str(shell.get_env_full())) + + f.write("# Full configuration command with needed environment variables is:\n") + f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.get_env_added()), + " ".join(quote(str(x)) for x in shell.get_cfg_cmd_excl_env()))) + + # .fuzzmanagerconf details + f.write("\n") + f.write("[Main]\n") + f.write("platform = %s\n" % fmconf_platform) + f.write("product = %s\n" % shell.get_repo_name()) + f.write("product_version = %s\n" % shell.get_hg_hash()) + f.write("os = %s\n" % fmconf_os) + + f.write("\n") + f.write("[Metadata]\n") + f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) + f.write("majorVersion = %s\n" % shell.get_version().split(".")[0]) + f.write("pathPrefix = %s/\n" % shell.get_repo_dir()) + f.write("version = %s\n" % shell.get_version()) + + +def extract_vers(objdir): # pylint: disable=inconsistent-return-statements + """Extract the version from js.pc and put it into *.fuzzmanagerconf. + + Args: + objdir (Path): Full path to the objdir + + Raises: + OSError: Raises when js.pc is not found + + Returns: + str: Version number of the compiled js shell + """ + jspc_file_path = objdir / "js" / "src" / "js.pc" + # Moved to /js/src/build/, see bug 1262241, Fx55 m-c rev 351194:2159959522f4 + jspc_new_file_path = objdir / "js" / "src" / "build" / "js.pc" + + if jspc_file_path.is_file(): + actual_path = jspc_file_path + elif jspc_new_file_path.is_file(): + actual_path = jspc_new_file_path + else: + raise OSError("js.pc file not found - needed to extract the version number") + + with io.open(str(actual_path), mode="r", encoding="utf-8", errors="replace") as f: + for line in f: + if line.startswith("Version: "): # Sample line: "Version: 47.0a2" + return line.split(": ")[1].rstrip() + + +def get_lock_dir_path(cache_dir_base, repo_dir, tbox_id=""): + """Return the name of the lock directory. + + Args: + cache_dir_base (Path): Base directory where the cache directory is located + repo_dir (Path): Full path to the repository + tbox_id (str): Tinderbox entry id + + Returns: + Path: Full path to the shell cache lock directory + """ + lockdir_name = "shell-%s-lock" % repo_dir.name + if tbox_id: + lockdir_name += "-%s" % tbox_id + return ensure_cache_dir(cache_dir_base) / lockdir_name + + +def verify_full_win_pageheap(shell_path): + """Turn on full page heap verification on Windows. + + Args: + shell_path (Path): Path to the compiled js shell + """ + # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx + # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ + gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" + if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member + print(subprocess.run([str(gflags_bin_path).decode("utf-8", errors="replace"), + "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"], + check=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout) diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index c0f768f1e..a5caee9fa 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -20,13 +20,11 @@ from funfuzz import util if sys.version_info.major == 2: - import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module from functools32 import lru_cache # pylint: disable=import-error from pathlib2 import Path else: from functools import lru_cache # pylint: disable=no-name-in-module from pathlib import Path # pylint: disable=import-error - import tempfile FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") logging.basicConfig(level=logging.DEBUG) @@ -39,20 +37,6 @@ class CompileShellTests(unittest.TestCase): mc_hg_repo = Path.home() / "trees" / "mozilla-central" shell_cache = Path.home() / "shell-cache" - def test_autoconf_run(self): # pylint: disable=no-self-use - """Test the autoconf runs properly.""" - with tempfile.TemporaryDirectory(suffix="autoconf_run_test") as tmp_dir: - tmp_dir = Path(tmp_dir) - - # configure.in is required by autoconf2.13 - (tmp_dir / "configure.in").touch() # pylint: disable=no-member - js.compile_shell.autoconf_run(tmp_dir) - - def test_ensure_cache_dir(self): - """Test the shell-cache dir is created properly if it does not exist, and things work even though it does.""" - self.assertTrue(js.compile_shell.ensure_cache_dir(None).is_dir()) - self.assertTrue(js.compile_shell.ensure_cache_dir(Path.home()).is_dir()) # pylint: disable=no-member - @pytest.mark.slow @lru_cache(maxsize=None) def test_shell_compile(self): diff --git a/tests/util/test_sm_compile_helpers.py b/tests/util/test_sm_compile_helpers.py new file mode 100644 index 000000000..7de5eb8b0 --- /dev/null +++ b/tests/util/test_sm_compile_helpers.py @@ -0,0 +1,43 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import logging +import sys +import unittest + +from funfuzz import util + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class SmCompileHelpersTests(unittest.TestCase): + """"TestCase class for functions in sm_compile_helpers.py""" + def test_autoconf_run(self): # pylint: disable=no-self-use + """Test the autoconf runs properly.""" + with tempfile.TemporaryDirectory(suffix="autoconf_run_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + + # configure.in is required by autoconf2.13 + (tmp_dir / "configure.in").touch() # pylint: disable=no-member + util.sm_compile_helpers.autoconf_run(tmp_dir) + + def test_ensure_cache_dir(self): + """Test the shell-cache dir is created properly if it does not exist, and things work even though it does.""" + self.assertTrue(util.sm_compile_helpers.ensure_cache_dir(None).is_dir()) + self.assertTrue(util.sm_compile_helpers.ensure_cache_dir(Path.home()).is_dir()) # pylint: disable=no-member From 8d9e38633ec7edd6421781a02dcf6d84b606e54d Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 18:18:05 -0700 Subject: [PATCH 081/110] Inline code involving version in compile_shell. --- src/funfuzz/js/compile_shell.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index cae8d2c88..c4db099d9 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -622,8 +622,7 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- if run_lib.is_file(): shutil.copy2(str(run_lib), str(shell.get_shell_cache_dir())) - version = sm_compile_helpers.extract_vers(shell.get_js_objdir()) - shell.set_version(version) + shell.set_version(sm_compile_helpers.extract_vers(shell.get_js_objdir())) if platform.system() == "Linux": # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) From 8ec1a8cb69177b7c1bea582eb9cd3b23b6568c35 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 18:19:02 -0700 Subject: [PATCH 082/110] Mostly stop catching generic Exceptions in compile_shell. --- src/funfuzz/js/compile_shell.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index c4db099d9..d274e88f3 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -346,7 +346,7 @@ def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missi try: cfgBin(shell) break - except Exception as ex: # pylint: disable=broad-except + except subprocess.CalledProcessError as ex: configure_try_count += 1 if configure_try_count > 3: print("Configuration of the js binary failed 3 times.") @@ -640,7 +640,7 @@ def testRev(rev): # pylint: disable=invalid-name,missing-docstring,missing-retu try: obtainShell(shell, updateToRev=rev) - except Exception: # pylint: disable=broad-except + except subprocess.CalledProcessError: return (options.compilationFailedLabel, "compilation failed") print("Testing...", end=" ") @@ -711,7 +711,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa except KeyboardInterrupt: sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) raise - except Exception as ex: + except subprocess.CalledProcessError as ex: # Remove the cache dir, but recreate it with only the .busted file. sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) shell.get_shell_cache_dir().mkdir() From 2094280ebf3e5421b11d107184bedfb60e642336 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 18:38:38 -0700 Subject: [PATCH 083/110] Refactor compileJs to sm_compile and fix previous WIP/FIXME items. --- src/funfuzz/js/compile_shell.py | 104 +++++++++++++------------------- 1 file changed, 43 insertions(+), 61 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index d274e88f3..5ff13cd6a 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -11,6 +11,7 @@ from builtins import object # pylint: disable=redefined-builtin import copy +import io import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module import os @@ -357,7 +358,7 @@ def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missi "Windows conftest.exe configuration permission" in repr(ex)): print("Trying once more...") continue - compileJs(shell) + sm_compile(shell) inspect_shell.verifyBinary(shell) compile_log = shell.get_shell_cache_dir() / (shell.get_shell_name_without_ext() + ".fuzzmanagerconf") @@ -556,65 +557,27 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ shell.set_cfg_cmd_excl_env(cfg_cmds) -# FIXME: Potential problem area: Note that having a non-zero exit code does not mean that the # pylint: disable=fixme -# operation did not succeed, for example when compiling a shell. A non-zero exit code -# can appear even though a shell compiled successfully. -# except OSError as e: -# raise Exception(repr(e.strerror) + ' error calling: ' + shellify(cmd)) -# if p.returncode != 0: -# oomErrorOutput = stdout if combineStderr else stderr -# if (isLinux or isMac) and oomErrorOutput: -# if 'internal compiler error: Killed (program cc1plus)' in oomErrorOutput: -# raise Exception('GCC running out of memory') -# elif 'error: unable to execute command: Killed' in oomErrorOutput: -# raise Exception('Clang running out of memory') -# if not ignoreExitCode: -# # Potential problem area: Note that having a non-zero exit code does not mean that the -# # operation did not succeed, for example when compiling a shell. A non-zero exit code -# # can appear even though a shell compiled successfully. -# print("Nonzero exit code from: ") -# print(" %s" % shellify(cmd)) -# print("stdout is:") -# print(stdout) -# if stderr is not None: -# print("stderr is:") -# print(stderr) -# if stderr and ignoreStderr: -# # During configure, there will always be stderr. Sometimes this stderr causes configure to -# # stop the entire script, especially on Windows. -# print("Return code not zero, and unexpected output on stderr from: ") -# print(" %s" % shellify(cmd)) -# print("%s %s" % (stdout, stderr)) -# raise Exception('Return code not zero, and unexpected output on stderr') -def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc - """Compile and copy a binary.""" - try: - cmd_list = [MAKE_BINARY, "-C", str(shell.get_js_objdir()), "-j" + str(COMPILATION_JOBS), "-s"] - out = subprocess.run(cmd_list, - cwd=str(shell.get_js_objdir()), - env=shell.get_env_full(), - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") - except Exception as ex: # pylint: disable=broad-except - # This exception message is returned from sps.captureStdout via cmd_list. - # FIXME: raise the Exception here after captureStdout removal? # pylint: disable=fixme - if (platform.system() == "Linux" or platform.system() == "Darwin") and \ - ("GCC running out of memory" in repr(ex) or "Clang running out of memory" in repr(ex)): - # FIXME: Absolute hack to retry after hitting OOM. # pylint: disable=fixme - print("Trying once more due to the compiler running out of memory...") - out = subprocess.run(cmd_list, - cwd=str(shell.get_js_objdir()), - env=shell.get_env_full(), - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") - # A non-zero error can be returned during make, but eventually a shell still gets compiled. - if shell.get_shell_compiled_path().is_file(): - print("A shell was compiled even though there was a non-zero exit code. Continuing...") - else: - print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) - raise +def sm_compile(shell): + """Compile a binary and copy essential compiled files into a desired structure. - # We could save the stdout here into a file if it throws + Args: + shell (object): SpiderMonkey shell parameters + + Raises: + OSError: Raises when a compiled shell is absent + + Returns: + Path: Path to the compiled shell + """ + cmd_list = [MAKE_BINARY, "-C", str(shell.get_js_objdir()), "-j" + str(COMPILATION_JOBS), "-s"] + # Note that having a non-zero exit code does not mean that the operation did not succeed, + # for example when compiling a shell. A non-zero exit code can appear even though a shell compiled successfully. + # Thus, we should *not* use check=True here. + out = subprocess.run(cmd_list, + cwd=str(shell.get_js_objdir()), + env=shell.get_env_full(), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") if shell.get_shell_compiled_path().is_file(): shutil.copy2(str(shell.get_shell_compiled_path()), str(shell.get_shell_cache_js_bin_path())) @@ -629,8 +592,27 @@ def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing- # files in the objdir or else the stacks from failing testcases will lack symbols. shutil.rmtree(str(shell.get_shell_cache_dir() / "objdir-js")) else: - print(out.decode("utf-8", errors="replace")) - raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.") + if ((platform.system() == "Linux" or platform.system() == "Darwin") and + ("internal compiler error: Killed (program cc1plus)" in out or # GCC running out of memory + "error: unable to execute command: Killed" in out)): # Clang running out of memory + print("Trying once more due to the compiler running out of memory...") + out = subprocess.run(cmd_list, + cwd=str(shell.get_js_objdir()), + env=shell.get_env_full(), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + # A non-zero error can be returned during make, but eventually a shell still gets compiled. + if shell.get_shell_compiled_path().is_file(): + print("A shell was compiled even though there was a non-zero exit code. Continuing...") + else: + print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) + with io.open(str(shell.get_shell_cache_dir() / ".busted.log"), "w") as f: + f.write("The first compilation of %s rev %s failed with the following output:\n" % + (shell.get_repo_name(), shell.get_hg_hash())) + f.write(out.decode("utf-8", errors="replace")) + raise OSError(MAKE_BINARY + " did not result in a js shell.") + + return shell.get_shell_compiled_path() def makeTestRev(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc From 57debe121a7d8a5d6eb2e0235ec5654c66ae42da Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 19:16:36 -0700 Subject: [PATCH 084/110] Fix more issues involving the move of functions to sm_compile_helpers. --- src/funfuzz/autobisectjs/autobisectjs.py | 7 ++++--- src/funfuzz/bot.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index 0b4effb20..fa3e54713 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -26,6 +26,7 @@ from ..js import inspect_shell from ..util import hg_helpers from ..util import s3cache +from ..util import sm_compile_helpers from ..util import subprocesses as sps from ..util.lock_dir import LockDir @@ -526,8 +527,8 @@ def main(): if options.build_options: repo_dir = options.build_options.repo_dir - with LockDir(compile_shell.get_lock_dir_path(Path.home(), options.nameOfTreeherderBranch, tbox_id="Tbox") - if options.useTreeherderBinaries else compile_shell.get_lock_dir_path(Path.home(), repo_dir)): + with LockDir(sm_compile_helpers.get_lock_dir_path(Path.home(), options.nameOfTreeherderBranch, tbox_id="Tbox") + if options.useTreeherderBinaries else sm_compile_helpers.get_lock_dir_path(Path.home(), repo_dir)): if options.useTreeherderBinaries: print_("TBD: We need to switch to the autobisect repository.", flush=True) sys.exit(0) @@ -536,4 +537,4 @@ def main(): # Last thing we do while we have a lock. # Note that this only clears old *local* cached directories, not remote ones. - rm_old_local_cached_dirs(compile_shell.ensure_cache_dir(Path.home())) + rm_old_local_cached_dirs(sm_compile_helpers.ensure_cache_dir(Path.home())) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index d110e4796..40f8b488f 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -28,6 +28,7 @@ from .util import create_collector from .util import fork_join from .util import hg_helpers +from .util import sm_compile_helpers from .util.lock_dir import LockDir if sys.version_info.major == 2: @@ -186,7 +187,7 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss options.build_options = build_options.parse_shell_opts(options.build_options) options.timeout = options.timeout or (300 if options.build_options.runWithVg else JS_SHELL_DEFAULT_TIMEOUT) - with LockDir(compile_shell.get_lock_dir_path(Path.home(), options.build_options.repo_dir)): + with LockDir(sm_compile_helpers.get_lock_dir_path(Path.home(), options.build_options.repo_dir)): bRev = hg_helpers.get_repo_hash_and_id(options.build_options.repo_dir)[0] # pylint: disable=invalid-name cshell = compile_shell.CompiledShell(options.build_options, bRev) updateLatestTxt = (options.build_options.repo_dir == "mozilla-central") # pylint: disable=invalid-name From fc3cb9e4cfa97f0fbe897375a8ad05069029e185 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 19:13:48 -0700 Subject: [PATCH 085/110] Use io.open instead of open throughout. --- src/funfuzz/bot.py | 3 ++- src/funfuzz/js/build_options.py | 3 ++- src/funfuzz/js/compare_jit.py | 5 +++-- src/funfuzz/js/js_interesting.py | 13 +++++++------ src/funfuzz/js/link_fuzzer.py | 3 ++- src/funfuzz/js/loop.py | 11 ++++++----- src/funfuzz/util/file_manipulation.py | 6 ++++-- src/funfuzz/util/fork_join.py | 7 ++++--- src/funfuzz/util/lithium_helpers.py | 23 ++++++++++++----------- src/funfuzz/util/os_ops.py | 7 ++++--- src/funfuzz/util/sm_compile_helpers.py | 4 ++-- src/funfuzz/util/subprocesses.py | 2 +- tests/js/test_link_fuzzer.py | 3 ++- 13 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 40f8b488f..14df716a9 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -11,6 +11,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin +import io import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module import os @@ -162,7 +163,7 @@ def print_machine_info(): hgrc_path = Path("~/.hg/hgrc").expanduser() if hgrc_path.is_file(): print("The hgrc of this repository is:") - with open(str(hgrc_path), "r") as f: + with io.open(str(hgrc_path), "r") as f: hgrc_contents = f.readlines() for line in hgrc_contents: print(line.rstrip()) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 97b4c2858..b45ddf198 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -12,6 +12,7 @@ import argparse from builtins import object # pylint: disable=redefined-builtin import hashlib +import io import platform import random import sys @@ -242,7 +243,7 @@ def computeShellType(build_options): # pylint: disable=invalid-name,missing-par if build_options.patch_file: # We take the name before the first dot, so Windows (hopefully) does not get confused. fileName.append(build_options.patch_file.name) - with open(str(build_options.patch_file.resolve()), "r") as f: + with io.open(str(build_options.patch_file.resolve()), "r") as f: readResult = f.read() # pylint: disable=invalid-name # Append the patch hash, but this is not equivalent to Mercurial's hash of the patch. fileName.append(hashlib.sha512(readResult).hexdigest()[:12]) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index a57469610..bebf3641b 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io from optparse import OptionParser # pylint: disable=deprecated-module import os import sys @@ -145,7 +146,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi r.lev, r.runinfo.elapsedtime))) summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") - with open(str(summary_log), "w") as f: + with io.open(str(summary_log), "w") as f: f.write("\n".join(r.issues + [" ".join(quote(str(x)) for x in command), "compare_jit found a more serious bug"]) + "\n") print(" %s" % " ".join(quote(str(x)) for x in command)) @@ -199,7 +200,7 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr summary = (" " + " ".join(quote(str(x)) for x in commands[0]) + "\n " + " ".join(quote(str(x)) for x in command) + "\n\n" + summary) summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") - with open(str(summary_log), "w") as f: + with io.open(str(summary_log), "w") as f: f.write(rerunCommand + "\n\n" + summary) print("%s | %s" % (str(infilename), js_interesting.summaryString( issues, js_interesting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime))) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index c7d6a9e97..b71fb9a69 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin +import io from optparse import OptionParser # pylint: disable=deprecated-module import os import platform @@ -100,10 +101,10 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and "readlines" everything into memory. out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") - with open(str(out_log)) as f: + with io.open(str(out_log), "r") as f: out = f.readlines() err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt") - with open(str(err_log)) as f: + with io.open(str(err_log), "r") as f: err = f.readlines() if options.valgrind and runinfo.return_code == VALGRIND_ERROR_EXIT_CODE: @@ -116,7 +117,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa elif runinfo.sta == timed_run.CRASHED: if os_ops.grab_crash_log(str(runthis[0]), runinfo.pid, logPrefix, True): crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") - with open(str(crash_log)) as f: + with io.open(str(crash_log), "r") as f: auxCrashData = [line.strip() for line in f.readlines()] elif file_manipulation.amiss(logPrefix): issues.append("malloc error") @@ -143,7 +144,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if activated and platform.system() == "Linux" and which("gdb") and not auxCrashData and not in_compare_jit: print("Note: No core file found on Linux - falling back to run via gdb") extracted_gdb_cmds = ["-ex", "run"] - with open(str(Path(__file__).parent.parent / "util" / "gdb_cmds.txt"), "r") as f: + with io.open(str(Path(__file__).parent.parent / "util" / "gdb_cmds.txt"), "r") as f: for line in f: if line.rstrip() and not line.startswith("#") and not line.startswith("echo"): extracted_gdb_cmds.append("-ex") @@ -182,7 +183,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if lev != JS_FINE: summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") - with open(str(summary_log), "w") as f: + with io.open(str(summary_log), "w") as f: f.writelines(["Number: " + str(logPrefix) + "\n", "Command: " + " ".join(quote(str(x)) for x in runthis) + "\n"] + ["Status: " + i + "\n" for i in issues]) @@ -249,7 +250,7 @@ def summaryString(issues, level, elapsedtime): # pylint: disable=invalid-name,m def truncateFile(fn, maxSize): # pylint: disable=invalid-name,missing-docstring if fn.is_file() and fn.stat().st_size > maxSize: - with open(str(fn), "r+") as f: + with io.open(str(fn), "r+") as f: f.truncate(maxSize) diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index c9941412f..662964659 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -9,6 +9,7 @@ from __future__ import absolute_import +import io import sys if sys.version_info.major == 2: @@ -26,7 +27,7 @@ def link_fuzzer(target_path, prologue=""): """ base_dir = Path(__file__).parent - with open(str(target_path), "w") as f: # Create the full jsfunfuzz file + with io.open(str(target_path), "w") as f: # Create the full jsfunfuzz file if prologue: f.write(prologue) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index fa5a33ad2..bbe1a7cc0 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io import json from optparse import OptionParser # pylint: disable=deprecated-module import os @@ -177,13 +178,13 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in reduced_log = (logPrefix.parent / (logPrefix.stem + "-reduced")).with_suffix(".js") [before, after] = file_manipulation.fuzzSplice(fuzzjs) - with open(str(out_log), "r") as f: + with io.open(str(out_log), "r") as f: newfileLines = before + [ # pylint: disable=invalid-name l.replace("/*FRC-", "/*") for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after orig_log = (logPrefix.parent / (logPrefix.stem + "-orig")).with_suffix(".js") - with open(str(orig_log), "w") as f: + with io.open(str(orig_log), "w") as f: f.writelines(newfileLines) - with open(str(reduced_log), "w") as f: + with io.open(str(reduced_log), "w") as f: f.writelines(newfileLines) # Run Lithium and autobisectjs (make a reduced testcase and find a regression window) @@ -231,7 +232,7 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") linesToCompare = jitCompareLines(out_log, "/*FCM*/") # pylint: disable=invalid-name jitcomparefilename = (logPrefix.parent / (logPrefix.stem + "-cj-in")).with_suffix(".js") - with open(str(jitcomparefilename), "w") as f: + with io.open(str(jitcomparefilename), "w") as f: f.writelines(linesToCompare) # pylint: disable=invalid-name anyBug = compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename, @@ -263,7 +264,7 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid "wasmIsSupported = function() { return true; };\n", "// DDBEGIN\n" ] - with open(str(jsfunfuzzOutputFilename), "r") as f: + with io.open(str(jsfunfuzzOutputFilename), "r") as f: for line in f: if line.startswith(marker): sline = line[len(marker):] diff --git a/src/funfuzz/util/file_manipulation.py b/src/funfuzz/util/file_manipulation.py index 95fd16263..2e99b2597 100644 --- a/src/funfuzz/util/file_manipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io + def amiss(log_prefix): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc """Look for "szone_error" (Tiger), "malloc_error_break" (Leopard), "MallocHelp" (?) @@ -16,7 +18,7 @@ def amiss(log_prefix): # pylint: disable=missing-param-doc,missing-return-doc,m """ found_something = False err_log = (log_prefix.parent / (log_prefix.stem + "-err")).with_suffix(".txt") - with open(str(err_log)) as f: + with io.open(str(err_log), "r") as f: for line in f: line = line.strip("\x07").rstrip("\n") if (line.find("szone_error") != -1 or @@ -35,7 +37,7 @@ def fuzzSplice(filename): # pylint: disable=invalid-name,missing-param-doc,miss """Return the lines of a file, minus the ones between the two lines containing SPLICE.""" before = [] after = [] - with open(str(filename), "r") as f: + with io.open(str(filename), "r") as f: for line in f: before.append(line) if line.find("SPLICE") != -1: diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index 38d11d5c0..d4dc3651a 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io import multiprocessing import sys @@ -27,7 +28,7 @@ def forkJoin(logDir, numProcesses, fun, *someArgs): # pylint: disable=invalid-n def showFile(fn): # pylint: disable=invalid-name,missing-docstring print("==== %s ====" % fn) print() - with open(str(fn)) as f: + with io.open(str(fn), "r") as f: for line in f: print(line.rstrip()) print() @@ -69,8 +70,8 @@ def log_name(log_dir, i, log_type): def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring - sys.stdout = open(log_name(logDir, i, "out"), "wb", buffering=0) # I WONDER .decode("utf-8", errors="replace") - sys.stderr = open(log_name(logDir, i, "err"), "wb", buffering=0) # I WONDER .decode("utf-8", errors="replace") + sys.stdout = io.open(log_name(logDir, i, "out"), "w", buffering=0) + sys.stderr = io.open(log_name(logDir, i, "err"), "w", buffering=0) fun(*(someArgs + (i,))) diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index 8905e5537..ff8c5f70b 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io import os import re import shutil @@ -69,11 +70,11 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ) print(" ".join(quote(str(x)) for x in autobisectCmd)) autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") - with open(str(autobisect_log), "w") as f: + with io.open(str(autobisect_log), "w") as f: subprocess.run(autobisectCmd, stderr=subprocess.STDOUT, stdout=f) print("Done running autobisectjs. Log: %s" % autobisect_log) - with open(str(autobisect_log), "r") as f: + with io.open(str(autobisect_log), "r") as f: lines = f.readlines() autobisect_log_trunc = file_manipulation.truncateMid(lines, 50, ["..."]) else: @@ -102,7 +103,7 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) print(" ".join(quote(str(x)) for x in runlithiumpy + lithArgs)) - with open(str(lithlogfn), "w") as f: + with io.open(str(lithlogfn), "w") as f: subprocess.run(runlithiumpy + lithArgs, stderr=subprocess.STDOUT, stdout=f) print("Done running Lithium") if deletableLithTemp: @@ -114,7 +115,7 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam def readLithiumResult(lithlogfn): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - with open(str(lithlogfn)) as f: + with io.open(str(lithlogfn), "r") as f: for line in f: if line.startswith("Lithium result"): print(line.rstrip()) @@ -178,7 +179,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis hasTryItOut = False # pylint: disable=invalid-name hasTryItOutRegex = re.compile(r'count=[0-9]+; tryItOut\("') # pylint: disable=invalid-name - with open(str(infilename), "r") as f: + with io.open(str(infilename), "r") as f: for line in file_manipulation.linesWith(f, '; tryItOut("'): # Checks if testcase came from jsfunfuzz or compare_jit. # Do not use .match here, it only matches from the start of the line: @@ -192,12 +193,12 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis tryItOutAndCountRegex = re.compile(r'"\);\ncount=([0-9]+); tryItOut\("', # pylint: disable=invalid-name re.MULTILINE) - with open(str(infilename), "r") as f: + with io.open(str(infilename), "r") as f: infileContents = f.read() # pylint: disable=invalid-name infileContents = re.sub(tryItOutAndCountRegex, # pylint: disable=invalid-name ';\\\n"); count=\\1; tryItOut("\\\n', infileContents) - with open(str(infilename), "w") as f: + with io.open(str(infilename), "w") as f: f.write(infileContents) print() @@ -210,7 +211,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis # 1-line offset. if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: intendedLines = [] # pylint: disable=invalid-name - with open(str(infilename), "r") as f: + with io.open(str(infilename), "r") as f: for line in f: # The testcase is likely to already be partially reduced. if "dumpln(cookie" not in line: # jsfunfuzz-specific line ignore # This should be simpler than re.compile. @@ -219,7 +220,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis # The 1-line offset is added here. .replace("SPLICE DDBEGIN", "SPLICE DDBEGIN\n")) - with open(str(infilename), "w") as f: + with io.open(str(infilename), "w") as f: f.writelines(intendedLines) print() print("Running 1 instance of 2-line reduction after moving count=X to its own line...") @@ -246,14 +247,14 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis # Step 6: Run line reduction after activating SECOND DDBEGIN with a 1-line offset. if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: infileContents = [] # pylint: disable=invalid-name - with open(str(infilename), "r") as f: + with io.open(str(infilename), "r") as f: for line in f: if "NIGEBDD" in line: infileContents.append(line.replace("NIGEBDD", "DDBEGIN")) infileContents.append("\n") # The 1-line offset is added here. continue infileContents.append(line) - with open(str(infilename), "w") as f: + with io.open(str(infilename), "w") as f: f.writelines(infileContents) print() diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index b4b546ac2..1fba69f94 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function # isort:skip +import io import os import platform import shutil @@ -110,7 +111,7 @@ def make_gdb_cmd(prog_full_path, crashed_pid): is_pid_used = False core_uses_pid_path = Path("/proc/sys/kernel/core_uses_pid") if core_uses_pid_path.is_file(): - with open(str(core_uses_pid_path)) as f: + with io.open(str(core_uses_pid_path), "r") as f: is_pid_used = bool(int(f.read()[0])) # Setting [0] turns the input to a str. core_name = "core." + str(crashed_pid) if is_pid_used else "core" core_name_path = Path.cwd() / core_name @@ -188,7 +189,7 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): dbggr_cmd, stdin=None, stderr=subprocess.STDOUT, - stdout=open(str(crash_log), "w") if use_logfiles else None, + stdout=io.open(str(crash_log), "w") if use_logfiles else None, # It would be nice to use this everywhere, but it seems to be broken on Windows # (http://docs.python.org/library/subprocess.html) close_fds=(os.name == "posix"), @@ -264,7 +265,7 @@ def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): for file_name in crash_logs: full_report_path = reports_dir / file_name try: - with open(str(full_report_path)) as f: + with io.open(str(full_report_path), "r") as f: first_line = f.readline() if first_line.rstrip().endswith("[%s]" % crash_pid): if use_log_files: diff --git a/src/funfuzz/util/sm_compile_helpers.py b/src/funfuzz/util/sm_compile_helpers.py index 90674deca..fad1f6058 100644 --- a/src/funfuzz/util/sm_compile_helpers.py +++ b/src/funfuzz/util/sm_compile_helpers.py @@ -69,7 +69,7 @@ def autoconf_run(working_dir): def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Create a .busted file with the exception message and backtrace included.""" - with open(str(filename), "w") as f: + with io.open(str(filename), "w") as f: f.write("Caught exception %r (%s)\n" % (e, e)) f.write("Backtrace:\n") f.write(traceback.format_exc() + "\n") @@ -90,7 +90,7 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi elif platform.system() == "Windows": fmconf_os = "windows" - with open(str(log), "a") as f: + with io.open(str(log), "a") as f: f.write("# Information about shell:\n# \n") f.write("# Create another shell in shell-cache like this one:\n") diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index 35cac7640..278acaa8e 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -35,7 +35,7 @@ def rm_tree_incl_readonly(dir_tree): # read_only_dir = os.path.join(test_dir, "nestedReadOnlyDir") # os.mkdir(read_only_dir) # filename = os.path.join(read_only_dir, "test.txt") -# with open(filename, "w") as f: +# with io.open(filename, "w") as f: # f.write("testing\n") # os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) diff --git a/tests/js/test_link_fuzzer.py b/tests/js/test_link_fuzzer.py index 5bcf0be3e..5c5e2814c 100644 --- a/tests/js/test_link_fuzzer.py +++ b/tests/js/test_link_fuzzer.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, unicode_literals # isort:skip +import io import logging import sys import unittest @@ -37,7 +38,7 @@ def test_link_fuzzer(self): link_fuzzer.link_fuzzer(jsfunfuzz_tmp) found = False - with open(str(jsfunfuzz_tmp)) as f: + with io.open(str(jsfunfuzz_tmp), "r") as f: for line in f: if "It's looking good" in line: found = True From 5cc6a2e1ef8a1ae403e9734a149d7672215ae1c5 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 19:52:29 -0700 Subject: [PATCH 086/110] Fix encoding issues with io.open --- src/funfuzz/bot.py | 2 +- src/funfuzz/js/build_options.py | 2 +- src/funfuzz/js/compare_jit.py | 4 ++-- src/funfuzz/js/compile_shell.py | 3 ++- src/funfuzz/js/js_interesting.py | 13 +++++++------ src/funfuzz/js/link_fuzzer.py | 2 +- src/funfuzz/js/loop.py | 10 +++++----- src/funfuzz/util/file_manipulation.py | 4 ++-- src/funfuzz/util/fork_join.py | 6 +++--- src/funfuzz/util/lithium_helpers.py | 22 +++++++++++----------- src/funfuzz/util/os_ops.py | 6 +++--- src/funfuzz/util/sm_compile_helpers.py | 4 ++-- src/funfuzz/util/subprocesses.py | 2 +- 13 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 14df716a9..44e09e7a7 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -163,7 +163,7 @@ def print_machine_info(): hgrc_path = Path("~/.hg/hgrc").expanduser() if hgrc_path.is_file(): print("The hgrc of this repository is:") - with io.open(str(hgrc_path), "r") as f: + with io.open(str(hgrc_path), "r", encoding="utf-8", errors="replace") as f: hgrc_contents = f.readlines() for line in hgrc_contents: print(line.rstrip()) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index b45ddf198..de2365adb 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -243,7 +243,7 @@ def computeShellType(build_options): # pylint: disable=invalid-name,missing-par if build_options.patch_file: # We take the name before the first dot, so Windows (hopefully) does not get confused. fileName.append(build_options.patch_file.name) - with io.open(str(build_options.patch_file.resolve()), "r") as f: + with io.open(str(build_options.patch_file.resolve()), "r", encoding="utf-8", errors="replace") as f: readResult = f.read() # pylint: disable=invalid-name # Append the patch hash, but this is not equivalent to Mercurial's hash of the patch. fileName.append(hashlib.sha512(readResult).hexdigest()[:12]) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index bebf3641b..163dd65d8 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -146,7 +146,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi r.lev, r.runinfo.elapsedtime))) summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") - with io.open(str(summary_log), "w") as f: + with io.open(str(summary_log), "w", encoding="utf-8", errors="replace") as f: f.write("\n".join(r.issues + [" ".join(quote(str(x)) for x in command), "compare_jit found a more serious bug"]) + "\n") print(" %s" % " ".join(quote(str(x)) for x in command)) @@ -200,7 +200,7 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr summary = (" " + " ".join(quote(str(x)) for x in commands[0]) + "\n " + " ".join(quote(str(x)) for x in command) + "\n\n" + summary) summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") - with io.open(str(summary_log), "w") as f: + with io.open(str(summary_log), "w", encoding="utf-8", errors="replace") as f: f.write(rerunCommand + "\n\n" + summary) print("%s | %s" % (str(infilename), js_interesting.summaryString( issues, js_interesting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime))) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 5ff13cd6a..bd7b300a5 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -606,7 +606,8 @@ def sm_compile(shell): print("A shell was compiled even though there was a non-zero exit code. Continuing...") else: print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) - with io.open(str(shell.get_shell_cache_dir() / ".busted.log"), "w") as f: + with io.open(str(shell.get_shell_cache_dir() / ".busted.log"), "w", + encoding="utf-8", errors="replace") as f: f.write("The first compilation of %s rev %s failed with the following output:\n" % (shell.get_repo_name(), shell.get_hg_hash())) f.write(out.decode("utf-8", errors="replace")) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index b71fb9a69..dd2d09971 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -101,10 +101,10 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and "readlines" everything into memory. out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") - with io.open(str(out_log), "r") as f: + with io.open(str(out_log), "r", encoding="utf-8", errors="replace") as f: out = f.readlines() err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt") - with io.open(str(err_log), "r") as f: + with io.open(str(err_log), "r", encoding="utf-8", errors="replace") as f: err = f.readlines() if options.valgrind and runinfo.return_code == VALGRIND_ERROR_EXIT_CODE: @@ -117,7 +117,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa elif runinfo.sta == timed_run.CRASHED: if os_ops.grab_crash_log(str(runthis[0]), runinfo.pid, logPrefix, True): crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") - with io.open(str(crash_log), "r") as f: + with io.open(str(crash_log), "r", encoding="utf-8", errors="replace") as f: auxCrashData = [line.strip() for line in f.readlines()] elif file_manipulation.amiss(logPrefix): issues.append("malloc error") @@ -144,7 +144,8 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if activated and platform.system() == "Linux" and which("gdb") and not auxCrashData and not in_compare_jit: print("Note: No core file found on Linux - falling back to run via gdb") extracted_gdb_cmds = ["-ex", "run"] - with io.open(str(Path(__file__).parent.parent / "util" / "gdb_cmds.txt"), "r") as f: + with io.open(str(Path(__file__).parent.parent / "util" / "gdb_cmds.txt"), "r", + encoding="utf-8", errors="replace") as f: for line in f: if line.rstrip() and not line.startswith("#") and not line.startswith("echo"): extracted_gdb_cmds.append("-ex") @@ -183,7 +184,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if lev != JS_FINE: summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") - with io.open(str(summary_log), "w") as f: + with io.open(str(summary_log), "w", encoding="utf-8", errors="replace") as f: f.writelines(["Number: " + str(logPrefix) + "\n", "Command: " + " ".join(quote(str(x)) for x in runthis) + "\n"] + ["Status: " + i + "\n" for i in issues]) @@ -250,7 +251,7 @@ def summaryString(issues, level, elapsedtime): # pylint: disable=invalid-name,m def truncateFile(fn, maxSize): # pylint: disable=invalid-name,missing-docstring if fn.is_file() and fn.stat().st_size > maxSize: - with io.open(str(fn), "r+") as f: + with io.open(str(fn), "r+", encoding="utf-8", errors="replace") as f: f.truncate(maxSize) diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index 662964659..87c58f0b0 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -27,7 +27,7 @@ def link_fuzzer(target_path, prologue=""): """ base_dir = Path(__file__).parent - with io.open(str(target_path), "w") as f: # Create the full jsfunfuzz file + with io.open(str(target_path), "w", encoding="utf-8", errors="replace") as f: # Create the full jsfunfuzz file if prologue: f.write(prologue) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index bbe1a7cc0..a3adb4737 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -178,13 +178,13 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in reduced_log = (logPrefix.parent / (logPrefix.stem + "-reduced")).with_suffix(".js") [before, after] = file_manipulation.fuzzSplice(fuzzjs) - with io.open(str(out_log), "r") as f: + with io.open(str(out_log), "r", encoding="utf-8", errors="replace") as f: newfileLines = before + [ # pylint: disable=invalid-name l.replace("/*FRC-", "/*") for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after orig_log = (logPrefix.parent / (logPrefix.stem + "-orig")).with_suffix(".js") - with io.open(str(orig_log), "w") as f: + with io.open(str(orig_log), "w", encoding="utf-8", errors="replace") as f: f.writelines(newfileLines) - with io.open(str(reduced_log), "w") as f: + with io.open(str(reduced_log), "w", encoding="utf-8", errors="replace") as f: f.writelines(newfileLines) # Run Lithium and autobisectjs (make a reduced testcase and find a regression window) @@ -232,7 +232,7 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") linesToCompare = jitCompareLines(out_log, "/*FCM*/") # pylint: disable=invalid-name jitcomparefilename = (logPrefix.parent / (logPrefix.stem + "-cj-in")).with_suffix(".js") - with io.open(str(jitcomparefilename), "w") as f: + with io.open(str(jitcomparefilename), "w", encoding="utf-8", errors="replace") as f: f.writelines(linesToCompare) # pylint: disable=invalid-name anyBug = compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename, @@ -264,7 +264,7 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid "wasmIsSupported = function() { return true; };\n", "// DDBEGIN\n" ] - with io.open(str(jsfunfuzzOutputFilename), "r") as f: + with io.open(str(jsfunfuzzOutputFilename), "r", encoding="utf-8", errors="replace") as f: for line in f: if line.startswith(marker): sline = line[len(marker):] diff --git a/src/funfuzz/util/file_manipulation.py b/src/funfuzz/util/file_manipulation.py index 2e99b2597..c3cff7236 100644 --- a/src/funfuzz/util/file_manipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -18,7 +18,7 @@ def amiss(log_prefix): # pylint: disable=missing-param-doc,missing-return-doc,m """ found_something = False err_log = (log_prefix.parent / (log_prefix.stem + "-err")).with_suffix(".txt") - with io.open(str(err_log), "r") as f: + with io.open(str(err_log), "r", encoding="utf-8", errors="replace") as f: for line in f: line = line.strip("\x07").rstrip("\n") if (line.find("szone_error") != -1 or @@ -37,7 +37,7 @@ def fuzzSplice(filename): # pylint: disable=invalid-name,missing-param-doc,miss """Return the lines of a file, minus the ones between the two lines containing SPLICE.""" before = [] after = [] - with io.open(str(filename), "r") as f: + with io.open(str(filename), "r", encoding="utf-8", errors="replace") as f: for line in f: before.append(line) if line.find("SPLICE") != -1: diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index d4dc3651a..1b08a33e0 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -28,7 +28,7 @@ def forkJoin(logDir, numProcesses, fun, *someArgs): # pylint: disable=invalid-n def showFile(fn): # pylint: disable=invalid-name,missing-docstring print("==== %s ====" % fn) print() - with io.open(str(fn), "r") as f: + with io.open(str(fn), "r", encoding="utf-8", errors="replace") as f: for line in f: print(line.rstrip()) print() @@ -70,8 +70,8 @@ def log_name(log_dir, i, log_type): def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring - sys.stdout = io.open(log_name(logDir, i, "out"), "w", buffering=0) - sys.stderr = io.open(log_name(logDir, i, "err"), "w", buffering=0) + sys.stdout = io.open(log_name(logDir, i, "out"), "wb", buffering=0, encoding="utf-8", errors="replace") + sys.stderr = io.open(log_name(logDir, i, "err"), "wb", buffering=0, encoding="utf-8", errors="replace") fun(*(someArgs + (i,))) diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index ff8c5f70b..26d6dc8a2 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -70,11 +70,11 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ) print(" ".join(quote(str(x)) for x in autobisectCmd)) autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") - with io.open(str(autobisect_log), "w") as f: + with io.open(str(autobisect_log), "w", encoding="utf-8", errors="replace") as f: subprocess.run(autobisectCmd, stderr=subprocess.STDOUT, stdout=f) print("Done running autobisectjs. Log: %s" % autobisect_log) - with io.open(str(autobisect_log), "r") as f: + with io.open(str(autobisect_log), "r", encoding="utf-8", errors="replace") as f: lines = f.readlines() autobisect_log_trunc = file_manipulation.truncateMid(lines, 50, ["..."]) else: @@ -103,7 +103,7 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) print(" ".join(quote(str(x)) for x in runlithiumpy + lithArgs)) - with io.open(str(lithlogfn), "w") as f: + with io.open(str(lithlogfn), "w", encoding="utf-8", errors="replace") as f: subprocess.run(runlithiumpy + lithArgs, stderr=subprocess.STDOUT, stdout=f) print("Done running Lithium") if deletableLithTemp: @@ -115,7 +115,7 @@ def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-nam def readLithiumResult(lithlogfn): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - with io.open(str(lithlogfn), "r") as f: + with io.open(str(lithlogfn), "r", encoding="utf-8", errors="replace") as f: for line in f: if line.startswith("Lithium result"): print(line.rstrip()) @@ -179,7 +179,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis hasTryItOut = False # pylint: disable=invalid-name hasTryItOutRegex = re.compile(r'count=[0-9]+; tryItOut\("') # pylint: disable=invalid-name - with io.open(str(infilename), "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: for line in file_manipulation.linesWith(f, '; tryItOut("'): # Checks if testcase came from jsfunfuzz or compare_jit. # Do not use .match here, it only matches from the start of the line: @@ -193,12 +193,12 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis tryItOutAndCountRegex = re.compile(r'"\);\ncount=([0-9]+); tryItOut\("', # pylint: disable=invalid-name re.MULTILINE) - with io.open(str(infilename), "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: infileContents = f.read() # pylint: disable=invalid-name infileContents = re.sub(tryItOutAndCountRegex, # pylint: disable=invalid-name ';\\\n"); count=\\1; tryItOut("\\\n', infileContents) - with io.open(str(infilename), "w") as f: + with io.open(str(infilename), "w", encoding="utf-8", errors="replace") as f: f.write(infileContents) print() @@ -211,7 +211,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis # 1-line offset. if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: intendedLines = [] # pylint: disable=invalid-name - with io.open(str(infilename), "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: for line in f: # The testcase is likely to already be partially reduced. if "dumpln(cookie" not in line: # jsfunfuzz-specific line ignore # This should be simpler than re.compile. @@ -220,7 +220,7 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis # The 1-line offset is added here. .replace("SPLICE DDBEGIN", "SPLICE DDBEGIN\n")) - with io.open(str(infilename), "w") as f: + with io.open(str(infilename), "w", encoding="utf-8", errors="replace") as f: f.writelines(intendedLines) print() print("Running 1 instance of 2-line reduction after moving count=X to its own line...") @@ -247,14 +247,14 @@ def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,mis # Step 6: Run line reduction after activating SECOND DDBEGIN with a 1-line offset. if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: infileContents = [] # pylint: disable=invalid-name - with io.open(str(infilename), "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: for line in f: if "NIGEBDD" in line: infileContents.append(line.replace("NIGEBDD", "DDBEGIN")) infileContents.append("\n") # The 1-line offset is added here. continue infileContents.append(line) - with io.open(str(infilename), "w") as f: + with io.open(str(infilename), "w", encoding="utf-8", errors="replace") as f: f.writelines(infileContents) print() diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index 1fba69f94..a204d041e 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -111,7 +111,7 @@ def make_gdb_cmd(prog_full_path, crashed_pid): is_pid_used = False core_uses_pid_path = Path("/proc/sys/kernel/core_uses_pid") if core_uses_pid_path.is_file(): - with io.open(str(core_uses_pid_path), "r") as f: + with io.open(str(core_uses_pid_path), "r", encoding="utf-8", errors="replace") as f: is_pid_used = bool(int(f.read()[0])) # Setting [0] turns the input to a str. core_name = "core." + str(crashed_pid) if is_pid_used else "core" core_name_path = Path.cwd() / core_name @@ -189,7 +189,7 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): dbggr_cmd, stdin=None, stderr=subprocess.STDOUT, - stdout=io.open(str(crash_log), "w") if use_logfiles else None, + stdout=io.open(str(crash_log), "w", encoding="utf-8", errors="replace") if use_logfiles else None, # It would be nice to use this everywhere, but it seems to be broken on Windows # (http://docs.python.org/library/subprocess.html) close_fds=(os.name == "posix"), @@ -265,7 +265,7 @@ def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): for file_name in crash_logs: full_report_path = reports_dir / file_name try: - with io.open(str(full_report_path), "r") as f: + with io.open(str(full_report_path), "r", encoding="utf-8", errors="replace") as f: first_line = f.readline() if first_line.rstrip().endswith("[%s]" % crash_pid): if use_log_files: diff --git a/src/funfuzz/util/sm_compile_helpers.py b/src/funfuzz/util/sm_compile_helpers.py index fad1f6058..5a1ffeb80 100644 --- a/src/funfuzz/util/sm_compile_helpers.py +++ b/src/funfuzz/util/sm_compile_helpers.py @@ -69,7 +69,7 @@ def autoconf_run(working_dir): def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Create a .busted file with the exception message and backtrace included.""" - with io.open(str(filename), "w") as f: + with io.open(str(filename), "w", encoding="utf-8", errors="replace") as f: f.write("Caught exception %r (%s)\n" % (e, e)) f.write("Backtrace:\n") f.write(traceback.format_exc() + "\n") @@ -90,7 +90,7 @@ def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missi elif platform.system() == "Windows": fmconf_os = "windows" - with io.open(str(log), "a") as f: + with io.open(str(log), "a", encoding="utf-8", errors="replace") as f: f.write("# Information about shell:\n# \n") f.write("# Create another shell in shell-cache like this one:\n") diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index 278acaa8e..48e938d8f 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -35,7 +35,7 @@ def rm_tree_incl_readonly(dir_tree): # read_only_dir = os.path.join(test_dir, "nestedReadOnlyDir") # os.mkdir(read_only_dir) # filename = os.path.join(read_only_dir, "test.txt") -# with io.open(filename, "w") as f: +# with io.open(filename, "w", encoding="utf-8", errors="replace") as f: # f.write("testing\n") # os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) From e3fa76301c7761111d3065bcc3d9a562d3b6891d Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 20:06:55 -0700 Subject: [PATCH 087/110] Remove unneeded parenthesis. --- src/funfuzz/js/compile_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index bd7b300a5..7bf8941a6 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -298,7 +298,7 @@ def get_shell_compiled_runlibs_path(self): Path: Full path to the original location of the libraries of js binary compiled in the shell cache """ return [ - self.get_js_objdir() / "dist" / "bin" / (runlib for runlib in inspect_shell.ALL_RUN_LIBS) + self.get_js_objdir() / "dist" / "bin" / runlib for runlib in inspect_shell.ALL_RUN_LIBS ] def get_shell_name_with_ext(self): From 6e8a0f202a6a996c145036cf454ffa976d5472c7 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 20:07:24 -0700 Subject: [PATCH 088/110] Fix more encoding issues with io.open in tests. --- tests/js/test_link_fuzzer.py | 2 +- tests/util/test_fork_join.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/js/test_link_fuzzer.py b/tests/js/test_link_fuzzer.py index 5c5e2814c..14e5cb4f0 100644 --- a/tests/js/test_link_fuzzer.py +++ b/tests/js/test_link_fuzzer.py @@ -38,7 +38,7 @@ def test_link_fuzzer(self): link_fuzzer.link_fuzzer(jsfunfuzz_tmp) found = False - with io.open(str(jsfunfuzz_tmp), "r") as f: + with io.open(str(jsfunfuzz_tmp), "r", encoding="utf-8", errors="replace") as f: for line in f: if "It's looking good" in line: found = True diff --git a/tests/util/test_fork_join.py b/tests/util/test_fork_join.py index 58e4deacd..47d309c90 100644 --- a/tests/util/test_fork_join.py +++ b/tests/util/test_fork_join.py @@ -35,7 +35,7 @@ def test_log_name(self): tmp_dir = Path(tmp_dir) log_path = tmp_dir / "forkjoin-1-out.txt" - with io.open(str(log_path), "w") as f: + with io.open(str(log_path), "w", encoding="utf-8", errors="replace") as f: f.writelines("test") self.assertEqual(fork_join.log_name(tmp_dir, 1, "out"), str(log_path)) From 2632e9e2d2fb9ee55938295ab61b382b8262bdba Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 20:08:31 -0700 Subject: [PATCH 089/110] Binary mode of io.open does not take an encoding argument. --- src/funfuzz/util/fork_join.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index 1b08a33e0..f2a50169a 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -70,8 +70,8 @@ def log_name(log_dir, i, log_type): def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring - sys.stdout = io.open(log_name(logDir, i, "out"), "wb", buffering=0, encoding="utf-8", errors="replace") - sys.stderr = io.open(log_name(logDir, i, "err"), "wb", buffering=0, encoding="utf-8", errors="replace") + sys.stdout = io.open(log_name(logDir, i, "out"), "wb", buffering=0) + sys.stderr = io.open(log_name(logDir, i, "err"), "wb", buffering=0) fun(*(someArgs + (i,))) From 74bc0919ef7ee7a405d1e030ddc45bb11abae4bf Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 20:42:54 -0700 Subject: [PATCH 090/110] Fix another encoding issue. --- src/funfuzz/js/link_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index 87c58f0b0..ba3fedd11 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -35,5 +35,5 @@ def link_fuzzer(target_path, prologue=""): entry = entry.rstrip() if entry and not entry.startswith("#"): file_path = base_dir / Path(entry) - f.write("\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:]) + f.write(("\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:]).decode("utf-8", errors="replace")) f.write(file_path.read_text()) # pylint: disable=no-member From 4906b1bc5aaf489063f58b9b3f5e2abee88e5699 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 20:54:10 -0700 Subject: [PATCH 091/110] Lithium only takes in strings, not Paths nor bytes. --- src/funfuzz/js/js_interesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index dd2d09971..09f9943f5 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -89,7 +89,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # logPrefix should be a string for timed_run in Lithium version 0.2.1 to work properly, apparently runinfo = timed_run.timed_run( - [str(x) if isinstance(x, Path) else x for x in runthis], # Convert all Paths to strings for Lithium + [str(x) else x for x in runthis], # Convert all Paths/bytes to strings for Lithium options.timeout, str(logPrefix).encode("utf-8"), preexec_fn=set_ulimit) @@ -152,7 +152,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa extracted_gdb_cmds.append("%s" % line.rstrip()) no_main_log_gdb_log = subprocess.run( (["gdb", "-n", "-batch"] + extracted_gdb_cmds + ["--args"] + - [str(x) if isinstance(x, Path) else x for x in runthis]), + [str(x) for x in runthis]), check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE From 0715ae811574a2069608b77714eb7ac29972918d Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 20:59:24 -0700 Subject: [PATCH 092/110] SyntaxError fix. --- src/funfuzz/js/js_interesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 09f9943f5..c31db202e 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -89,7 +89,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # logPrefix should be a string for timed_run in Lithium version 0.2.1 to work properly, apparently runinfo = timed_run.timed_run( - [str(x) else x for x in runthis], # Convert all Paths/bytes to strings for Lithium + [str(x) for x in runthis], # Convert all Paths/bytes to strings for Lithium options.timeout, str(logPrefix).encode("utf-8"), preexec_fn=set_ulimit) From 7dc93cc4d600bc19affb76158b7329ad4ab58c30 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 21:16:58 -0700 Subject: [PATCH 093/110] Revert "Remove unneeded 'if __name__ == "__main__":' lines." This reverts commit 1eddcfc9d1c7f58777eec9fea57e70c62ea5c3e7. --- src/funfuzz/js/build_options.py | 4 ++++ src/funfuzz/js/compare_jit.py | 4 ++++ src/funfuzz/js/compile_shell.py | 4 ++++ src/funfuzz/js/js_interesting.py | 4 ++++ src/funfuzz/js/loop.py | 6 ++++++ src/funfuzz/loop_bot.py | 4 ++++ src/funfuzz/util/repos_update.py | 4 ++++ 7 files changed, 30 insertions(+) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index de2365adb..e6b48e2f8 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -382,3 +382,7 @@ def main(): # pylint: disable=missing-docstring print("Running this file directly doesn't do anything, but here's our subparser help:") print() parse_shell_opts("--help") + + +if __name__ == "__main__": + main() diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 163dd65d8..1d105b5e3 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -317,3 +317,7 @@ def main(): print(compareLevel( options.jsengine, options.flags, options.infilename, # pylint: disable=no-member tempfile.mkdtemp("compare_jitmain"), options, True, False)[0]) + + +if __name__ == "__main__": + main() diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 7bf8941a6..2c681d3aa 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -720,3 +720,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa def main(): """Execute main() function in CompiledShell class.""" exit(CompiledShell.main()) + + +if __name__ == "__main__": + main() diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index c31db202e..c7a2a62cd 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -365,3 +365,7 @@ def main(): # pylint: disable=missing-docstring options.collector.submit(res.crashInfo, testcaseFilename, quality) # pylint: disable=no-member else: print("Not submitting (not interesting)") + + +if __name__ == "__main__": + main() diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index a3adb4737..203e5970b 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -278,3 +278,9 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid "// DDEND\n" ] return lines + + +if __name__ == "__main__": + # pylint: disable=no-member + many_timed_runs(None, sps.make_wtmp_dir(Path(os.getcwdu() if sys.version_info.major == 2 else os.getcwd())), + sys.argv[1:], create_collector.make_collector()) diff --git a/src/funfuzz/loop_bot.py b/src/funfuzz/loop_bot.py index 3431773cc..ebc110132 100644 --- a/src/funfuzz/loop_bot.py +++ b/src/funfuzz/loop_bot.py @@ -51,3 +51,7 @@ def main(): # pylint: disable=missing-docstring [sys.executable, "-u", "-m", "funfuzz.util.repos_update"], [sys.executable, "-u", "-m", "funfuzz.bot"] + sys.argv[1:] ], 60) + + +if __name__ == "__main__": + main() diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index 19f74e39d..15090407c 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -138,3 +138,7 @@ def main(): # pylint: disable=missing-docstring logger.info("WARNING: OSError hit:") logger.info(ex) logger.info(time.asctime()) + + +if __name__ == "__main__": + main() From 66819275013bfc2fad342a26f6ad4afa4f56bfc0 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 21:19:24 -0700 Subject: [PATCH 094/110] Cast more paths to strings before decoding them. --- src/funfuzz/js/compile_shell.py | 2 +- src/funfuzz/util/sm_compile_helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 2c681d3aa..7fec290af 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -678,7 +678,7 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa # Print *with* a trailing newline to avoid breaking other stuff print("Updating to rev %s in the %s repository..." % ( updateToRev.decode("utf-8", errors="replace"), - shell.build_opts.repo_dir.decode("utf-8", errors="replace"))) + str(shell.build_opts.repo_dir).decode("utf-8", errors="replace"))) subprocess.run(["hg", "-R", shell.build_opts.repo_dir, "update", "-C", "-r", updateToRev], check=True, # pylint: disable=no-member diff --git a/src/funfuzz/util/sm_compile_helpers.py b/src/funfuzz/util/sm_compile_helpers.py index 5a1ffeb80..06fbeef9e 100644 --- a/src/funfuzz/util/sm_compile_helpers.py +++ b/src/funfuzz/util/sm_compile_helpers.py @@ -177,7 +177,7 @@ def verify_full_win_pageheap(shell_path): gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member print(subprocess.run([str(gflags_bin_path).decode("utf-8", errors="replace"), - "-p", "/enable", shell_path.decode("utf-8", errors="replace"), "/full"], + "-p", "/enable", str(shell_path).decode("utf-8", errors="replace"), "/full"], check=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout) From 2cd117cab15d282e4c406d7589c4fd0584c0f52a Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 21:30:03 -0700 Subject: [PATCH 095/110] Most repo_dirs are now Paths. --- src/funfuzz/autobisectjs/autobisectjs.py | 2 +- src/funfuzz/js/compile_shell.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index fa3e54713..612d2f6bb 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -360,7 +360,7 @@ def checkBlameParents(repo_dir, blamedRev, blamedGoodOrBad, labels, testRev, sta missedCommonAncestor = False hg_parent_output = subprocess.run( - ["hg", "-R", repo_dir] + ["parent", "--template={node|short},", "-r", blamedRev], + ["hg", "-R", str(repo_dir)] + ["parent", "--template={node|short},", "-r", blamedRev], check=True, cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member stdout=subprocess.PIPE, diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 7fec290af..c5b6df9cd 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -679,7 +679,8 @@ def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disa print("Updating to rev %s in the %s repository..." % ( updateToRev.decode("utf-8", errors="replace"), str(shell.build_opts.repo_dir).decode("utf-8", errors="replace"))) - subprocess.run(["hg", "-R", shell.build_opts.repo_dir, "update", "-C", "-r", updateToRev], + subprocess.run(["hg", "-R", str(shell.build_opts.repo_dir), + "update", "-C", "-r", updateToRev], check=True, # pylint: disable=no-member cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), From 480c25db6fdd3549670923fef16917c5cda2024d Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 21:33:08 -0700 Subject: [PATCH 096/110] Fix missing imports. --- src/funfuzz/js/loop.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 203e5970b..2251cf986 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -20,8 +20,10 @@ from . import js_interesting from . import link_fuzzer from . import shell_flags +from ..util import create_collector from ..util import file_manipulation from ..util import lithium_helpers +from ..util import subprocesses as sps if sys.version_info.major == 2: if os.name == "posix": From 5c90b8360c825335a1aa66a3abdf2ffafcdc3827 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 21:42:56 -0700 Subject: [PATCH 097/110] Improve dual Python 2/3 code. --- src/funfuzz/js/link_fuzzer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py index ba3fedd11..8d90a43b5 100644 --- a/src/funfuzz/js/link_fuzzer.py +++ b/src/funfuzz/js/link_fuzzer.py @@ -35,5 +35,8 @@ def link_fuzzer(target_path, prologue=""): entry = entry.rstrip() if entry and not entry.startswith("#"): file_path = base_dir / Path(entry) - f.write(("\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:]).decode("utf-8", errors="replace")) + file_name = "\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:] + if isinstance(file_name, bytes): # For dual Python 2 and 3 compatibility + file_name = file_name.decode("utf-8", errors="replace") + f.write(file_name) f.write(file_path.read_text()) # pylint: disable=no-member From 387d8c7a59c7a45e1853c1cad96f57c619597530 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 21:50:12 -0700 Subject: [PATCH 098/110] knownPath is still present so account for it for now. --- src/funfuzz/js/js_interesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index c7a2a62cd..2a72dea23 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -319,7 +319,7 @@ def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missin if len(args) < 2: raise Exception("Not enough positional arguments") options.knownPath = args[0] - options.jsengineWithArgs = [Path(args[1]).resolve()] + args[1:-1] + [Path(args[-1]).resolve()] + options.jsengineWithArgs = [Path(args[1]).resolve()] + args[2:-1] + [Path(args[-1]).resolve()] assert options.jsengineWithArgs[0].is_file() # js shell assert options.jsengineWithArgs[-1].is_file() # testcase options.collector = create_collector.make_collector() From ffb33e60d9c0dc7328a81dd025ca79405a27e087 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 21:59:24 -0700 Subject: [PATCH 099/110] Yet another pathlib issue. --- src/funfuzz/js/js_interesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 2a72dea23..8c4a7216e 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -115,7 +115,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if valgrindErrorPrefix and line.startswith(valgrindErrorPrefix): issues.append(line.rstrip()) elif runinfo.sta == timed_run.CRASHED: - if os_ops.grab_crash_log(str(runthis[0]), runinfo.pid, logPrefix, True): + if os_ops.grab_crash_log(runthis[0], runinfo.pid, logPrefix, True): crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") with io.open(str(crash_log), "r", encoding="utf-8", errors="replace") as f: auxCrashData = [line.strip() for line in f.readlines()] From 54484954e215ed228a83c40d434043ae21c5e64d Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Fri, 1 Jun 2018 22:05:09 -0700 Subject: [PATCH 100/110] Ensure that cwd_prefix is a Path in js_interesting. --- src/funfuzz/js/js_interesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 8c4a7216e..248caccad 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -341,6 +341,7 @@ def init(args): # pylint: disable=missing-docstring # FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme def interesting(_args, cwd_prefix): # pylint: disable=missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc + cwd_prefix = Path(cwd_prefix) # Lithium uses this function and cwd_prefix from Lithium is not a Path options = gOptions # options, runthis, logPrefix, in_compare_jit res = ShellResult(options, options.jsengineWithArgs, cwd_prefix, False) From b5db2ac2292e748d4147b10e3bfa8df69935af04 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 14:05:32 -0700 Subject: [PATCH 101/110] Collector takes in strings, not Paths. --- src/funfuzz/js/js_interesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 248caccad..ae4995a78 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -363,7 +363,7 @@ def main(): # pylint: disable=missing-docstring testcaseFilename = options.jsengineWithArgs[-1] # pylint: disable=invalid-name,no-member print("Submitting %s" % testcaseFilename) quality = 0 - options.collector.submit(res.crashInfo, testcaseFilename, quality) # pylint: disable=no-member + options.collector.submit(res.crashInfo, str(testcaseFilename), quality) # pylint: disable=no-member else: print("Not submitting (not interesting)") From 8a281eebbb020e5c71d35400591c085395e74a20 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 14:21:17 -0700 Subject: [PATCH 102/110] Expand the repo_dir path first, in build_options. --- src/funfuzz/js/build_options.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index e6b48e2f8..9090ff99b 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -191,7 +191,9 @@ def parse_shell_opts(args): # Ensures releng machines do not enter the if block and assumes mozilla-central always exists if DEFAULT_TREES_LOCATION.is_dir(): # pylint: disable=no-member # Repositories do not get randomized if a repository is specified. - if build_options.repo_dir is None: + if build_options.repo_dir: + build_options.repo_dir = build_options.repo_dir.expanduser() + else: # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead if build_options.enableRandom and not build_options.patch_file: build_options.repo_dir = get_random_valid_repo(DEFAULT_TREES_LOCATION) From 5529beb091ec06b7dcc93482d6196382cb7fbe97 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 15:16:19 -0700 Subject: [PATCH 103/110] Add an unrelated pylint ignore line. --- src/funfuzz/js/build_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 9090ff99b..36205a5fa 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -162,7 +162,7 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): # pylint return parser, randomizer -def parse_shell_opts(args): +def parse_shell_opts(args): # pylint: disable=too-many-branches """Parses shell options into a build_options object. Args: From 73adcebfa562998b530939c99ac0ef6544341c05 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 15:26:53 -0700 Subject: [PATCH 104/110] Run Paths in the parsing function of compare_jit through .expanduser() first. --- src/funfuzz/js/compare_jit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 1d105b5e3..a5e3ae370 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -284,9 +284,9 @@ def parseOptions(args): # pylint: disable=invalid-name options, args = parser.parse_args(args) if len(args) != 3: raise Exception("Wrong number of positional arguments. Need 3 (knownPath, jsengine, infilename).") - options.knownPath = Path(args[0]).resolve() - options.jsengine = Path(args[1]).resolve() - options.infilename = Path(args[2]).resolve() + options.knownPath = Path(args[0]).expanduser().resolve() + options.jsengine = Path(args[1]).expanduser().resolve() + options.infilename = Path(args[2]).expanduser().resolve() options.flags = options.flagsSpaceSep.split(" ") if options.flagsSpaceSep else [] if not options.jsengine.is_file(): raise Exception("js shell does not exist: " + options.jsengine) From b61466d20a055a45e57433cd3c1878b43aefb08c Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 15:27:31 -0700 Subject: [PATCH 105/110] Raise an OSError if a file does not exist in the parsing function of compare_jit instead of a generic Exception. --- src/funfuzz/js/compare_jit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index a5e3ae370..4a9087f20 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -289,7 +289,7 @@ def parseOptions(args): # pylint: disable=invalid-name options.infilename = Path(args[2]).expanduser().resolve() options.flags = options.flagsSpaceSep.split(" ") if options.flagsSpaceSep else [] if not options.jsengine.is_file(): - raise Exception("js shell does not exist: " + options.jsengine) + raise OSError("js shell does not exist: " + options.jsengine) # For js_interesting: options.valgrind = False From a676d8ef1eb2cb5a76b24ecf448dbd06332758a7 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 15:27:57 -0700 Subject: [PATCH 106/110] Ensure cwd_prefix is a Path in compare_jit. --- src/funfuzz/js/compare_jit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 4a9087f20..ef48a8381 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -307,6 +307,7 @@ def init(args): # FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme def interesting(_args, cwd_prefix): + cwd_prefix = Path(cwd_prefix) # Lithium uses this function and cwd_prefix from Lithium is not a Path actualLevel = compareLevel( # pylint: disable=invalid-name gOptions.jsengine, gOptions.flags, gOptions.infilename, cwd_prefix, gOptions, False, False)[0] return actualLevel >= gOptions.minimumInterestingLevel From 59644e5ac7b48febda3796dc2ba1cbcd02bf0016 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 15:28:32 -0700 Subject: [PATCH 107/110] When running compare_jit standalone, ensure the temp folder that is created returns its result in the form of a Path. --- src/funfuzz/js/compare_jit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index ef48a8381..0fc6fd07c 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -317,7 +317,7 @@ def main(): options = parseOptions(sys.argv[1:]) print(compareLevel( options.jsengine, options.flags, options.infilename, # pylint: disable=no-member - tempfile.mkdtemp("compare_jitmain"), options, True, False)[0]) + Path(tempfile.mkdtemp("compare_jitmain")), options, True, False)[0]) if __name__ == "__main__": From c806081c8a1bc1d9ae4774978012e9a028c0b5b4 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 15:36:43 -0700 Subject: [PATCH 108/110] Path does not care that it is already working on a Path instance. --- src/funfuzz/js/compare_jit.py | 3 +-- src/funfuzz/js/js_interesting.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 0fc6fd07c..a08b7c515 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -70,8 +70,7 @@ def compare_jit(jsEngine, # pylint: disable=invalid-name,missing-param-doc,miss """ # pylint: disable=too-many-locals # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. - if not isinstance(logPrefix, Path): - logPrefix = Path(logPrefix) + logPrefix = Path(logPrefix) initialdir_name = (logPrefix.parent / (logPrefix.stem + "-initial")) # pylint: disable=invalid-name cl = compareLevel(jsEngine, flags, infilename, initialdir_name, options, False, True) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index ae4995a78..3921a91e6 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -71,8 +71,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # pylint: disable=too-many-locals,too-many-statements # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. - if not isinstance(logPrefix, Path): - logPrefix = Path(logPrefix) + logPrefix = Path(logPrefix) pathToBinary = runthis[0].resolve() # pylint: disable=invalid-name # This relies on the shell being a local one from compile_shell: # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like "./js" From 6e61ad194f7c9a7edd6687d3c0f643d9db1dfeaa Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 15:37:15 -0700 Subject: [PATCH 109/110] Make sure also to run pathToBinary through .expanduser() --- src/funfuzz/js/js_interesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 3921a91e6..6c93ace7e 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -72,7 +72,7 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. logPrefix = Path(logPrefix) - pathToBinary = runthis[0].resolve() # pylint: disable=invalid-name + pathToBinary = runthis[0].expanduser().resolve() # pylint: disable=invalid-name # This relies on the shell being a local one from compile_shell: # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like "./js" # pylint: disable=invalid-name From 79eb6a9f55c912e7042b334fe836e2e8a1836e06 Mon Sep 17 00:00:00 2001 From: Gary Kwong Date: Sat, 2 Jun 2018 16:49:50 -0700 Subject: [PATCH 110/110] Add more pylint ignore lines to suit PyPy3. --- src/funfuzz/bot.py | 1 + src/funfuzz/js/js_interesting.py | 4 ++-- src/funfuzz/util/os_ops.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index 44e09e7a7..b58013d37 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -171,6 +171,7 @@ def print_machine_info(): try: # resource library is only applicable to Linux or Mac platforms. import resource # pylint: disable=import-error + # pylint: disable=no-member print("Corefile size (soft limit, hard limit) is: %r" % (resource.getrlimit(resource.RLIMIT_CORE),)) except ImportError: print("Not checking corefile size as resource module is unavailable") diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index 6c93ace7e..b9ef08443 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -285,11 +285,11 @@ def set_ulimit(): # log.debug("Limit address space to 2GB (or 1GB on ARM boards such as ODROID)") giga_byte = 2**30 - resource.setrlimit(resource.RLIMIT_AS, (2 * giga_byte, 2 * giga_byte)) + resource.setrlimit(resource.RLIMIT_AS, (2 * giga_byte, 2 * giga_byte)) # pylint: disable=no-member # log.debug("Limit corefiles to 0.5 GB") half_giga_byte = int(giga_byte // 2) - resource.setrlimit(resource.RLIMIT_CORE, (half_giga_byte, half_giga_byte)) + resource.setrlimit(resource.RLIMIT_CORE, (half_giga_byte, half_giga_byte)) # pylint: disable=no-member except ImportError: # log.debug("Skipping resource import as a non-POSIX platform was detected: %s", platform.system()) return diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py index a204d041e..8d3853ac9 100644 --- a/src/funfuzz/util/os_ops.py +++ b/src/funfuzz/util/os_ops.py @@ -230,6 +230,7 @@ def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): elif platform.system() == "Linux": print("Warning: grab_crash_log() did not find a core file for PID %d." % crashed_pid) print("Note: Your soft limit for core file sizes is currently %d. " + # pylint: disable=indexing-exception 'You can increase it with "ulimit -c" in bash.' % get_core_limit()[0])