diff --git a/src/funfuzz/autobisectjs/autobisectjs.py b/src/funfuzz/autobisectjs/autobisectjs.py index 8936f6af1..612d2f6bb 100644 --- a/src/funfuzz/autobisectjs/autobisectjs.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -7,13 +7,12 @@ """autobisectjs, for bisecting changeset regression windows. Supports Mercurial repositories and SpiderMonkey only. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, unicode_literals # isort:skip from optparse import OptionParser # pylint: disable=deprecated-module import os import re import shutil -import subprocess import sys import tempfile import time @@ -27,9 +26,18 @@ from ..js import inspect_shell from ..util import hg_helpers from ..util import s3cache +from ..util import sm_compile_helpers from ..util import subprocesses as sps from ..util.lock_dir import LockDir +if sys.version_info.major == 2: + from pathlib2 import Path + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc # pylint: disable=too-many-branches,too-complex,too-many-statements @@ -112,16 +120,17 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur print_("TBD: Bisection using downloaded shells is temporarily not supported.", flush=True) sys.exit(0) - options.build_options = build_options.parseShellOptions(options.build_options) + options.build_options = build_options.parse_shell_opts(options.build_options) options.skipRevs = " + ".join(kbew.known_broken_ranges(options.build_options)) - options.paramList = [sps.normExpUserPath(x) for x in options.parameters.split(" ") if x] + options.runtime_params = [x for x in options.parameters.split(" ") if x] + # First check that the testcase is present. - if "-e 42" not in options.parameters and not os.path.isfile(options.paramList[-1]): + if "-e 42" not in options.parameters and not Path(options.runtime_params[-1]).expanduser().is_file(): print_(flush=True) - print_("List of parameters to be passed to the shell is: %s" % " ".join(options.paramList), flush=True) + print_("List of parameters to be passed to the shell is: %s" % " ".join(options.runtime_params), flush=True) print_(flush=True) - raise Exception("Testcase at " + options.paramList[-1] + " is not present.") + raise OSError("Testcase at %s is not present." % options.runtime_params[-1]) assert options.compilationFailedLabel in ("bad", "good", "skip") @@ -141,24 +150,24 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur options.testAndLabel = internalTestAndLabel(options) earliestKnownQuery = kbew.earliest_known_working_rev( # pylint: disable=invalid-name - options.build_options, options.paramList + extraFlags, options.skipRevs) + options.build_options, options.runtime_params + extraFlags, options.skipRevs) earliestKnown = "" # pylint: disable=invalid-name if not options.useTreeherderBinaries: # pylint: disable=invalid-name - earliestKnown = hg_helpers.getRepoHashAndId(options.build_options.repoDir, repoRev=earliestKnownQuery)[0] + earliestKnown = hg_helpers.get_repo_hash_and_id(options.build_options.repo_dir, repo_rev=earliestKnownQuery)[0] if options.startRepo is None: if options.useTreeherderBinaries: options.startRepo = "default" else: options.startRepo = earliestKnown - # elif not (options.useTreeherderBinaries or hg_helpers.isAncestor(options.build_options.repoDir, + # elif not (options.useTreeherderBinaries or hg_helpers.isAncestor(options.build_options.repo_dir, # earliestKnown, options.startRepo)): # raise Exception("startRepo is not a descendant of kbew.earliestKnownWorkingRev for this configuration") # - # if not options.useTreeherderBinaries and not hg_helpers.isAncestor(options.build_options.repoDir, + # if not options.useTreeherderBinaries and not hg_helpers.isAncestor(options.build_options.repo_dir, # earliestKnown, options.endRepo): # raise Exception("endRepo is not a descendant of kbew.earliestKnownWorkingRev for this configuration") @@ -172,29 +181,36 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur return options -def findBlamedCset(options, repoDir, testRev): # pylint: disable=invalid-name,missing-docstring,too-complex +def findBlamedCset(options, repo_dir, testRev): # pylint: disable=invalid-name,missing-docstring,too-complex # pylint: disable=too-many-locals,too-many-statements - print_("%s | Bisecting on: %s" % (time.asctime(), repoDir), flush=True) + repo_dir = str(repo_dir) + print_("%s | Bisecting on: %s" % (time.asctime(), repo_dir), flush=True) - hgPrefix = ["hg", "-R", repoDir] # pylint: disable=invalid-name + hgPrefix = ["hg", "-R", repo_dir] # pylint: disable=invalid-name # Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8". # pylint: disable=invalid-name - realStartRepo = sRepo = hg_helpers.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0] + realStartRepo = sRepo = hg_helpers.get_repo_hash_and_id(repo_dir, repo_rev=options.startRepo)[0] # pylint: disable=invalid-name - realEndRepo = eRepo = hg_helpers.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] + realEndRepo = eRepo = hg_helpers.get_repo_hash_and_id(repo_dir, repo_rev=options.endRepo)[0] sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo) # Refresh source directory (overwrite all local changes) to default tip if required. if options.resetRepoFirst: - subprocess.check_call(hgPrefix + ["update", "-C", "default"]) + subprocess.run(hgPrefix + ["update", "-C", "default"], check=True) # Throws exit code 255 if purge extension is not enabled in .hgrc: - subprocess.check_call(hgPrefix + ["purge", "--all"]) + subprocess.run(hgPrefix + ["purge", "--all"], check=True) # Reset bisect ranges and set skip ranges. - sps.captureStdout(hgPrefix + ["bisect", "-r"]) + subprocess.run(hgPrefix + ["bisect", "-r"], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + timeout=99) if options.skipRevs: - sps.captureStdout(hgPrefix + ["bisect", "--skip", options.skipRevs]) + subprocess.run(hgPrefix + ["bisect", "--skip", options.skipRevs], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + timeout=300) labels = {} # Specify `hg bisect` ranges. @@ -203,9 +219,15 @@ def findBlamedCset(options, repoDir, testRev): # pylint: disable=invalid-name,m else: labels[sRepo] = ("good", "assumed start rev is good") labels[eRepo] = ("bad", "assumed end rev is bad") - subprocess.check_call(hgPrefix + ["bisect", "-U", "-g", sRepo]) + subprocess.run(hgPrefix + ["bisect", "-U", "-g", sRepo], check=True) + mid_bisect_output = subprocess.run( + hgPrefix + ["bisect", "-U", "-b", eRepo], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=300).stdout.decode("utf-8", errors="replace") currRev = hg_helpers.get_cset_hash_from_bisect_msg( - sps.captureStdout(hgPrefix + ["bisect", "-U", "-b", eRepo])[0].split("\n")[0]) + mid_bisect_output.split("\n")) iterNum = 1 if options.testInitialRevs: @@ -248,15 +270,18 @@ def findBlamedCset(options, repoDir, testRev): # pylint: disable=invalid-name,m print_("This iteration took %.3f seconds to run." % oneRunTime, flush=True) if blamedRev is not None: - checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, + checkBlameParents(repo_dir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, realEndRepo) sps.vdump("Resetting bisect") - subprocess.check_call(hgPrefix + ["bisect", "-U", "-r"]) + subprocess.run(hgPrefix + ["bisect", "-U", "-r"], check=True) sps.vdump("Resetting working directory") - sps.captureStdout(hgPrefix + ["update", "-C", "-r", "default"], ignoreStderr=True) - hg_helpers.destroyPyc(repoDir) + subprocess.run(hgPrefix + ["update", "-C", "-r", "default"], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + timeout=999) + hg_helpers.destroyPyc(repo_dir) print_(time.asctime(), flush=True) @@ -267,7 +292,7 @@ def internalTestAndLabel(options): # pylint: disable=invalid-name,missing-param def inner(shellFilename, _hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc,too-many-return-statements # pylint: disable=invalid-name - (stdoutStderr, exitCode) = inspect_shell.testBinary(shellFilename, options.paramList, + (stdoutStderr, exitCode) = inspect_shell.testBinary(shellFilename, options.runtime_params, options.build_options.runWithVg) if (stdoutStderr.find(options.output) != -1) and (options.output != ""): @@ -310,30 +335,37 @@ def externalTestAndLabel(options, interestingness): # pylint: disable=invalid-n def inner(shellFilename, hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList # pylint: disable=invalid-name - tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash) # pylint: disable=invalid-name - tempPrefix = os.path.join(tempDir, "t") # pylint: disable=invalid-name + # pylint: disable=invalid-name + conditionArgs = conditionArgPrefix + [str(shellFilename)] + options.runtime_params + temp_dir = Path(tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash)) + temp_prefix = temp_dir / "t" if hasattr(conditionScript, "init"): # Since we're changing the js shell name, call init() again! conditionScript.init(conditionArgs) - if conditionScript.interesting(conditionArgs, tempPrefix): + if conditionScript.interesting(conditionArgs, str(temp_prefix)): innerResult = ("bad", "interesting") # pylint: disable=invalid-name else: innerResult = ("good", "not interesting") # pylint: disable=invalid-name - if os.path.isdir(tempDir): - sps.rmTreeIncludingReadOnly(tempDir) + if temp_dir.is_dir(): + sps.rm_tree_incl_readonly(str(temp_dir)) return innerResult return inner # pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-many-arguments -def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo): +def checkBlameParents(repo_dir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo): """If bisect blamed a merge, try to figure out why.""" + repo_dir = str(repo_dir) bisectLied = False missedCommonAncestor = False - parents = sps.captureStdout(["hg", "-R", repoDir] + ["parent", "--template={node|short},", - "-r", blamedRev])[0].split(",")[:-1] + hg_parent_output = subprocess.run( + ["hg", "-R", str(repo_dir)] + ["parent", "--template={node|short},", "-r", blamedRev], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=99).stdout.decode("utf-8", errors="replace") + parents = hg_parent_output.split(",")[:-1] if len(parents) == 1: return @@ -343,8 +375,8 @@ def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, star if labels.get(p) is None: print_(flush=True) print_("Oops! We didn't test rev %s, a parent of the blamed revision! Let's do that now." % p, flush=True) - if not hg_helpers.isAncestor(repoDir, startRepo, p) and \ - not hg_helpers.isAncestor(repoDir, endRepo, p): + if not hg_helpers.isAncestor(repo_dir, startRepo, p) and \ + not hg_helpers.isAncestor(repo_dir, endRepo, p): print_("We did not test rev %s because it is not a descendant of either %s or %s." % ( p, startRepo, endRepo), flush=True) # Note this in case we later decide the bisect result is wrong. @@ -366,7 +398,7 @@ def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, star # Explain why bisect blamed the merge. if bisectLied: if missedCommonAncestor: - ca = hg_helpers.findCommonAncestor(repoDir, parents[0], parents[1]) + ca = hg_helpers.findCommonAncestor(repo_dir, parents[0], parents[1]) print_(flush=True) print_("Bisect blamed the merge because our initial range did not include one", flush=True) print_("of the parents.", flush=True) @@ -394,7 +426,7 @@ def sanitizeCsetMsg(msg, repo): # pylint: disable=missing-param-doc,missing-ret for line in msgList: if line.find("<") != -1 and line.find("@") != -1 and line.find(">") != -1: line = " ".join(line.split(" ")[:-1]) - elif line.startswith("changeset:") and "mozilla-central" in repo: + elif line.startswith("changeset:") and "mozilla-central" in str(repo): line = "changeset: https://hg.mozilla.org/mozilla-central/rev/" + line.split(":")[-1] sanitizedMsgList.append(line) return "\n".join(sanitizedMsgList) @@ -405,15 +437,20 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl # pylint: disable=too-many-arguments """Tell hg what we learned about the revision.""" assert hgLabel in ("good", "bad", "skip") - outputResult = sps.captureStdout(hgPrefix + ["bisect", "-U", "--" + hgLabel, currRev])[0] + outputResult = subprocess.run( + hgPrefix + ["bisect", "-U", "--" + hgLabel, currRev], + check=True, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=999).stdout.decode("utf-8", errors="replace") outputLines = outputResult.split("\n") if options.build_options: - repoDir = options.build_options.repoDir + repo_dir = options.build_options.repo_dir if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]): print_(flush=True) - print_(sanitizeCsetMsg(outputResult, repoDir), flush=True) + print_(sanitizeCsetMsg(outputResult, repo_dir), flush=True) print_(flush=True) return None, None, None, startRepo, endRepo @@ -424,7 +461,7 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl print_(flush=True) print_("autobisectjs shows this is probably related to the following changeset:", flush=True) print_(flush=True) - print_(sanitizeCsetMsg(outputResult, repoDir), flush=True) + print_(sanitizeCsetMsg(outputResult, repo_dir), flush=True) print_(flush=True) blamedGoodOrBad = m.group(1) blamedRev = hg_helpers.get_cset_hash_from_bisect_msg(outputLines[1]) @@ -439,8 +476,8 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl currRev = hg_helpers.get_cset_hash_from_bisect_msg(outputLines[0]) if currRev is None: print_("Resetting to default revision...", flush=True) - subprocess.check_call(hgPrefix + ["update", "-C", "default"]) - hg_helpers.destroyPyc(repoDir) + subprocess.run(hgPrefix + ["update", "-C", "default"], check=True) + hg_helpers.destroyPyc(repo_dir) raise Exception("hg did not suggest a changeset to test!") # Update the startRepo/endRepo values. @@ -456,8 +493,15 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pyl return None, None, currRev, start, end -def rmOldLocalCachedDirs(cacheDir): # pylint: disable=missing-param-doc,missing-type-doc - """Remove old local cached directories, which were created four weeks ago.""" +def rm_old_local_cached_dirs(cache_dir): + """Remove old local cached directories, which were created four weeks ago. + + Args: + cache_dir (Path): Full path to the cache directory + """ + assert isinstance(cache_dir, Path) + cache_dir = cache_dir.expanduser() + # This is in autobisectjs because it has a lock so we do not race while removing directories # Adapted from http://stackoverflow.com/a/11337407 SECONDS_IN_A_DAY = 24 * 60 * 60 @@ -467,14 +511,13 @@ def rmOldLocalCachedDirs(cacheDir): # pylint: disable=missing-param-doc,missing else: NUMBER_OF_DAYS = 28 - cacheDir = sps.normExpUserPath(cacheDir) - names = [os.path.join(cacheDir, fname) for fname in os.listdir(cacheDir)] + names = [cache_dir / x for x in cache_dir.iterdir()] for name in names: - if os.path.isdir(name): - timediff = time.mktime(time.gmtime()) - os.stat(name).st_atime + if name.is_dir(): + timediff = time.mktime(time.gmtime()) - Path.stat(name).st_atime if timediff > SECONDS_IN_A_DAY * NUMBER_OF_DAYS: - shutil.rmtree(name) + shutil.rmtree(str(name)) def main(): @@ -482,16 +525,16 @@ def main(): options = parseOpts() if options.build_options: - repoDir = options.build_options.repoDir + repo_dir = options.build_options.repo_dir - with LockDir(compile_shell.getLockDirPath(options.nameOfTreeherderBranch, tboxIdentifier="Tbox") - if options.useTreeherderBinaries else compile_shell.getLockDirPath(repoDir)): + with LockDir(sm_compile_helpers.get_lock_dir_path(Path.home(), options.nameOfTreeherderBranch, tbox_id="Tbox") + if options.useTreeherderBinaries else sm_compile_helpers.get_lock_dir_path(Path.home(), repo_dir)): if options.useTreeherderBinaries: print_("TBD: We need to switch to the autobisect repository.", flush=True) sys.exit(0) else: # Bisect using local builds - findBlamedCset(options, repoDir, compile_shell.makeTestRev(options)) + findBlamedCset(options, repo_dir, compile_shell.makeTestRev(options)) # Last thing we do while we have a lock. # Note that this only clears old *local* cached directories, not remote ones. - rmOldLocalCachedDirs(compile_shell.ensureCacheDir()) + rm_old_local_cached_dirs(sm_compile_helpers.ensure_cache_dir(Path.home())) diff --git a/src/funfuzz/autobisectjs/known_broken_earliest_working.py b/src/funfuzz/autobisectjs/known_broken_earliest_working.py index 20093a8e9..01c7bb97b 100644 --- a/src/funfuzz/autobisectjs/known_broken_earliest_working.py +++ b/src/funfuzz/autobisectjs/known_broken_earliest_working.py @@ -7,7 +7,7 @@ """Known broken changeset ranges of SpiderMonkey are specified in this file. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, unicode_literals # isort:skip import os import platform @@ -129,7 +129,8 @@ def earliest_known_working_rev(options, flags, skip_revs): # pylint: disable=mi # Note that the sed version check only works with GNU sed, not BSD sed found in macOS. if (platform.system() == "Linux" and parse_version(subprocess.run(["sed", "--version"], - stdout=subprocess.PIPE).stdout.split()[3]) >= parse_version("4.3")): + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace").split()[3]) + >= parse_version("4.3")): required.append("ebcbf47a83e7") # m-c 328765 Fx53, 1st w/ working builds using sed 4.3+ found on Ubuntu 17.04+ if options.disableProfiling: required.append("800a887c705e") # m-c 324836 Fx53, 1st w/ --disable-profiling, see bug 1321065 diff --git a/src/funfuzz/bot.py b/src/funfuzz/bot.py index ba3957585..b58013d37 100644 --- a/src/funfuzz/bot.py +++ b/src/funfuzz/bot.py @@ -8,9 +8,10 @@ """ -from __future__ import absolute_import, division, print_function # isort:skip +from __future__ import absolute_import, division, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin +import io import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module import os @@ -20,19 +21,26 @@ import tempfile import time +from whichcraft import which + from .js import build_options from .js import compile_shell from .js import loop from .util import create_collector from .util import fork_join from .util import hg_helpers -from .util import subprocesses as sps +from .util import sm_compile_helpers from .util.lock_dir import LockDir if sys.version_info.major == 2: + from pathlib2 import Path import psutil + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + import subprocess + from pathlib import Path # pylint: disable=import-error -path0 = os.path.dirname(os.path.abspath(__file__)) # pylint: disable=invalid-name JS_SHELL_DEFAULT_TIMEOUT = 24 # see comments in loop for tradeoffs @@ -84,7 +92,8 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur if args: print("Warning: bot does not use positional arguments") - if not options.useTreeherderBuilds and not os.path.isdir(build_options.DEFAULT_TREES_LOCATION): + # pylint: disable=no-member + if not options.useTreeherderBuilds and not build_options.DEFAULT_TREES_LOCATION.is_dir(): # We don't have trees, so we must use treeherder builds. options.useTreeherderBuilds = True print() @@ -102,11 +111,11 @@ def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-retur def main(): # pylint: disable=missing-docstring - printMachineInfo() + print_machine_info() options = parseOpts() - collector = create_collector.createCollector("jsfunfuzz") + collector = create_collector.make_collector() try: collector.refresh() except RuntimeError as ex: @@ -118,10 +127,10 @@ def main(): # pylint: disable=missing-docstring print(options.tempDir) build_info = ensureBuild(options) - assert os.path.isdir(build_info.buildDir) + assert build_info.buildDir.is_dir() number_of_processes = multiprocessing.cpu_count() - if "-asan" in build_info.buildDir: + if "-asan" in str(build_info.buildDir): # This should really be based on the amount of RAM available, but I don't know how to compute that in Python. # I could guess 1 GB RAM per core, but that wanders into sketchyville. number_of_processes = max(number_of_processes // 2, 1) @@ -131,22 +140,19 @@ def main(): # pylint: disable=missing-docstring shutil.rmtree(options.tempDir) -def printMachineInfo(): # pylint: disable=invalid-name +def print_machine_info(): """Log information about the machine.""" print("Platform details: %s" % " ".join(platform.uname())) - print("hg version: %s" % sps.captureStdout(["hg", "-q", "version"])[0]) - # In here temporarily to see if mock Linux slaves on TBPL have gdb installed - try: - print("gdb version: %s" % sps.captureStdout(["gdb", "--version"], combineStderr=True, - ignoreStderr=True, ignoreExitCode=True)[0]) - except (KeyboardInterrupt, Exception) as ex: # pylint: disable=broad-except - print("Error involving gdb is: %r" % (ex,)) - - # FIXME: Should have if os.path.exists(path to git) or something # pylint: disable=fixme - # print("git version: %s" % sps.captureStdout(["git", "--version"], combineStderr=True, - # ignoreStderr=True, ignoreExitCode=True)[0]) + print("hg info: %s" % subprocess.run(["hg", "-q", "version"], check=True, stdout=subprocess.PIPE).stdout.rstrip()) + if which("gdb"): + gdb_version = subprocess.run(["gdb", "--version"], + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + print("gdb info: %s" % gdb_version.split("\n")[0]) + if which("git"): + print("git info: %s" % subprocess.run(["git", "version"], check=True, stdout=subprocess.PIPE).stdout.rstrip()) print("Python version: %s" % sys.version.split()[0]) + print("Number of cores visible to OS: %d" % multiprocessing.cpu_count()) if sys.version_info.major == 2: rootdir_free_space = psutil.disk_usage("/").free / (1024 ** 3) @@ -154,18 +160,21 @@ def printMachineInfo(): # pylint: disable=invalid-name rootdir_free_space = shutil.disk_usage("/").free / (1024 ** 3) # pylint: disable=no-member print("Free space (GB): %.2f" % rootdir_free_space) - hgrc_path = os.path.join(path0, ".hg", "hgrc") - if os.path.isfile(hgrc_path): + hgrc_path = Path("~/.hg/hgrc").expanduser() + if hgrc_path.is_file(): print("The hgrc of this repository is:") - with open(hgrc_path, "r") as f: + with io.open(str(hgrc_path), "r", encoding="utf-8", errors="replace") as f: hgrc_contents = f.readlines() for line in hgrc_contents: print(line.rstrip()) - if os.name == "posix": + try: # resource library is only applicable to Linux or Mac platforms. import resource # pylint: disable=import-error + # pylint: disable=no-member print("Corefile size (soft limit, hard limit) is: %r" % (resource.getrlimit(resource.RLIMIT_CORE),)) + except ImportError: + print("Not checking corefile size as resource module is unavailable") def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -177,16 +186,16 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss bRev = "" # pylint: disable=invalid-name manyTimedRunArgs = [] # pylint: disable=invalid-name elif not options.useTreeherderBuilds: - options.build_options = build_options.parseShellOptions(options.build_options) + options.build_options = build_options.parse_shell_opts(options.build_options) options.timeout = options.timeout or (300 if options.build_options.runWithVg else JS_SHELL_DEFAULT_TIMEOUT) - with LockDir(compile_shell.getLockDirPath(options.build_options.repoDir)): - bRev = hg_helpers.getRepoHashAndId(options.build_options.repoDir)[0] # pylint: disable=invalid-name + with LockDir(sm_compile_helpers.get_lock_dir_path(Path.home(), options.build_options.repo_dir)): + bRev = hg_helpers.get_repo_hash_and_id(options.build_options.repo_dir)[0] # pylint: disable=invalid-name cshell = compile_shell.CompiledShell(options.build_options, bRev) - updateLatestTxt = (options.build_options.repoDir == "mozilla-central") # pylint: disable=invalid-name + updateLatestTxt = (options.build_options.repo_dir == "mozilla-central") # pylint: disable=invalid-name compile_shell.obtainShell(cshell, updateLatestTxt=updateLatestTxt) - bDir = cshell.getShellCacheDir() # pylint: disable=invalid-name + bDir = cshell.get_shell_cache_dir() # pylint: disable=invalid-name # Strip out first 3 chars or else the dir name in fuzzing jobs becomes: # js-js-dbg-opt-64-dm-linux bType = build_options.computeShellType(options.build_options)[3:] # pylint: disable=invalid-name @@ -199,9 +208,9 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss "==============================================\n\n" % ( "funfuzz.js.compile_shell", options.build_options.build_options_str, - options.build_options.repoDir, + options.build_options.repo_dir, bRev, - cshell.getRepoName(), + cshell.get_repo_name(), time.asctime() )) @@ -216,7 +225,7 @@ def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,miss def loopFuzzingAndReduction(options, buildInfo, collector, i): # pylint: disable=invalid-name,missing-docstring - tempDir = tempfile.mkdtemp("loop" + str(i)) # pylint: disable=invalid-name + tempDir = Path(tempfile.mkdtemp("loop" + str(i))) # pylint: disable=invalid-name loop.many_timed_runs(options.targetTime, tempDir, buildInfo.mtrArgs, collector) @@ -224,7 +233,7 @@ def mtrArgsCreation(options, cshell): # pylint: disable=invalid-name,missing-pa # pylint: disable=missing-return-type-doc,missing-type-doc """Create many_timed_run arguments for compiled builds.""" manyTimedRunArgs = [] # pylint: disable=invalid-name - manyTimedRunArgs.append("--repo=" + sps.normExpUserPath(options.build_options.repoDir)) + manyTimedRunArgs.append("--repo=%s" % options.build_options.repo_dir) manyTimedRunArgs.append("--build=" + options.build_options.build_options_str) if options.build_options.runWithVg: manyTimedRunArgs.append("--valgrind") @@ -236,10 +245,6 @@ def mtrArgsCreation(options, cshell): # pylint: disable=invalid-name,missing-pa # Ordering of elements in manyTimedRunArgs is important. manyTimedRunArgs.append(str(options.timeout)) - manyTimedRunArgs.append(cshell.getRepoName()) # known bugs' directory - manyTimedRunArgs.append(cshell.getShellCacheFullPath()) + manyTimedRunArgs.append(cshell.get_repo_name()) # known bugs' directory + manyTimedRunArgs.append(cshell.get_shell_cache_js_bin_path()) return manyTimedRunArgs - - -if __name__ == "__main__": - main() diff --git a/src/funfuzz/ccoverage/get_build.py b/src/funfuzz/ccoverage/get_build.py index e7a2f96b1..40ec48f3a 100644 --- a/src/funfuzz/ccoverage/get_build.py +++ b/src/funfuzz/ccoverage/get_build.py @@ -11,12 +11,17 @@ import io import logging -import os +import sys import tarfile import zipfile import requests +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + RUN_COV_LOG = logging.getLogger("funfuzz") @@ -41,7 +46,8 @@ def get_coverage_build(dirpath, args): js_cov_bin_name = "js" js_cov_bin = extract_folder / "dist" / "bin" / js_cov_bin_name - os.chmod(str(js_cov_bin), os.stat(str(js_cov_bin)).st_mode | 0o111) # Ensure the js binary is executable + + Path.chmod(js_cov_bin, Path.stat(js_cov_bin).st_mode | 0o111) # Ensure the js binary is executable assert js_cov_bin.is_file() # Check that the binary is non-debug. diff --git a/src/funfuzz/js/__init__.py b/src/funfuzz/js/__init__.py index f4bf3a0fa..91b691b62 100644 --- a/src/funfuzz/js/__init__.py +++ b/src/funfuzz/js/__init__.py @@ -13,5 +13,6 @@ from . import compile_shell from . import inspect_shell from . import js_interesting +from . import link_fuzzer from . import loop from . import shell_flags diff --git a/src/funfuzz/js/build_options.py b/src/funfuzz/js/build_options.py index 803175d7a..36205a5fa 100644 --- a/src/funfuzz/js/build_options.py +++ b/src/funfuzz/js/build_options.py @@ -7,12 +7,12 @@ """Allows specification of build configuration parameters. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import argparse from builtins import object # pylint: disable=redefined-builtin import hashlib -import os +import io import platform import random import sys @@ -20,9 +20,13 @@ from past.builtins import range from ..util import hg_helpers -from ..util import subprocesses as sps -DEFAULT_TREES_LOCATION = sps.normExpUserPath(os.path.join("~", "trees")) +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + +DEFAULT_TREES_LOCATION = Path.home() / "trees" def chance(p): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -67,11 +71,13 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): # pylint action="store_true", default=False, help='Chooses sensible random build options. Defaults to "%(default)s".') - parser.add_argument("-R", "--repoDir", - dest="repoDir", + parser.add_argument("-R", "--repodir", + dest="repo_dir", + type=Path, help="Sets the source repository.") parser.add_argument("-P", "--patch", - dest="patchFile", + dest="patch_file", + type=Path, help="Define the path to a single JS patch. Ensure mq is installed.") # Basic spidermonkey options @@ -156,11 +162,17 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): # pylint return parser, randomizer -def parseShellOptions(inputArgs): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Return a "build_options" object, which is intended to be immutable.""" +def parse_shell_opts(args): # pylint: disable=too-many-branches + """Parses shell options into a build_options object. + + Args: + args (object): Arguments to be parsed + + Returns: + build_options: An immutable build_options object + """ parser, randomizer = addParserOptions() - build_options = parser.parse_args(inputArgs.split()) + build_options = parser.parse_args(args.split()) if platform.system() == "Darwin": build_options.buildWithClang = True # Clang seems to be the only supported compiler @@ -171,31 +183,31 @@ def parseShellOptions(inputArgs): # pylint: disable=invalid-name,missing-param- if build_options.enableRandom: build_options = generateRandomConfigurations(parser, randomizer) else: - build_options.build_options_str = inputArgs + build_options.build_options_str = args valid = areArgsValid(build_options) if not valid[0]: print("WARNING: This set of build options is not tested well because: %s" % valid[1]) # Ensures releng machines do not enter the if block and assumes mozilla-central always exists - if os.path.isdir(DEFAULT_TREES_LOCATION): + if DEFAULT_TREES_LOCATION.is_dir(): # pylint: disable=no-member # Repositories do not get randomized if a repository is specified. - if build_options.repoDir is None: + if build_options.repo_dir: + build_options.repo_dir = build_options.repo_dir.expanduser() + else: # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead - if build_options.enableRandom and not build_options.patchFile: - build_options.repoDir = getRandomValidRepo(DEFAULT_TREES_LOCATION) + if build_options.enableRandom and not build_options.patch_file: + build_options.repo_dir = get_random_valid_repo(DEFAULT_TREES_LOCATION) else: - build_options.repoDir = os.path.realpath(sps.normExpUserPath( - os.path.join(DEFAULT_TREES_LOCATION, "mozilla-central"))) + build_options.repo_dir = DEFAULT_TREES_LOCATION / "mozilla-central" - if not os.path.isdir(build_options.repoDir): - sys.exit("repoDir is not specified, and a default repository location cannot be confirmed. Exiting...") + if not build_options.repo_dir.is_dir(): + sys.exit("repo_dir is not specified, and a default repository location cannot be confirmed. Exiting...") - assert hg_helpers.isRepoValid(build_options.repoDir) + assert (build_options.repo_dir / ".hg" / "hgrc").is_file() - if build_options.patchFile: - hg_helpers.ensureMqEnabled() - build_options.patchFile = sps.normExpUserPath(build_options.patchFile) - assert os.path.isfile(build_options.patchFile) + if build_options.patch_file: + hg_helpers.ensure_mq_enabled() + assert build_options.patch_file.resolve().is_file() else: sys.exit("DEFAULT_TREES_LOCATION not found at: %s. Exiting..." % DEFAULT_TREES_LOCATION) @@ -230,10 +242,10 @@ def computeShellType(build_options): # pylint: disable=invalid-name,missing-par if build_options.enableSimulatorArm32 or build_options.enableSimulatorArm64: fileName.append("armSim") fileName.append("windows" if platform.system() == "Windows" else platform.system().lower()) - if build_options.patchFile: + if build_options.patch_file: # We take the name before the first dot, so Windows (hopefully) does not get confused. - fileName.append(os.path.basename(build_options.patchFile).split(".")[0]) - with open(os.path.abspath(build_options.patchFile), "r") as f: + fileName.append(build_options.patch_file.name) + with io.open(str(build_options.patch_file.resolve()), "r", encoding="utf-8", errors="replace") as f: readResult = f.read() # pylint: disable=invalid-name # Append the patch hash, but this is not equivalent to Mercurial's hash of the patch. fileName.append(hashlib.sha512(readResult).hexdigest()[:12]) @@ -332,20 +344,28 @@ def generateRandomConfigurations(parser, randomizer): # pylint: disable=invalid return build_options -def getRandomValidRepo(treeLocation): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - validRepos = [] # pylint: disable=invalid-name - for repo in ["mozilla-central", "mozilla-beta"]: - if os.path.isfile(sps.normExpUserPath(os.path.join( - treeLocation, repo, ".hg", "hgrc"))): - validRepos.append(repo) +def get_random_valid_repo(tree): + """Given a path to Mozilla Mercurial repositories, return a randomly chosen valid one. + + Args: + tree (Path): Intended location of Mozilla Mercurial repositories + + Returns: + Path: Location of a valid Mozilla repository + """ + assert isinstance(tree, Path) + tree = tree.resolve() + + valid_repos = [] + for branch in ["mozilla-central", "mozilla-beta"]: + if (tree / branch / ".hg" / "hgrc").is_file(): + valid_repos.append(branch) # After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen - if "mozilla-beta" in validRepos and chance(0.5): - validRepos.remove("mozilla-beta") + if "mozilla-beta" in valid_repos and chance(0.5): + valid_repos.remove("mozilla-beta") - return os.path.realpath(sps.normExpUserPath( - os.path.join(treeLocation, random.choice(validRepos)))) + return tree / random.choice(valid_repos) def main(): # pylint: disable=missing-docstring @@ -363,7 +383,7 @@ def main(): # pylint: disable=missing-docstring print() print("Running this file directly doesn't do anything, but here's our subparser help:") print() - parseShellOptions("--help") + parse_shell_opts("--help") if __name__ == "__main__": diff --git a/src/funfuzz/js/compare_jit.py b/src/funfuzz/js/compare_jit.py index 54ba0ffd5..a08b7c515 100644 --- a/src/funfuzz/js/compare_jit.py +++ b/src/funfuzz/js/compare_jit.py @@ -7,8 +7,9 @@ """Test comparing the output of SpiderMonkey using various flags (usually JIT-related). """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io from optparse import OptionParser # pylint: disable=deprecated-module import os import sys @@ -16,22 +17,38 @@ # These pylint errors exist because FuzzManager is not Python 3-compatible yet from FTB.ProgramConfiguration import ProgramConfiguration # pylint: disable=import-error import FTB.Signatures.CrashInfo as CrashInfo # pylint: disable=import-error,no-name-in-module -from past.builtins import range from shellescape import quote from . import js_interesting from . import shell_flags from ..util import create_collector from ..util import lithium_helpers -from ..util import subprocesses as sps + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + import tempfile gOptions = "" # pylint: disable=invalid-name lengthLimit = 1000000 # pylint: disable=invalid-name -def ignoreSomeOfStderr(e): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc +def ignore_some_stderr(err_inp): + """Ignores parts of a list depending on whether they are needed. + + Args: + err_inp (list): Stderr + + Returns: + list: Stderr with potentially some lines removed + """ lines = [] - for line in e: + for line in err_inp: if line.endswith("malloc: enabling scribbling to detect mods to free blocks"): # MallocScribble prints a line that includes the process's pid. # We don't want to include that pid in the comparison! @@ -44,14 +61,19 @@ def ignoreSomeOfStderr(e): # pylint: disable=invalid-name,missing-docstring,mis return lines -# For use by loop -# Returns True if any kind of bug is found -def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, targetTime, options): - # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - # pylint: disable=too-many-arguments,too-many-locals +def compare_jit(jsEngine, # pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-many-arguments + flags, infilename, logPrefix, repo, build_options_str, targetTime, options): + """For use in loop.py + Returns: + bool: True if any kind of bug is found, otherwise False + """ + # pylint: disable=too-many-locals + # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. + logPrefix = Path(logPrefix) + initialdir_name = (logPrefix.parent / (logPrefix.stem + "-initial")) # pylint: disable=invalid-name - cl = compareLevel(jsEngine, flags, infilename, logPrefix + "-initial", options, False, True) + cl = compareLevel(jsEngine, flags, infilename, initialdir_name, options, False, True) lev = cl[0] if lev != js_interesting.JS_FINE: @@ -61,7 +83,8 @@ def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, itest, logPrefix, jsEngine, [], infilename, repo, build_options_str, targetTime, lev) if lithResult == lithium_helpers.LITH_FINISHED: print("Retesting %s after running Lithium:" % infilename) - retest_cl = compareLevel(jsEngine, flags, infilename, logPrefix + "-final", options, True, False) + finaldir_name = (logPrefix.parent / (logPrefix.stem + "-final")) + retest_cl = compareLevel(jsEngine, flags, infilename, finaldir_name, options, True, False) if retest_cl[0] != js_interesting.JS_FINE: cl = retest_cl quality = 0 @@ -74,7 +97,7 @@ def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, metadata = {} if autoBisectLog: metadata = {"autoBisectLog": "".join(autoBisectLog)} - options.collector.submit(cl[1], infilename, quality, metaData=metadata) + options.collector.submit(cl[1], str(infilename), quality, metaData=metadata) return True return False @@ -88,6 +111,8 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi # we also use it directly for knownPath, timeout, and collector # Return: (lev, crashInfo) or (js_interesting.JS_FINE, None) + assert isinstance(infilename, Path) + combos = shell_flags.basic_flag_sets(jsEngine) if quickMode: @@ -97,38 +122,39 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi if flags: combos.insert(0, flags) - commands = [[jsEngine] + combo + [infilename] for combo in combos] + commands = [[jsEngine] + combo + [str(infilename)] for combo in combos] - for i in range(0, len(commands)): - prefix = logPrefix + "-r" + str(i) + for i, command in enumerate(commands): + prefix = (logPrefix.parent / ("%s-r%s" % (logPrefix.stem, str(i)))) command = commands[i] r = js_interesting.ShellResult(options, command, prefix, True) # pylint: disable=invalid-name oom = js_interesting.oomed(r.err) - r.err = ignoreSomeOfStderr(r.err) + r.err = ignore_some_stderr(r.err) if (r.return_code == 1 or r.return_code == 2) and (anyLineContains(r.out, "[[script] scriptArgs*]") or ( anyLineContains(r.err, "[scriptfile] [scriptarg...]"))): print("Got usage error from:") - print(" %s" % " ".join(quote(x) for x in command)) + print(" %s" % " ".join(quote(str(x)) for x in command)) assert i js_interesting.deleteLogs(prefix) elif r.lev > js_interesting.JS_OVERALL_MISMATCH: # would be more efficient to run lithium on one or the other, but meh - print("%s | %s" % (infilename, + print("%s | %s" % (str(infilename), js_interesting.summaryString(r.issues + ["compare_jit found a more serious bug"], r.lev, r.runinfo.elapsedtime))) - with open(logPrefix + "-summary.txt", "w") as f: - f.write("\n".join(r.issues + [" ".join(quote(x) for x in command), + summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") + with io.open(str(summary_log), "w", encoding="utf-8", errors="replace") as f: + f.write("\n".join(r.issues + [" ".join(quote(str(x)) for x in command), "compare_jit found a more serious bug"]) + "\n") - print(" %s" % " ".join(quote(x) for x in command)) + print(" %s" % " ".join(quote(str(x)) for x in command)) return (r.lev, r.crashInfo) elif r.lev != js_interesting.JS_FINE or r.return_code != 0: - print("%s | %s" % (infilename, js_interesting.summaryString( + print("%s | %s" % (str(infilename), js_interesting.summaryString( r.issues + ["compare_jit is not comparing output, because the shell exited strangely"], r.lev, r.runinfo.elapsedtime))) - print(" %s" % " ".join(quote(x) for x in command)) + print(" %s" % " ".join(quote(str(x)) for x in command)) js_interesting.deleteLogs(prefix) if not i: return (js_interesting.JS_FINE, None) @@ -136,7 +162,7 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi # If the shell or python hit a memory limit, we consider the rest of the computation # "tainted" for the purpose of correctness comparison. message = "compare_jit is not comparing output: OOM" - print("%s | %s" % (infilename, js_interesting.summaryString( + print("%s | %s" % (str(infilename), js_interesting.summaryString( r.issues + [message], r.lev, r.runinfo.elapsedtime))) js_interesting.deleteLogs(prefix) if not i: @@ -163,18 +189,19 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr if mismatchErr or mismatchOut: # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. # pylint: disable=invalid-name - rerunCommand = " ".join(quote(x) for x in ["python -m funfuzz.js.compare_jit", - "--flags=" + " ".join(flags), - "--timeout=" + str(options.timeout), - options.knownPath, - jsEngine, - os.path.basename(infilename)]) + rerunCommand = " ".join(quote(str(x)) for x in ["python -m funfuzz.js.compare_jit", + "--flags=" + " ".join(flags), + "--timeout=" + str(options.timeout), + options.knownPath, + jsEngine, + str(infilename.name)]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) - summary = (" " + " ".join(quote(x) for x in commands[0]) + "\n " + - " ".join(quote(x) for x in command) + "\n\n" + summary) - with open(logPrefix + "-summary.txt", "w") as f: + summary = (" " + " ".join(quote(str(x)) for x in commands[0]) + "\n " + + " ".join(quote(str(x)) for x in command) + "\n\n" + summary) + summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") + with io.open(str(summary_log), "w", encoding="utf-8", errors="replace") as f: f.write(rerunCommand + "\n\n" + summary) - print("%s | %s" % (infilename, js_interesting.summaryString( + print("%s | %s" % (str(infilename), js_interesting.summaryString( issues, js_interesting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime))) if quickMode: print(rerunCommand) @@ -196,25 +223,32 @@ def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstr # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc -def summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix): +def summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix1): issues = [] summary = "" if mismatchErr: issues.append("[Non-crash bug] Mismatch on stderr") summary += "[Non-crash bug] Mismatch on stderr\n" - summary += diffFiles(prefix0 + "-err.txt", prefix + "-err.txt") + err0_log = (prefix0.parent / (prefix0.stem + "-err")).with_suffix(".txt") + err1_log = (prefix1.parent / (prefix1.stem + "-err")).with_suffix(".txt") + summary += diffFiles(err0_log, err1_log) if mismatchOut: issues.append("[Non-crash bug] Mismatch on stdout") summary += "[Non-crash bug] Mismatch on stdout\n" - summary += diffFiles(prefix0 + "-out.txt", prefix + "-out.txt") + out0_log = (prefix0.parent / (prefix0.stem + "-out")).with_suffix(".txt") + out1_log = (prefix1.parent / (prefix1.stem + "-out")).with_suffix(".txt") + summary += diffFiles(out0_log, out1_log) return (summary, issues) def diffFiles(f1, f2): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Return a command to diff two files, along with the diff output (if it's short).""" - diffcmd = ["diff", "-u", f1, f2] + diffcmd = ["diff", "-u", str(f1), str(f2)] s = " ".join(diffcmd) + "\n\n" # pylint: disable=invalid-name - diff = sps.captureStdout(diffcmd, ignoreExitCode=True)[0] + diff = subprocess.run(diffcmd, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=99).stdout.decode("utf-8", errors="replace") if len(diff) < 10000: s += diff + "\n\n" # pylint: disable=invalid-name else: @@ -249,17 +283,17 @@ def parseOptions(args): # pylint: disable=invalid-name options, args = parser.parse_args(args) if len(args) != 3: raise Exception("Wrong number of positional arguments. Need 3 (knownPath, jsengine, infilename).") - options.knownPath = args[0] - options.jsengine = args[1] - options.infilename = args[2] + options.knownPath = Path(args[0]).expanduser().resolve() + options.jsengine = Path(args[1]).expanduser().resolve() + options.infilename = Path(args[2]).expanduser().resolve() options.flags = options.flagsSpaceSep.split(" ") if options.flagsSpaceSep else [] - if not os.path.exists(options.jsengine): - raise Exception("js shell does not exist: " + options.jsengine) + if not options.jsengine.is_file(): + raise OSError("js shell does not exist: " + options.jsengine) # For js_interesting: options.valgrind = False options.shellIsDeterministic = True # We shouldn't be in compare_jit with a non-deterministic build - options.collector = create_collector.createCollector("jsfunfuzz") + options.collector = create_collector.make_collector() return options @@ -271,18 +305,18 @@ def init(args): # FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme -def interesting(_args, tempPrefix): # pylint: disable=invalid-name +def interesting(_args, cwd_prefix): + cwd_prefix = Path(cwd_prefix) # Lithium uses this function and cwd_prefix from Lithium is not a Path actualLevel = compareLevel( # pylint: disable=invalid-name - gOptions.jsengine, gOptions.flags, gOptions.infilename, tempPrefix, gOptions, False, False)[0] + gOptions.jsengine, gOptions.flags, gOptions.infilename, cwd_prefix, gOptions, False, False)[0] return actualLevel >= gOptions.minimumInterestingLevel def main(): - import tempfile options = parseOptions(sys.argv[1:]) print(compareLevel( options.jsengine, options.flags, options.infilename, # pylint: disable=no-member - tempfile.mkdtemp("compare_jitmain"), options, True, False)[0]) + Path(tempfile.mkdtemp("compare_jitmain")), options, True, False)[0]) if __name__ == "__main__": diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py index 0f46ef68d..c5b6df9cd 100644 --- a/src/funfuzz/js/compile_shell.py +++ b/src/funfuzz/js/compile_shell.py @@ -11,7 +11,6 @@ from builtins import object # pylint: disable=redefined-builtin import copy -import ctypes import io import multiprocessing from optparse import OptionParser # pylint: disable=deprecated-module @@ -20,7 +19,6 @@ import shutil import sys import tarfile -import traceback from pkg_resources import parse_version from shellescape import quote @@ -30,12 +28,16 @@ from . import inspect_shell from ..util import hg_helpers from ..util import s3cache +from ..util import sm_compile_helpers from ..util import subprocesses as sps from ..util.lock_dir import LockDir -if sys.version_info.major == 2 and os.name == "posix": - import subprocess32 as subprocess # pylint: disable=import-error +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path else: + from pathlib import Path # pylint: disable=import-error import subprocess S3_SHELL_CACHE_DIRNAME = "shell-cache" # Used by autobisectjs @@ -67,38 +69,56 @@ class CompiledShellError(Exception): pass -class CompiledShell(object): # pylint: disable=missing-docstring,too-many-instance-attributes,too-many-public-methods - def __init__(self, buildOpts, hgHash): - self.shellNameWithoutExt = build_options.computeShellName(buildOpts, hgHash) # pylint: disable=invalid-name - self.hgHash = hgHash # pylint: disable=invalid-name - self.build_opts = buildOpts +class CompiledShell(object): # pylint: disable=too-many-instance-attributes,too-many-public-methods + """A CompiledShell object represents an actual compiled shell binary. + + Args: + build_opts (object): Object containing the build options defined in build_options.py + hg_hash (str): Changeset hash + """ + def __init__(self, build_opts, hg_hash): + self.shell_name_without_ext = build_options.computeShellName(build_opts, hg_hash) + self.hg_hash = hg_hash + self.build_opts = build_opts - self.jsObjdir = "" # pylint: disable=invalid-name + self.js_objdir = "" - self.cfg = "" + self.cfg = [] self.destDir = "" # pylint: disable=invalid-name - self.addedEnv = "" # pylint: disable=invalid-name - self.fullEnv = "" # pylint: disable=invalid-name - self.jsCfgFile = "" # pylint: disable=invalid-name + self.added_env = "" + self.full_env = "" + self.js_cfg_file = "" - self.jsMajorVersion = "" # pylint: disable=invalid-name - self.jsVersion = "" # pylint: disable=invalid-name + self.js_version = "" # pylint: disable=invalid-name @classmethod - def main(cls, args=None): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc - # logging.basicConfig(format="%(message)s", level=logging.INFO) + def main(cls, args=None): + """Main function of CompiledShell class. + Args: + args (object): Additional parameters + + Returns: + int: 0, to denote a successful compile and 1, to denote a failed compile + """ + # logging.basicConfig(format="%(message)s", level=logging.INFO) try: return cls.run(args) - except CompiledShellError as ex: print(repr(ex)) # log.error(ex) return 1 @staticmethod - def run(argv=None): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc - """Build a shell and place it in the autobisectjs cache.""" + def run(argv=None): + """Build a shell and place it in the autobisectjs cache. + + Args: + argv (object): Additional parameters + + Returns: + int: 0, to denote a successful compile + """ usage = "Usage: %prog [options]" parser = OptionParser(usage) parser.disable_interspersed_args() @@ -118,176 +138,216 @@ def run(argv=None): # pylint: disable=missing-param-doc,missing-return-doc,miss help="Specify revision to build") options = parser.parse_args(argv)[0] - options.build_opts = build_options.parseShellOptions(options.build_opts) + options.build_opts = build_options.parse_shell_opts(options.build_opts) - with LockDir(getLockDirPath(options.build_opts.repoDir)): + with LockDir(sm_compile_helpers.get_lock_dir_path(Path.home(), options.build_opts.repo_dir)): if options.revision: shell = CompiledShell(options.build_opts, options.revision) else: - local_orig_hg_hash = hg_helpers.getRepoHashAndId(options.build_opts.repoDir)[0] + local_orig_hg_hash = hg_helpers.get_repo_hash_and_id(options.build_opts.repo_dir)[0] shell = CompiledShell(options.build_opts, local_orig_hg_hash) obtainShell(shell, updateToRev=options.revision) - print(shell.getShellCacheFullPath()) + print(shell.get_shell_cache_js_bin_path()) return 0 - def getCfgCmdExclEnv(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.cfg - - def setCfgCmdExclEnv(self, cfg): # pylint: disable=invalid-name,missing-docstring - self.cfg = cfg - - def setEnvAdded(self, addedEnv): # pylint: disable=invalid-name,missing-docstring - self.addedEnv = addedEnv - - def getEnvAdded(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.addedEnv - - def setEnvFull(self, fullEnv): # pylint: disable=invalid-name,missing-docstring - self.fullEnv = fullEnv - - def getEnvFull(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.fullEnv - - def getHgHash(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.hgHash - - def getJsCfgPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - self.jsCfgFile = sps.normExpUserPath(os.path.join(self.getRepoDirJsSrc(), "configure")) - assert os.path.isfile(self.jsCfgFile) - return self.jsCfgFile + def get_cfg_cmd_excl_env(self): + """Retrieve the configure command excluding the enviroment variables. - def getJsObjdir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.jsObjdir + Returns: + list: Configure command + """ + return self.cfg - def setJsObjdir(self, oDir): # pylint: disable=invalid-name,missing-docstring - self.jsObjdir = oDir + def set_cfg_cmd_excl_env(self, cfg): + """Sets the configure command excluding the enviroment variables. - def getRepoDir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.build_opts.repoDir + Args: + cfg (list): Configure command + """ + self.cfg = cfg - def getRepoDirJsSrc(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(self.getRepoDir(), "js", "src")) + def set_env_added(self, added_env): + """Set environment variables that were added. - def getRepoName(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return hg_helpers.getRepoNameFromHgrc(self.build_opts.repoDir) + Args: + added_env (list): Added environment variables + """ + self.added_env = added_env - def getS3TarballWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.getShellNameWithoutExt() + ".tar.bz2" + def get_env_added(self): + """Retrieve environment variables that were added. - def getS3TarballWithExtFullPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getS3TarballWithExt())) + Returns: + list: Added environment variables + """ + return self.added_env - def getShellCacheDir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getShellNameWithoutExt())) + def set_env_full(self, full_env): + """Set the full environment including the newly added variables. - def getShellCacheFullPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath(os.path.join(self.getShellCacheDir(), self.getShellNameWithExt())) + Args: + full_env (list): Full environment + """ + self.full_env = full_env - def getShellCompiledPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return sps.normExpUserPath( - os.path.join(self.getJsObjdir(), "dist", "bin", "js" + (".exe" if platform.system() == "Windows" else ""))) + def get_env_full(self): + """Retrieve the full environment including the newly added variables. - def getShellCompiledRunLibsPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - libs_list = [ - sps.normExpUserPath(os.path.join(self.getJsObjdir(), "dist", "bin", runLib)) - for runLib in inspect_shell.ALL_RUN_LIBS + Returns: + list: Full environment + """ + return self.full_env + + def get_hg_hash(self): + """Retrieve the hash of the current changeset of the repository. + + Returns: + str: Changeset hash + """ + return self.hg_hash + + def get_js_cfg_path(self): + """Retrieve the configure file in a js/src directory. + + Returns: + Path: Full path to the configure file + """ + self.js_cfg_file = self.get_repo_dir() / "js" / "src" / "configure" + return self.js_cfg_file + + def get_js_objdir(self): + """Retrieve the objdir of the js shell to be compiled. + + Returns: + Path: Full path to the js shell objdir + """ + return self.js_objdir + + def set_js_objdir(self, objdir): + """Set the objdir of the js shell to be compiled. + + Args: + objdir (Path): Full path to the objdir of the js shell to be compiled + """ + self.js_objdir = objdir + + def get_repo_dir(self): + """Retrieve the directory of a Mercurial repository. + + Returns: + Path: Full path to the repository + """ + return self.build_opts.repo_dir + + def get_repo_name(self): + """Retrieve the name of a Mercurial repository. + + Returns: + str: Name of the repository + """ + return hg_helpers.hgrc_repo_name(self.build_opts.repo_dir) + + def get_s3_tar_name_with_ext(self): + """Retrieve the name of the compressed shell tarball to be obtained from/sent to S3. + + Returns: + str: Name of the tarball + """ + return self.get_shell_name_without_ext() + ".tar.bz2" + + def get_s3_tar_with_ext_full_path(self): + """Retrieve the path to the tarball downloaded from S3. + + Returns: + Path: Full path to the tarball in the local shell cache directory + """ + return sm_compile_helpers.ensure_cache_dir(Path.home()) / self.get_s3_tar_name_with_ext() + + def get_shell_cache_dir(self): + """Retrieve the shell cache directory of the intended js binary. + + Returns: + Path: Full path to the shell cache directory of the intended js binary + """ + return sm_compile_helpers.ensure_cache_dir(Path.home()) / self.get_shell_name_without_ext() + + def get_shell_cache_js_bin_path(self): + """Retrieve the full path to the js binary located in the shell cache. + + Returns: + Path: Full path to the js binary in the shell cache + """ + return (sm_compile_helpers.ensure_cache_dir(Path.home()) / + self.get_shell_name_without_ext() / self.get_shell_name_with_ext()) + + def get_shell_compiled_path(self): + """Retrieve the full path to the original location of js binary compiled in the shell cache. + + Returns: + Path: Full path to the original location of js binary compiled in the shell cache + """ + full_path = self.get_js_objdir() / "dist" / "bin" / "js" + return full_path.with_suffix(".exe") if platform.system() == "Windows" else full_path + + def get_shell_compiled_runlibs_path(self): + """Retrieve the full path to the original location of the libraries of js binary compiled in the shell cache. + + Returns: + Path: Full path to the original location of the libraries of js binary compiled in the shell cache + """ + return [ + self.get_js_objdir() / "dist" / "bin" / runlib for runlib in inspect_shell.ALL_RUN_LIBS ] - return libs_list - - def getShellNameWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.shellNameWithoutExt + (".exe" if platform.system() == "Windows" else "") - def getShellNameWithoutExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.shellNameWithoutExt + def get_shell_name_with_ext(self): + """Retrieve the name of the compiled js shell with the file extension. - # Version numbers - def getMajorVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return self.jsMajorVersion + Returns: + str: Name of the compiled js shell with the file extension + """ + return self.shell_name_without_ext + (".exe" if platform.system() == "Windows" else "") - def setMajorVersion(self, jsMajorVersion): # pylint: disable=invalid-name,missing-docstring - self.jsMajorVersion = jsMajorVersion + def get_shell_name_without_ext(self): + """Retrieve the name of the compiled js shell without the file extension. - def getVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return self.jsVersion + Returns: + str: Name of the compiled js shell without the file extension + """ + return self.shell_name_without_ext - def setVersion(self, jsVersion): # pylint: disable=invalid-name,missing-docstring - self.jsVersion = jsVersion + def get_version(self): + """Retrieve the version number of the js shell as extracted from js.pc + Returns: + str: Version number of the js shell + """ + return self.js_version -def ensureCacheDir(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc - """Return a cache directory for compiled shells to live in, and create one if needed.""" - cache_dir = os.path.join(sps.normExpUserPath("~"), "shell-cache") - ensureDir(cache_dir) + def set_version(self, js_version): + """Set the version number of the js shell as extracted from js.pc - # Expand long Windows paths (overcome legacy MS-DOS 8.3 stuff) - # This has to occur after the shell-cache directory is created - if platform.system() == "Windows": # adapted from http://stackoverflow.com/a/3931799 - if sys.version_info.major == 2: - utext = unicode # noqa pylint: disable=undefined-variable,unicode-builtin - else: - utext = str - win_temp_dir = utext(cache_dir) - get_long_path_name = ctypes.windll.kernel32.GetLongPathNameW - unicode_buf = ctypes.create_unicode_buffer(get_long_path_name(win_temp_dir, 0, 0)) - get_long_path_name(win_temp_dir, unicode_buf, len(unicode_buf)) - cache_dir = sps.normExpUserPath(str(unicode_buf.value)) # convert back to a str - - return cache_dir - - -def ensureDir(directory): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Create a directory, if it does not already exist.""" - if not os.path.exists(directory): - os.mkdir(directory) - assert os.path.isdir(directory) - - -def autoconfRun(cwDir): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Run autoconf binaries corresponding to the platform.""" - if platform.system() == "Darwin": - autoconf213_mac_bin = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" if which("brew") else "autoconf213" - # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 - if not os.path.isfile(sps.normExpUserPath(autoconf213_mac_bin)): - autoconf213_mac_bin = "autoconf213" - subprocess.check_call([autoconf213_mac_bin], cwd=cwDir) - elif platform.system() == "Linux": - if which("autoconf2.13"): - subprocess.run(["autoconf2.13"], check=True, cwd=cwDir) - elif which("autoconf-2.13"): - subprocess.run(["autoconf-2.13"], check=True, cwd=cwDir) - elif which("autoconf213"): - subprocess.run(["autoconf213"], check=True, cwd=cwDir) - elif platform.system() == "Windows": - # Windows needs to call sh to be able to find autoconf. - subprocess.check_call(["sh", "autoconf-2.13"], cwd=cwDir) + Args: + js_version (str): Version number of the js shell + """ + self.js_version = js_version def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc """Configures, compiles and copies a js shell according to required parameters.""" print("Compiling...") # Print *with* a trailing newline to avoid breaking other stuff - os.mkdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js"))) - shell.setJsObjdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js"))) + js_objdir_path = shell.get_shell_cache_dir() / "objdir-js" + js_objdir_path.mkdir() + shell.set_js_objdir(js_objdir_path) - autoconfRun(shell.getRepoDirJsSrc()) + sm_compile_helpers.autoconf_run(shell.get_repo_dir() / "js" / "src") configure_try_count = 0 while True: try: cfgBin(shell) break - except Exception as ex: # pylint: disable=broad-except + except subprocess.CalledProcessError as ex: configure_try_count += 1 if configure_try_count > 3: print("Configuration of the js binary failed 3 times.") @@ -298,13 +358,12 @@ def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missi "Windows conftest.exe configuration permission" in repr(ex)): print("Trying once more...") continue - compileJs(shell) + sm_compile(shell) inspect_shell.verifyBinary(shell) - compile_log = sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), - shell.getShellNameWithoutExt() + ".fuzzmanagerconf")) - if not os.path.isfile(compile_log): - envDump(shell, compile_log) + compile_log = shell.get_shell_cache_dir() / (shell.get_shell_name_without_ext() + ".fuzzmanagerconf") + if not compile_log.is_file(): + sm_compile_helpers.envDump(shell, compile_log) def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-complex,too-many-branches @@ -314,38 +373,34 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_env = copy.deepcopy(os.environ) orig_cfg_env = copy.deepcopy(os.environ) cfg_env["AR"] = "ar" - if shell.build_opts.enable32 and os.name == "posix": + if shell.build_opts.enable32 and platform.system() == "Linux": # 32-bit shell on 32/64-bit x86 Linux - if platform.system() == "Linux": - cfg_env["PKG_CONFIG_LIBDIR"] = "/usr/lib/pkgconfig" - if shell.build_opts.buildWithClang: - cfg_env["CC"] = cfg_env["HOST_CC"] = str( - "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) - cfg_env["CXX"] = cfg_env["HOST_CXX"] = str( - "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) - else: - # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. - cfg_env["CC"] = "gcc -m32 %s" % SSE2_FLAGS - cfg_env["CXX"] = "g++ -m32 %s" % SSE2_FLAGS - if shell.build_opts.buildWithAsan: - cfg_env["CC"] += " " + CLANG_ASAN_PARAMS - cfg_env["CXX"] += " " + CLANG_ASAN_PARAMS - cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) - cfg_cmds.append("--target=i686-pc-linux") - if shell.build_opts.buildWithAsan: - cfg_cmds.append("--enable-address-sanitizer") - if shell.build_opts.enableSimulatorArm32: - # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 - # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated - # Newer configure.in changes mean that things blow up if unknown/removed configure - # options are entered, so specify it only if it's requested. - if shell.build_opts.enableArmSimulatorObsolete: - cfg_cmds.append("--enable-arm-simulator") - cfg_cmds.append("--enable-simulator=arm") + cfg_env["PKG_CONFIG_LIBDIR"] = "/usr/lib/pkgconfig" + if shell.build_opts.buildWithClang: + cfg_env["CC"] = cfg_env["HOST_CC"] = str( + "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) + cfg_env["CXX"] = cfg_env["HOST_CXX"] = str( + "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) else: - cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. + cfg_env["CC"] = "gcc -m32 %s" % SSE2_FLAGS + cfg_env["CXX"] = "g++ -m32 %s" % SSE2_FLAGS + if shell.build_opts.buildWithAsan: + cfg_env["CC"] += " " + CLANG_ASAN_PARAMS + cfg_env["CXX"] += " " + CLANG_ASAN_PARAMS + cfg_cmds.append("sh") + cfg_cmds.append(str(shell.get_js_cfg_path())) + cfg_cmds.append("--target=i686-pc-linux") + if shell.build_opts.buildWithAsan: + cfg_cmds.append("--enable-address-sanitizer") + if shell.build_opts.enableSimulatorArm32: + # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 + # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated + # Newer configure.in changes mean that things blow up if unknown/removed configure + # options are entered, so specify it only if it's requested. + if shell.build_opts.enableArmSimulatorObsolete: + cfg_cmds.append("--enable-arm-simulator") + cfg_cmds.append("--enable-simulator=arm") # 64-bit shell on Mac OS X 10.11 El Capitan and greater elif parse_version(platform.mac_ver()[0]) >= parse_version("10.11") and not shell.build_opts.enable32: cfg_env["CC"] = "clang " + CLANG_PARAMS @@ -356,7 +411,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ if which("brew"): cfg_env["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append(str(shell.get_js_cfg_path())) cfg_cmds.append("--target=x86_64-apple-darwin15.6.0") # El Capitan 10.11.6 cfg_cmds.append("--disable-xcode-checks") if shell.build_opts.buildWithAsan: @@ -380,7 +435,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_env["HOST_LDFLAGS"] = " " cfg_env["LIB"] += r"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows" cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append(str(shell.get_js_cfg_path())) if shell.build_opts.enable32: if shell.build_opts.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 @@ -406,7 +461,7 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_env["CC"] += " " + CLANG_ASAN_PARAMS cfg_env["CXX"] += " " + CLANG_ASAN_PARAMS cfg_cmds.append("sh") - cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append(str(shell.get_js_cfg_path())) if shell.build_opts.buildWithAsan: cfg_cmds.append("--enable-address-sanitizer") @@ -452,12 +507,11 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ cfg_cmds.append("--enable-debug-symbols") # gets debug symbols on opt shells cfg_cmds.append("--disable-tests") - if os.name == "nt": + if platform.system() == "Windows": # FIXME: Replace this with shellescape's quote # pylint: disable=fixme counter = 0 for entry in cfg_cmds: if os.sep in entry: - assert platform.system() == "Windows" # MozillaBuild on Windows sometimes confuses "/" and "\". cfg_cmds[counter] = cfg_cmds[counter].replace(os.sep, "//") counter = counter + 1 @@ -468,11 +522,10 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ '"' if " " in cfg_env[str(env_var)] else env_var + "=" + cfg_env[str(env_var)]) env_vars.append(str_to_be_appended) - sps.vdump("Command to be run is: " + " ".join(quote(x) for x in env_vars) + " " + - " ".join(quote(x) for x in cfg_cmds)) + sps.vdump("Command to be run is: " + " ".join(quote(str(x)) for x in env_vars) + " " + + " ".join(quote(str(x)) for x in cfg_cmds)) - js_objdir = shell.getJsObjdir() - assert os.path.isdir(js_objdir) + assert shell.get_js_objdir().is_dir() if platform.system() == "Windows": changed_cfg_cmds = [] @@ -483,140 +536,84 @@ def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-typ if "\\" in entry: entry = entry.replace("\\", "/") changed_cfg_cmds.append(entry) - sps.captureStdout(changed_cfg_cmds, ignoreStderr=True, currWorkingDir=js_objdir, env=cfg_env) + subprocess.run(changed_cfg_cmds, + check=True, + cwd=str(shell.get_js_objdir()), + env=cfg_env, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") else: - sps.captureStdout(cfg_cmds, ignoreStderr=True, currWorkingDir=js_objdir, env=cfg_env) - - shell.setEnvAdded(env_vars) - shell.setEnvFull(cfg_env) - shell.setCfgCmdExclEnv(cfg_cmds) + subprocess.run(cfg_cmds, + check=True, + cwd=str(shell.get_js_objdir()), + env=cfg_env, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + + # We could save the stdout here into a file if it throws + + shell.set_env_added(env_vars) + shell.set_env_full(cfg_env) + shell.set_cfg_cmd_excl_env(cfg_cmds) + + +def sm_compile(shell): + """Compile a binary and copy essential compiled files into a desired structure. + + Args: + shell (object): SpiderMonkey shell parameters + + Raises: + OSError: Raises when a compiled shell is absent + + Returns: + Path: Path to the compiled shell + """ + cmd_list = [MAKE_BINARY, "-C", str(shell.get_js_objdir()), "-j" + str(COMPILATION_JOBS), "-s"] + # Note that having a non-zero exit code does not mean that the operation did not succeed, + # for example when compiling a shell. A non-zero exit code can appear even though a shell compiled successfully. + # Thus, we should *not* use check=True here. + out = subprocess.run(cmd_list, + cwd=str(shell.get_js_objdir()), + env=shell.get_env_full(), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") + + if shell.get_shell_compiled_path().is_file(): + shutil.copy2(str(shell.get_shell_compiled_path()), str(shell.get_shell_cache_js_bin_path())) + for run_lib in shell.get_shell_compiled_runlibs_path(): + if run_lib.is_file(): + shutil.copy2(str(run_lib), str(shell.get_shell_cache_dir())) + + shell.set_version(sm_compile_helpers.extract_vers(shell.get_js_objdir())) - -def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc - """Compile and copy a binary.""" - try: - cmd_list = [MAKE_BINARY, "-C", shell.getJsObjdir(), "-j" + str(COMPILATION_JOBS), "-s"] - out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, - currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] - except Exception as ex: # pylint: disable=broad-except - # This exception message is returned from sps.captureStdout via cmd_list. - if (platform.system() == "Linux" or platform.system() == "Darwin") and \ - ("GCC running out of memory" in repr(ex) or "Clang running out of memory" in repr(ex)): - # FIXME: Absolute hack to retry after hitting OOM. # pylint: disable=fixme + if platform.system() == "Linux": + # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) + # files in the objdir or else the stacks from failing testcases will lack symbols. + shutil.rmtree(str(shell.get_shell_cache_dir() / "objdir-js")) + else: + if ((platform.system() == "Linux" or platform.system() == "Darwin") and + ("internal compiler error: Killed (program cc1plus)" in out or # GCC running out of memory + "error: unable to execute command: Killed" in out)): # Clang running out of memory print("Trying once more due to the compiler running out of memory...") - out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, - currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] + out = subprocess.run(cmd_list, + cwd=str(shell.get_js_objdir()), + env=shell.get_env_full(), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace") # A non-zero error can be returned during make, but eventually a shell still gets compiled. - if os.path.exists(shell.getShellCompiledPath()): + if shell.get_shell_compiled_path().is_file(): print("A shell was compiled even though there was a non-zero exit code. Continuing...") else: print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) - raise + with io.open(str(shell.get_shell_cache_dir() / ".busted.log"), "w", + encoding="utf-8", errors="replace") as f: + f.write("The first compilation of %s rev %s failed with the following output:\n" % + (shell.get_repo_name(), shell.get_hg_hash())) + f.write(out.decode("utf-8", errors="replace")) + raise OSError(MAKE_BINARY + " did not result in a js shell.") - if os.path.exists(shell.getShellCompiledPath()): - shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) - for run_lib in shell.getShellCompiledRunLibsPath(): - if os.path.isfile(run_lib): - shutil.copy2(run_lib, shell.getShellCacheDir()) - - version = extractVersions(shell.getJsObjdir()) - shell.setMajorVersion(version.split(".")[0]) - shell.setVersion(version) - - if platform.system() == "Linux": - # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) - # files in the objdir or else the stacks from failing testcases will lack symbols. - shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js"))) - else: - print(out.decode("utf-8", errors="replace")) - raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.") - - -def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Create a .busted file with the exception message and backtrace included.""" - with open(filename, "w") as f: - f.write("Caught exception %r (%s)\n" % (e, e)) - f.write("Backtrace:\n") - f.write(traceback.format_exc() + "\n") - - print("Compilation failed (%s) (details in %s)" % (e, filename)) - - -def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Dump environment to a .fuzzmanagerconf file.""" - # Platform and OS detection for the spec, part of which is in: - # https://wiki.mozilla.org/Security/CrashSignatures - fmconf_platform = "x86" if shell.build_opts.enable32 else "x86-64" - - if platform.system() == "Linux": - fmconf_os = "linux" - elif platform.system() == "Darwin": - fmconf_os = "macosx" - elif platform.system() == "Windows": - fmconf_os = "windows" - - with open(log, "a") as f: - f.write("# Information about shell:\n# \n") - - f.write("# Create another shell in shell-cache like this one:\n") - f.write('# python -u -m %s -b "%s" -r %s\n# \n' % ("funfuzz.js.compile_shell", - shell.build_opts.build_options_str, shell.getHgHash())) - - f.write("# Full environment is:\n") - f.write("# %s\n# \n" % str(shell.getEnvFull())) - - f.write("# Full configuration command with needed environment variables is:\n") - f.write("# %s %s\n# \n" % (" ".join(quote(x) for x in shell.getEnvAdded()), - " ".join(quote(x) for x in shell.getCfgCmdExclEnv()))) - - # .fuzzmanagerconf details - f.write("\n") - f.write("[Main]\n") - f.write("platform = %s\n" % fmconf_platform) - f.write("product = %s\n" % shell.getRepoName()) - f.write("product_version = %s\n" % shell.getHgHash()) - f.write("os = %s\n" % fmconf_os) - - f.write("\n") - f.write("[Metadata]\n") - f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) - f.write("majorVersion = %s\n" % shell.getMajorVersion()) - f.write("pathPrefix = %s%s\n" % (shell.getRepoDir(), - "/" if not shell.getRepoDir().endswith("/") else "")) - f.write("version = %s\n" % shell.getVersion()) - - -def extractVersions(objdir): # pylint: disable=inconsistent-return-statements,invalid-name,missing-param-doc - # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc - """Extract the version from js.pc and put it into *.fuzzmanagerconf.""" - jspc_dir = sps.normExpUserPath(os.path.join(objdir, "js", "src")) - jspc_name = os.path.join(jspc_dir, "js.pc") - # Moved to /js/src/build/, see bug 1262241, Fx55 m-c rev 351194:2159959522f4 - jspc_new_dir = os.path.join(jspc_dir, "build") - jspc_new_name = os.path.join(jspc_new_dir, "js.pc") - - def fixateVer(pcfile): # pylint: disable=inconsistent-return-statements,invalid-name,missing-param-doc - # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc - """Returns the current version number (47.0a2).""" - with io.open(pcfile, mode="r", encoding="utf-8", errors="replace") as f: - for line in f: - if line.startswith("Version: "): - # Sample line: "Version: 47.0a2" - return line.split(": ")[1].rstrip() - - if os.path.isfile(jspc_name): - return fixateVer(jspc_name) - elif os.path.isfile(jspc_new_name): - return fixateVer(jspc_new_name) - - -def getLockDirPath(repoDir, tboxIdentifier=""): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Return the name of the lock directory, which is in the cache directory by default.""" - lockdir_name = ["shell", os.path.basename(repoDir), "lock"] - if tboxIdentifier: - lockdir_name.append(tboxIdentifier) - return os.path.join(ensureCacheDir(), "-".join(lockdir_name)) + return shell.get_shell_compiled_path() def makeTestRev(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -626,104 +623,99 @@ def testRev(rev): # pylint: disable=invalid-name,missing-docstring,missing-retu try: obtainShell(shell, updateToRev=rev) - except Exception: # pylint: disable=broad-except + except subprocess.CalledProcessError: return (options.compilationFailedLabel, "compilation failed") print("Testing...", end=" ") - return options.testAndLabel(shell.getShellCacheFullPath(), rev) + return options.testAndLabel(shell.get_shell_cache_js_bin_path(), rev) return testRev def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disable=invalid-name,missing-param-doc # pylint: disable=missing-raises-doc,missing-type-doc,too-many-branches,too-complex,too-many-statements """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" - assert os.path.isdir(getLockDirPath(shell.build_opts.repoDir)) - cached_no_shell = shell.getShellCacheFullPath() + ".busted" + assert sm_compile_helpers.get_lock_dir_path(Path.home(), shell.build_opts.repo_dir).is_dir() + cached_no_shell = shell.get_shell_cache_js_bin_path().with_suffix(".busted") - if os.path.isfile(shell.getShellCacheFullPath()): + if shell.get_shell_cache_js_bin_path().is_file(): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print("Found cached shell...") # Assuming that since the binary is present, everything else (e.g. symbols) is also present - verifyFullWinPageHeap(shell.getShellCacheFullPath()) + if platform.system() == "Windows": + sm_compile_helpers.verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) return - elif os.path.isfile(cached_no_shell): + elif cached_no_shell.is_file(): raise Exception("Found a cached shell that failed compilation...") - elif os.path.isdir(shell.getShellCacheDir()): + elif shell.get_shell_cache_dir().is_dir(): print("Found a cache dir without a successful/failed shell...") - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) + sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) - os.mkdir(shell.getShellCacheDir()) - hg_helpers.destroyPyc(shell.build_opts.repoDir) + shell.get_shell_cache_dir().mkdir() + hg_helpers.destroyPyc(shell.build_opts.repo_dir) s3cache_obj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) use_s3cache = s3cache_obj.connect() if use_s3cache: - if s3cache_obj.downloadFile(shell.getShellNameWithoutExt() + ".busted", - shell.getShellCacheFullPath() + ".busted"): - raise Exception("Found a .busted file for rev " + shell.getHgHash()) + if s3cache_obj.downloadFile(str(shell.get_shell_name_without_ext() + ".busted"), + str(shell.get_shell_cache_js_bin_path()) + ".busted"): + raise Exception("Found a .busted file for rev " + shell.get_hg_hash()) - if s3cache_obj.downloadFile(shell.getShellNameWithoutExt() + ".tar.bz2", - shell.getS3TarballWithExtFullPath()): + if s3cache_obj.downloadFile(str(shell.get_shell_name_without_ext() + ".tar.bz2"), + str(shell.get_s3_tar_with_ext_full_path())): print("Extracting shell...") - with tarfile.open(shell.getS3TarballWithExtFullPath(), "r") as f: - f.extractall(shell.getShellCacheDir()) + with tarfile.open(str(shell.get_s3_tar_with_ext_full_path()), "r") as f: + f.extractall(str(shell.get_shell_cache_dir())) # Delete tarball after downloading from S3 - os.remove(shell.getS3TarballWithExtFullPath()) - verifyFullWinPageHeap(shell.getShellCacheFullPath()) + shell.get_s3_tar_with_ext_full_path().unlink() + if platform.system() == "Windows": + sm_compile_helpers.verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) return try: if updateToRev: - updateRepo(shell.build_opts.repoDir, updateToRev) - if shell.build_opts.patchFile: - hg_helpers.patchHgRepoUsingMq(shell.build_opts.patchFile, shell.getRepoDir()) + # Print *with* a trailing newline to avoid breaking other stuff + print("Updating to rev %s in the %s repository..." % ( + updateToRev.decode("utf-8", errors="replace"), + str(shell.build_opts.repo_dir).decode("utf-8", errors="replace"))) + subprocess.run(["hg", "-R", str(shell.build_opts.repo_dir), + "update", "-C", "-r", updateToRev], + check=True, + # pylint: disable=no-member + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), + stderr=subprocess.DEVNULL, + timeout=999) + if shell.build_opts.patch_file: + hg_helpers.patch_hg_repo_with_mq(shell.build_opts.patch_file, shell.get_repo_dir()) cfgJsCompile(shell) - verifyFullWinPageHeap(shell.getShellCacheFullPath()) + if platform.system() == "Windows": + sm_compile_helpers.verify_full_win_pageheap(shell.get_shell_cache_js_bin_path()) except KeyboardInterrupt: - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) + sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) raise - except Exception as ex: + except subprocess.CalledProcessError as ex: # Remove the cache dir, but recreate it with only the .busted file. - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) - os.mkdir(shell.getShellCacheDir()) - createBustedFile(cached_no_shell, ex) + sps.rm_tree_incl_readonly(shell.get_shell_cache_dir()) + shell.get_shell_cache_dir().mkdir() + sm_compile_helpers.createBustedFile(cached_no_shell, ex) if use_s3cache: - s3cache_obj.uploadFileToS3(shell.getShellCacheFullPath() + ".busted") + s3cache_obj.uploadFileToS3(str(shell.get_shell_cache_js_bin_path()) + ".busted") raise finally: - if shell.build_opts.patchFile: - hg_helpers.hgQpopQrmAppliedPatch(shell.build_opts.patchFile, shell.getRepoDir()) + if shell.build_opts.patch_file: + hg_helpers.qpop_qrm_applied_patch(shell.build_opts.patch_file, shell.get_repo_dir()) if use_s3cache: - s3cache_obj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) + s3cache_obj.compressAndUploadDirTarball(str(shell.get_shell_cache_dir()), + str(shell.get_s3_tar_with_ext_full_path())) if updateLatestTxt: # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. - txt_info = "-".join(shell.getS3TarballWithExt().split("-")[:-1] + ["latest"]) + ".txt" - s3cache_obj.uploadStrToS3("", txt_info, shell.getS3TarballWithExt()) - os.remove(shell.getS3TarballWithExtFullPath()) - - -def updateRepo(repo, rev): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Update repository to the specific revision.""" - # Print *with* a trailing newline to avoid breaking other stuff - print("Updating to rev %s in the %s repository..." % (rev.decode("utf-8", errors="replace"), - repo.decode("utf-8", errors="replace"))) - sps.captureStdout(["hg", "-R", repo, "update", "-C", "-r", rev], ignoreStderr=True) - - -def verifyFullWinPageHeap(shellPath): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc - """Turn on full page heap verification on Windows.""" - # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx - # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ - if platform.system() == "Windows": - gflags_bin_path = os.path.join(os.getenv("PROGRAMW6432"), "Debugging Tools for Windows (x64)", "gflags.exe") - if os.path.isfile(gflags_bin_path) and os.path.isfile(shellPath): - print(subprocess.check_output([gflags_bin_path.decode("utf-8", errors="replace"), - "-p", "/enable", shellPath.decode("utf-8", errors="replace"), "/full"])) + txt_info = "-".join(str(shell.get_s3_tar_name_with_ext()).split("-")[:-1] + ["latest"]) + ".txt" + s3cache_obj.uploadStrToS3("", txt_info, str(shell.get_s3_tar_name_with_ext())) + shell.get_s3_tar_with_ext_full_path().unlink() def main(): diff --git a/src/funfuzz/js/inspect_shell.py b/src/funfuzz/js/inspect_shell.py index 56742afac..ea393fad6 100644 --- a/src/funfuzz/js/inspect_shell.py +++ b/src/funfuzz/js/inspect_shell.py @@ -7,16 +7,24 @@ """Allows inspection of the SpiderMonkey shell to ensure that it is compiled as intended with specified configurations. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, unicode_literals # isort:skip +import json import os import platform +import sys from lithium.interestingness.utils import env_with_path from shellescape import quote from ..util import subprocesses as sps +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + import subprocess + RUN_NSPR_LIB = "" RUN_PLDS_LIB = "" RUN_PLC_LIB = "" @@ -78,8 +86,12 @@ def archOfBinary(binary): # pylint: disable=inconsistent-return-statements,inva # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Test if a binary is 32-bit or 64-bit.""" # We can possibly use the python-magic-bin PyPI library in the future - unsplit_file_type = sps.captureStdout(["file", binary])[0] - filetype = unsplit_file_type.decode("utf-8", errors="replace").split(":", 1)[1] + unsplit_file_type = subprocess.run( + ["file", str(binary)], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stdout=subprocess.PIPE, + timeout=99).stdout.decode("utf-8", errors="replace") + filetype = unsplit_file_type.split(":", 1)[1] if platform.system() == "Windows": assert "MS Windows" in filetype return "32" if "Intel 80386 32-bit" in filetype else "64" @@ -133,11 +145,16 @@ def shellSupports(shellPath, args): # pylint: disable=invalid-name,missing-para def testBinary(shellPath, args, useValgrind): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Test the given shell with the given args.""" - test_cmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args - sps.vdump("The testing command is: " + " ".join(quote(x) for x in test_cmd)) - out, return_code = sps.captureStdout(test_cmd, combineStderr=True, ignoreStderr=True, - ignoreExitCode=True, env=env_with_path( - os.path.dirname(os.path.abspath(shellPath)))) + test_cmd = (constructVgCmdList() if useValgrind else []) + [str(shellPath)] + args + sps.vdump("The testing command is: " + " ".join(quote(str(x)) for x in test_cmd)) + test_cmd_result = subprocess.run( + test_cmd, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + env=env_with_path(str(shellPath.parent)), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=999) + out, return_code = test_cmd_result.stdout.decode("utf-8", errors="replace"), test_cmd_result.returncode sps.vdump("The exit code is: " + str(return_code)) return out, return_code @@ -151,14 +168,14 @@ def testJsShellOrXpcshell(s): # pylint: disable=invalid-name,missing-param-doc, def queryBuildConfiguration(s, parameter): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Test if a binary is compiled with specified parameters, in getBuildConfiguration().""" - ans = testBinary(s, ["-e", 'print(getBuildConfiguration()["' + parameter + '"])'], - False)[0] - return ans.decode("utf-8", errors="replace").find("true") != -1 + return json.loads(testBinary(s, + ["-e", 'print(getBuildConfiguration()["' + parameter + '"])'], + False)[0].rstrip().lower()) def verifyBinary(sh): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Verify that the binary is compiled as intended.""" - binary = sh.getShellCacheFullPath() + binary = sh.get_shell_cache_js_bin_path() assert archOfBinary(binary) == ("32" if sh.build_opts.enable32 else "64") diff --git a/src/funfuzz/js/js_interesting.py b/src/funfuzz/js/js_interesting.py index f6e81f3fc..b9ef08443 100644 --- a/src/funfuzz/js/js_interesting.py +++ b/src/funfuzz/js/js_interesting.py @@ -7,9 +7,10 @@ """Check whether a testcase causes an interesting result in a shell. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin +import io from optparse import OptionParser # pylint: disable=deprecated-module import os import platform @@ -24,8 +25,8 @@ from . import inspect_shell from ..util import create_collector -from ..util import detect_malloc_errors -from ..util import subprocesses as sps +from ..util import file_manipulation +from ..util import os_ops if sys.version_info.major == 2: if os.name == "posix": @@ -68,12 +69,15 @@ class ShellResult(object): # pylint: disable=missing-docstring,too-many-instanc # options dict should include: timeout, knownPath, collector, valgrind, shellIsDeterministic def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disable=too-complex,too-many-branches # pylint: disable=too-many-locals,too-many-statements - pathToBinary = runthis[0] # pylint: disable=invalid-name + + # If Lithium uses this as an interestingness test, logPrefix is likely not a Path object, so make it one. + logPrefix = Path(logPrefix) + pathToBinary = runthis[0].expanduser().resolve() # pylint: disable=invalid-name # This relies on the shell being a local one from compile_shell: # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like "./js" # pylint: disable=invalid-name - assert os.path.isfile(os.path.abspath(pathToBinary + ".fuzzmanagerconf")) - pc = ProgramConfiguration.fromBinary(os.path.abspath(pathToBinary).split(".")[0]) + assert pathToBinary.with_suffix(".fuzzmanagerconf").is_file() + pc = ProgramConfiguration.fromBinary(str(pathToBinary.parent / pathToBinary.stem)) pc.addProgramArguments(runthis[1:-1]) if options.valgrind: @@ -82,9 +86,12 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa valgrindSuppressions() + runthis) - preexec_fn = ulimitSet if os.name == "posix" else None # logPrefix should be a string for timed_run in Lithium version 0.2.1 to work properly, apparently - runinfo = timed_run.timed_run(runthis, options.timeout, logPrefix.encode("utf-8"), preexec_fn=preexec_fn) + runinfo = timed_run.timed_run( + [str(x) for x in runthis], # Convert all Paths/bytes to strings for Lithium + options.timeout, + str(logPrefix).encode("utf-8"), + preexec_fn=set_ulimit) lev = JS_FINE issues = [] @@ -92,9 +99,11 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and "readlines" everything into memory. - with open(logPrefix + "-out.txt") as f: + out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") + with io.open(str(out_log), "r", encoding="utf-8", errors="replace") as f: out = f.readlines() - with open(logPrefix + "-err.txt") as f: + err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt") + with io.open(str(err_log), "r", encoding="utf-8", errors="replace") as f: err = f.readlines() if options.valgrind and runinfo.return_code == VALGRIND_ERROR_EXIT_CODE: @@ -105,10 +114,11 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if valgrindErrorPrefix and line.startswith(valgrindErrorPrefix): issues.append(line.rstrip()) elif runinfo.sta == timed_run.CRASHED: - if sps.grabCrashLog(runthis[0], runinfo.pid, logPrefix, True): - with open(logPrefix + "-crash.txt") as f: + if os_ops.grab_crash_log(runthis[0], runinfo.pid, logPrefix, True): + crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") + with io.open(str(crash_log), "r", encoding="utf-8", errors="replace") as f: auxCrashData = [line.strip() for line in f.readlines()] - elif detect_malloc_errors.amiss(logPrefix): + elif file_manipulation.amiss(logPrefix): issues.append("malloc error") lev = max(lev, JS_NEW_ASSERT_OR_CRASH) elif runinfo.return_code == 0 and not in_compare_jit: @@ -133,13 +143,15 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa if activated and platform.system() == "Linux" and which("gdb") and not auxCrashData and not in_compare_jit: print("Note: No core file found on Linux - falling back to run via gdb") extracted_gdb_cmds = ["-ex", "run"] - with open(str(Path(__file__).parent.parent / "util" / "gdb_cmds.txt"), "r") as f: + with io.open(str(Path(__file__).parent.parent / "util" / "gdb_cmds.txt"), "r", + encoding="utf-8", errors="replace") as f: for line in f: if line.rstrip() and not line.startswith("#") and not line.startswith("echo"): extracted_gdb_cmds.append("-ex") extracted_gdb_cmds.append("%s" % line.rstrip()) no_main_log_gdb_log = subprocess.run( - ["gdb", "-n", "-batch"] + extracted_gdb_cmds + ["--args"] + runthis, + (["gdb", "-n", "-batch"] + extracted_gdb_cmds + ["--args"] + + [str(x) for x in runthis]), check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE @@ -170,9 +182,10 @@ def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disa print("%s | %s" % (logPrefix, summaryString(issues, lev, runinfo.elapsedtime))) if lev != JS_FINE: - with open(logPrefix + "-summary.txt", "w") as f: - f.writelines(["Number: " + logPrefix + "\n", - "Command: " + " ".join(quote(x) for x in runthis) + "\n"] + + summary_log = (logPrefix.parent / (logPrefix.stem + "-summary")).with_suffix(".txt") + with io.open(str(summary_log), "w", encoding="utf-8", errors="replace") as f: + f.writelines(["Number: " + str(logPrefix) + "\n", + "Command: " + " ".join(quote(str(x)) for x in runthis) + "\n"] + ["Status: " + i + "\n" for i in issues]) self.lev = lev @@ -236,8 +249,8 @@ def summaryString(issues, level, elapsedtime): # pylint: disable=invalid-name,m def truncateFile(fn, maxSize): # pylint: disable=invalid-name,missing-docstring - if os.path.exists(fn) and os.path.getsize(fn) > maxSize: - with open(fn, "r+") as f: + if fn.is_file() and fn.stat().st_size > maxSize: + with io.open(str(fn), "r+", encoding="utf-8", errors="replace") as f: f.truncate(maxSize) @@ -250,30 +263,36 @@ def deleteLogs(logPrefix): # pylint: disable=invalid-name,missing-param-doc,mis """Whoever might call baseLevel should eventually call this function (unless a bug was found).""" # If this turns up a WindowsError on Windows, remember to have excluded fuzzing locations in # the search indexer, anti-virus realtime protection and backup applications. - os.remove(logPrefix + "-out.txt") - os.remove(logPrefix + "-err.txt") - if os.path.exists(logPrefix + "-crash.txt"): - os.remove(logPrefix + "-crash.txt") - if os.path.exists(logPrefix + "-vg.xml"): - os.remove(logPrefix + "-vg.xml") + (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt").unlink() + (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt").unlink() + crash_log = (logPrefix.parent / (logPrefix.stem + "-crash")).with_suffix(".txt") + if crash_log.is_file(): + crash_log.unlink() + valgrind_xml = (logPrefix.parent / (logPrefix.stem + "-vg")).with_suffix(".xml") + if valgrind_xml.is_file(): + valgrind_xml.unlink() # pylint: disable=fixme # FIXME: in some cases, subprocesses gzips a core file only for us to delete it immediately. - if os.path.exists(logPrefix + "-core.gz"): - os.remove(logPrefix + "-core.gz") + core_gzip = (logPrefix.parent / (logPrefix.stem + "-core")).with_suffix(".gz") + if core_gzip.is_file(): + core_gzip.unlink() -def ulimitSet(): # pylint: disable=invalid-name - """When called as a preexec_fn, sets appropriate resource limits for the JS shell. Must only be called on POSIX.""" - # module only available on POSIX - import resource # pylint: disable=import-error +def set_ulimit(): + """Sets appropriate resource limits for the JS shell when on POSIX.""" + try: + import resource # pylint: disable=import-error - # Limit address space to 2GB (or 1GB on ARM boards such as ODROID). - GB = 2**30 # pylint: disable=invalid-name - resource.setrlimit(resource.RLIMIT_AS, (2 * GB, 2 * GB)) + # log.debug("Limit address space to 2GB (or 1GB on ARM boards such as ODROID)") + giga_byte = 2**30 + resource.setrlimit(resource.RLIMIT_AS, (2 * giga_byte, 2 * giga_byte)) # pylint: disable=no-member - # Limit corefiles to 0.5 GB. - halfGB = int(GB // 2) # pylint: disable=invalid-name - resource.setrlimit(resource.RLIMIT_CORE, (halfGB, halfGB)) + # log.debug("Limit corefiles to 0.5 GB") + half_giga_byte = int(giga_byte // 2) + resource.setrlimit(resource.RLIMIT_CORE, (half_giga_byte, half_giga_byte)) # pylint: disable=no-member + except ImportError: + # log.debug("Skipping resource import as a non-POSIX platform was detected: %s", platform.system()) + return def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc @@ -299,10 +318,10 @@ def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missin if len(args) < 2: raise Exception("Not enough positional arguments") options.knownPath = args[0] - options.jsengineWithArgs = args[1:] - options.collector = create_collector.createCollector("jsfunfuzz") - if not os.path.exists(options.jsengineWithArgs[0]): - raise Exception("js shell does not exist: " + options.jsengineWithArgs[0]) + options.jsengineWithArgs = [Path(args[1]).resolve()] + args[2:-1] + [Path(args[-1]).resolve()] + assert options.jsengineWithArgs[0].is_file() # js shell + assert options.jsengineWithArgs[-1].is_file() # testcase + options.collector = create_collector.make_collector() options.shellIsDeterministic = inspect_shell.queryBuildConfiguration( options.jsengineWithArgs[0], "more-deterministic") @@ -319,28 +338,31 @@ def init(args): # pylint: disable=missing-docstring # FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme -def interesting(_args, tempPrefix): # pylint: disable=invalid-name,missing-docstring,missing-return-doc +def interesting(_args, cwd_prefix): # pylint: disable=missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc + cwd_prefix = Path(cwd_prefix) # Lithium uses this function and cwd_prefix from Lithium is not a Path options = gOptions # options, runthis, logPrefix, in_compare_jit - res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) - truncateFile(tempPrefix + "-out.txt", 1000000) - truncateFile(tempPrefix + "-err.txt", 1000000) + res = ShellResult(options, options.jsengineWithArgs, cwd_prefix, False) + out_log = (cwd_prefix.parent / (cwd_prefix.stem + "-out")).with_suffix(".txt") + err_log = (cwd_prefix.parent / (cwd_prefix.stem + "-err")).with_suffix(".txt") + truncateFile(out_log, 1000000) + truncateFile(err_log, 1000000) return res.lev >= gOptions.minimumInterestingLevel # For direct, manual use def main(): # pylint: disable=missing-docstring options = parseOptions(sys.argv[1:]) - tempPrefix = "m" # pylint: disable=invalid-name - res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) # pylint: disable=no-member + cwd_prefix = Path.cwd() / "m" + res = ShellResult(options, options.jsengineWithArgs, cwd_prefix, False) # pylint: disable=no-member print(res.lev) if options.submit: # pylint: disable=no-member if res.lev >= options.minimumInterestingLevel: # pylint: disable=no-member testcaseFilename = options.jsengineWithArgs[-1] # pylint: disable=invalid-name,no-member print("Submitting %s" % testcaseFilename) quality = 0 - options.collector.submit(res.crashInfo, testcaseFilename, quality) # pylint: disable=no-member + options.collector.submit(res.crashInfo, str(testcaseFilename), quality) # pylint: disable=no-member else: print("Not submitting (not interesting)") diff --git a/src/funfuzz/js/link_fuzzer.py b/src/funfuzz/js/link_fuzzer.py new file mode 100644 index 000000000..8d90a43b5 --- /dev/null +++ b/src/funfuzz/js/link_fuzzer.py @@ -0,0 +1,42 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Concatenate js files to create jsfunfuzz. +""" + +from __future__ import absolute_import + +import io +import sys + +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + + +def link_fuzzer(target_path, prologue=""): + """Links files together to create the full jsfunfuzz file. + + Args: + target_path (Path): Target file with full path, to be created + prologue (str): Contents to be prepended to the target file + """ + base_dir = Path(__file__).parent + + with io.open(str(target_path), "w", encoding="utf-8", errors="replace") as f: # Create the full jsfunfuzz file + if prologue: + f.write(prologue) + + for entry in (base_dir / "files_to_link.txt").read_text().split(): # pylint: disable=no-member + entry = entry.rstrip() + if entry and not entry.startswith("#"): + file_path = base_dir / Path(entry) + file_name = "\n\n// %s\n\n" % str(file_path).split("funfuzz", 1)[1][1:] + if isinstance(file_name, bytes): # For dual Python 2 and 3 compatibility + file_name = file_name.decode("utf-8", errors="replace") + f.write(file_name) + f.write(file_path.read_text()) # pylint: disable=no-member diff --git a/src/funfuzz/js/loop.py b/src/funfuzz/js/loop.py index 6508b23d3..2251cf986 100644 --- a/src/funfuzz/js/loop.py +++ b/src/funfuzz/js/loop.py @@ -7,24 +7,32 @@ """Allows the funfuzz harness to run continuously. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io import json from optparse import OptionParser # pylint: disable=deprecated-module import os -import subprocess import sys import time from . import compare_jit from . import js_interesting +from . import link_fuzzer from . import shell_flags from ..util import create_collector from ..util import file_manipulation -from ..util import link_js from ..util import lithium_helpers from ..util import subprocesses as sps +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc parser = OptionParser() @@ -39,8 +47,8 @@ def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-r default=False, help="Pass a random set of flags (e.g. --ion-eager) to the js engine") parser.add_option("--repo", - action="store", dest="repo", - default=os.path.expanduser("~/trees/mozilla-central/"), + action="store", + dest="repo", help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection") parser.add_option("--build", action="store", dest="build_options_str", @@ -52,6 +60,12 @@ def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-r help="use valgrind with a reasonable set of options") options, args = parser.parse_args(args) + # optparse does not recognize pathlib - we will need to move to argparse + if options.repo: + options.repo = Path(options.repo) + else: + options.repo = Path.home() / "trees" / "mozilla-central" + if options.valgrind and options.use_compare_jit: print("Note: When running compare_jit, the --valgrind option will be ignored") @@ -63,7 +77,7 @@ def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-r # FIXME: We can probably remove args[1] # pylint: disable=fixme options.knownPath = "mozilla-central" - options.jsEngine = args[2] + options.jsEngine = Path(args[2]) options.engineFlags = args[3:] return options @@ -74,45 +88,37 @@ def showtail(filename): # pylint: disable=missing-docstring # FIXME: Get jsfunfuzz to output start & end of interesting result boundaries instead of this. cmd = [] cmd.extend(["tail", "-n", "20"]) - cmd.append(filename) + cmd.append(str(filename)) print(" ".join(cmd)) print() - subprocess.check_call(cmd) + subprocess.run(cmd, check=True) print() print() -def linkFuzzer(target_fn, prologue): # pylint: disable=invalid-name,missing-docstring - source_base = os.path.dirname(os.path.abspath(__file__)) - file_list_fn = sps.normExpUserPath(os.path.join(source_base, "files_to_link.txt")) - link_js.link_js(target_fn, file_list_fn, source_base, prologue) - - def makeRegressionTestPrologue(repo): # pylint: disable=invalid-name,missing-param-doc # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests.""" - repo = sps.normExpUserPath(repo) + os.sep - return """ const regressionTestsRoot = %s; const libdir = regressionTestsRoot + %s; // needed by jit-tests const regressionTestList = %s; -""" % (json.dumps(repo), - json.dumps(os.path.join("js", "src", "jit-test", "lib") + os.sep), +""" % (json.dumps(str(repo) + os.sep), + json.dumps(os.sep.join(["js", "src", "jit-test", "lib"]) + os.sep), json.dumps(inTreeRegressionTests(repo))) def inTreeRegressionTests(repo): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - jit_tests = jsFilesIn(len(repo), os.path.join(repo, "js", "src", "jit-test", "tests")) - js_tests = jsFilesIn(len(repo), os.path.join(repo, "js", "src", "tests")) + jit_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "jit-test" / "tests") + js_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests") return jit_tests + js_tests def jsFilesIn(repoPathLength, root): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc return [os.path.join(path, filename)[repoPathLength:] - for path, _dirs, files in os.walk(sps.normExpUserPath(root)) + for path, _dirs, files in os.walk(str(root)) for filename in files if filename.endswith(".js")] @@ -124,21 +130,23 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in engineFlags = options.engineFlags # pylint: disable=invalid-name startTime = time.time() # pylint: disable=invalid-name - if os.path.isdir(sps.normExpUserPath(options.repo)): + if options.repo.is_dir(): regressionTestPrologue = makeRegressionTestPrologue(options.repo) # pylint: disable=invalid-name else: regressionTestPrologue = "" # pylint: disable=invalid-name - fuzzjs = sps.normExpUserPath(os.path.join(wtmpDir, "jsfunfuzz.js")) - linkFuzzer(fuzzjs, regressionTestPrologue) + fuzzjs = wtmpDir / "jsfunfuzz.js" + + link_fuzzer.link_fuzzer(fuzzjs, regressionTestPrologue) + assert fuzzjs.is_file() iteration = 0 while True: if targetTime and time.time() > startTime + targetTime: print("Out of time!") - os.remove(fuzzjs) - if not os.listdir(wtmpDir): - os.rmdir(wtmpDir) + fuzzjs.unlink() + if not os.listdir(str(wtmpDir)): + wtmpDir.remove() break # Construct command needed to loop jsfunfuzz fuzzing. @@ -156,26 +164,29 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in js_interesting_options = js_interesting.parseOptions(js_interesting_args) iteration += 1 - logPrefix = sps.normExpUserPath(os.path.join(wtmpDir, "w" + str(iteration))) # pylint: disable=invalid-name + logPrefix = wtmpDir / ("w" + str(iteration)) # pylint: disable=invalid-name res = js_interesting.ShellResult(js_interesting_options, # pylint: disable=no-member js_interesting_options.jsengineWithArgs, logPrefix, False) if res.lev != js_interesting.JS_FINE: - showtail(logPrefix + "-out.txt") - showtail(logPrefix + "-err.txt") + out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") + showtail(out_log) + err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt") + showtail(err_log) # splice jsfunfuzz.js with `grep "/*FRC-" wN-out` - filenameToReduce = logPrefix + "-reduced.js" # pylint: disable=invalid-name + reduced_log = (logPrefix.parent / (logPrefix.stem + "-reduced")).with_suffix(".js") [before, after] = file_manipulation.fuzzSplice(fuzzjs) - with open(logPrefix + "-out.txt", "r") as f: + with io.open(str(out_log), "r", encoding="utf-8", errors="replace") as f: newfileLines = before + [ # pylint: disable=invalid-name l.replace("/*FRC-", "/*") for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after - with open(logPrefix + "-orig.js", "w") as f: + orig_log = (logPrefix.parent / (logPrefix.stem + "-orig")).with_suffix(".js") + with io.open(str(orig_log), "w", encoding="utf-8", errors="replace") as f: f.writelines(newfileLines) - with open(filenameToReduce, "w") as f: + with io.open(str(reduced_log), "w", encoding="utf-8", errors="replace") as f: f.writelines(newfileLines) # Run Lithium and autobisectjs (make a reduced testcase and find a regression window) @@ -187,15 +198,18 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in itest.append("--timeout=" + str(options.timeout)) itest.append(options.knownPath) (lithResult, _lithDetails, autoBisectLog) = lithium_helpers.pinpoint( # pylint: disable=invalid-name - itest, logPrefix, options.jsEngine, engineFlags, filenameToReduce, options.repo, + itest, logPrefix, options.jsEngine, engineFlags, reduced_log, options.repo, options.build_options_str, targetTime, res.lev) # Upload with final output if lithResult == lithium_helpers.LITH_FINISHED: # pylint: disable=no-member - fargs = js_interesting_options.jsengineWithArgs[:-1] + [filenameToReduce] + fargs = js_interesting_options.jsengineWithArgs[:-1] + [reduced_log] # pylint: disable=invalid-name - retestResult = js_interesting.ShellResult(js_interesting_options, fargs, logPrefix + "-final", False) + retestResult = js_interesting.ShellResult(js_interesting_options, + fargs, + logPrefix.parent / (logPrefix.stem + "-final"), + False) if retestResult.lev > js_interesting.JS_FINE: res = retestResult quality = 0 @@ -204,29 +218,30 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=in else: quality = 10 - print("Submitting %s (quality=%s) at %s" % (filenameToReduce, quality, time.asctime())) + print("Submitting %s (quality=%s) at %s" % (reduced_log, quality, time.asctime())) metadata = {} if autoBisectLog: metadata = {"autoBisectLog": "".join(autoBisectLog)} - collector.submit(res.crashInfo, filenameToReduce, quality, metaData=metadata) - print("Submitted %s" % filenameToReduce) + collector.submit(res.crashInfo, str(reduced_log), quality, metaData=metadata) + print("Submitted %s" % reduced_log) else: are_flags_deterministic = "--dump-bytecode" not in engineFlags and "-D" not in engineFlags # pylint: disable=no-member if options.use_compare_jit and res.lev == js_interesting.JS_FINE and \ js_interesting_options.shellIsDeterministic and are_flags_deterministic: - linesToCompare = jitCompareLines(logPrefix + "-out.txt", "/*FCM*/") # pylint: disable=invalid-name - jitcomparefilename = logPrefix + "-cj-in.js" - with open(jitcomparefilename, "w") as f: + out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt") + linesToCompare = jitCompareLines(out_log, "/*FCM*/") # pylint: disable=invalid-name + jitcomparefilename = (logPrefix.parent / (logPrefix.stem + "-cj-in")).with_suffix(".js") + with io.open(str(jitcomparefilename), "w", encoding="utf-8", errors="replace") as f: f.writelines(linesToCompare) # pylint: disable=invalid-name anyBug = compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename, - logPrefix + "-cj", options.repo, + logPrefix.parent / (logPrefix.stem + "-cj"), options.repo, options.build_options_str, targetTime, js_interesting_options) if not anyBug: - os.remove(jitcomparefilename) + jitcomparefilename.unlink() js_interesting.deleteLogs(logPrefix) @@ -251,7 +266,7 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid "wasmIsSupported = function() { return true; };\n", "// DDBEGIN\n" ] - with open(jsfunfuzzOutputFilename, "r") as f: + with io.open(str(jsfunfuzzOutputFilename), "r", encoding="utf-8", errors="replace") as f: for line in f: if line.startswith(marker): sline = line[len(marker):] @@ -269,5 +284,5 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid if __name__ == "__main__": # pylint: disable=no-member - many_timed_runs(None, sps.createWtmpDir(os.getcwdu() if sys.version_info.major == 2 else os.getcwd()), - sys.argv[1:], create_collector.createCollector("jsfunfuzz")) + many_timed_runs(None, sps.make_wtmp_dir(Path(os.getcwdu() if sys.version_info.major == 2 else os.getcwd())), + sys.argv[1:], create_collector.make_collector()) diff --git a/src/funfuzz/js/shell_flags.py b/src/funfuzz/js/shell_flags.py index 63b533679..2d70d96b5 100644 --- a/src/funfuzz/js/shell_flags.py +++ b/src/funfuzz/js/shell_flags.py @@ -7,7 +7,7 @@ """Allows detection of support for various command-line flags. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, unicode_literals # isort:skip import multiprocessing import random @@ -33,9 +33,7 @@ def shell_supports_flag(shell_path, flag): Returns: bool: True if the flag is supported, i.e. does not cause the shell to throw an error, False otherwise. """ - dummy_parameters = ["-e", "42"] - # This can be refactored when sps.captureStdout is gone - out = inspect_shell.shellSupports(shell_path, [flag] + dummy_parameters) + out = inspect_shell.shellSupports(shell_path, [flag, "-e", "42"]) return out diff --git a/src/funfuzz/loop_bot.py b/src/funfuzz/loop_bot.py index c3980bf1c..ebc110132 100644 --- a/src/funfuzz/loop_bot.py +++ b/src/funfuzz/loop_bot.py @@ -7,18 +7,24 @@ """Loop of { update repos, call bot } to allow things to run unattended All command-line options are passed through to bot -Since this script updates the fuzzing repo, it should be very simple, and use subprocess.call() rather than import +This script used to update funfuzz itself (when run as scripts, no longer supported) +so it uses subprocess.run() rather than import Config-ish bits should move to bot, OR move into a config file, -OR this file should subprocess-call ITSELF rather than using a while loop. +OR this file should subprocess-run ITSELF rather than using a while loop. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip -import subprocess +import os import sys import time +if sys.version_info.major == 2 and os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + import subprocess + def loop_seq(cmd_seq, wait_time): # pylint: disable=missing-param-doc,missing-type-doc """Call a sequence of commands in a loop. @@ -29,7 +35,7 @@ def loop_seq(cmd_seq, wait_time): # pylint: disable=missing-param-doc,missing-t print("localLoop #%d!" % i) for cmd in cmd_seq: try: - subprocess.check_call(cmd) + subprocess.run(cmd, check=True) except subprocess.CalledProcessError as ex: print("Something went wrong when calling: %r" % (cmd,)) print("%r" % (ex,)) diff --git a/src/funfuzz/util/__init__.py b/src/funfuzz/util/__init__.py index 30015c124..1cd755106 100644 --- a/src/funfuzz/util/__init__.py +++ b/src/funfuzz/util/__init__.py @@ -10,13 +10,13 @@ from . import crashesat from . import create_collector -from . import detect_malloc_errors from . import file_manipulation from . import fork_join from . import hg_helpers -from . import link_js from . import lithium_helpers from . import lock_dir +from . import os_ops from . import repos_update from . import s3cache +from . import sm_compile_helpers from . import subprocesses diff --git a/src/funfuzz/util/crashesat.py b/src/funfuzz/util/crashesat.py index 7a3758821..bde87f9fd 100644 --- a/src/funfuzz/util/crashesat.py +++ b/src/funfuzz/util/crashesat.py @@ -7,10 +7,10 @@ """Lithium's "crashesat" interestingness test to assess whether a binary crashes with a possibly-desired signature on the stack. -Not merged into Lithium as it still relies on grabCrashLog. +Not merged into Lithium as it still relies on grab_crash_log. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip import argparse import logging @@ -19,7 +19,7 @@ import lithium.interestingness.timed_run as timed_run from lithium.interestingness.utils import file_contains -from . import subprocesses as sps +from . import os_ops if sys.version_info.major == 2: from pathlib2 import Path @@ -53,7 +53,7 @@ def interesting(cli_args, temp_prefix): # Examine stack for crash signature, this is needed if args.sig is specified. runinfo = timed_run.timed_run(args.cmd_with_flags, args.timeout, temp_prefix) if runinfo.sta == timed_run.CRASHED: - sps.grabCrashLog(args.cmd_with_flags[0], runinfo.pid, temp_prefix, True) + os_ops.grab_crash_log(args.cmd_with_flags[0], runinfo.pid, temp_prefix, True) crash_log = Path(temp_prefix + "-crash.txt") time_str = " (%.3f seconds)" % runinfo.elapsedtime diff --git a/src/funfuzz/util/create_collector.py b/src/funfuzz/util/create_collector.py index 96a4df6c2..11953d35c 100644 --- a/src/funfuzz/util/create_collector.py +++ b/src/funfuzz/util/create_collector.py @@ -7,22 +7,27 @@ """Functions here make use of a Collector created from FuzzManager. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip -import os +import sys from Collector.Collector import Collector +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error -def createCollector(tool): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - assert tool == "jsfunfuzz" - cache_dir = os.path.normpath(os.path.expanduser(os.path.join("~", "sigcache"))) - try: - os.mkdir(cache_dir) - except OSError: - pass # cache_dir already exists - collector = Collector(sigCacheDir=cache_dir, tool=tool) - return collector + +def make_collector(): + """Creates a jsfunfuzz collector specifying ~/sigcache as the signature cache dir + + Returns: + Collector: jsfunfuzz collector object + """ + sigcache_path = Path.home() / "sigcache" + sigcache_path.mkdir(exist_ok=True) # pylint: disable=no-member + return Collector(sigCacheDir=str(sigcache_path), tool="jsfunfuzz") def printCrashInfo(crashInfo): # pylint: disable=invalid-name,missing-docstring diff --git a/src/funfuzz/util/detect_malloc_errors.py b/src/funfuzz/util/detect_malloc_errors.py deleted file mode 100644 index 7a6c5568e..000000000 --- a/src/funfuzz/util/detect_malloc_errors.py +++ /dev/null @@ -1,50 +0,0 @@ -# coding=utf-8 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -"""Look for "szone_error" (Tiger), "malloc_error_break" (Leopard), "MallocHelp" (?) -which are signs of malloc being unhappy (double free, out-of-memory, etc). -""" - -from __future__ import absolute_import, print_function # isort:skip - -PLINE = "" -PPLINE = "" - - -def amiss(log_prefix): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc - found_something = False - global PLINE, PPLINE # pylint: disable=global-statement - - PLINE = "" - PPLINE = "" - - with open(log_prefix + "-err.txt") as f: - for line in f: - if scanLine(line): - found_something = True - break # Don't flood the log with repeated malloc failures - - return found_something - - -def scanLine(line): # pylint: disable=inconsistent-return-statements,invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - global PPLINE, PLINE # pylint: disable=global-statement - - line = line.strip("\x07").rstrip("\n") - - if (line.find("szone_error") != -1 or - line.find("malloc_error_break") != -1 or - line.find("MallocHelp") != -1): - if PLINE.find("can't allocate region") == -1: - print() - print(PPLINE) - print(PLINE) - print(line) - return True - - PPLINE = PLINE - PLINE = line diff --git a/src/funfuzz/util/file_manipulation.py b/src/funfuzz/util/file_manipulation.py index 56d29b006..c3cff7236 100644 --- a/src/funfuzz/util/file_manipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -7,7 +7,29 @@ """Functions dealing with files and their contents. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip + +import io + + +def amiss(log_prefix): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc + """Look for "szone_error" (Tiger), "malloc_error_break" (Leopard), "MallocHelp" (?) + which are signs of malloc being unhappy (double free, out-of-memory, etc). + """ + found_something = False + err_log = (log_prefix.parent / (log_prefix.stem + "-err")).with_suffix(".txt") + with io.open(str(err_log), "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip("\x07").rstrip("\n") + if (line.find("szone_error") != -1 or + line.find("malloc_error_break") != -1 or + line.find("MallocHelp") != -1): + print() + print(line) + found_something = True + break # Don't flood the log with repeated malloc failures + + return found_something def fuzzSplice(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc @@ -15,7 +37,7 @@ def fuzzSplice(filename): # pylint: disable=invalid-name,missing-param-doc,miss """Return the lines of a file, minus the ones between the two lines containing SPLICE.""" before = [] after = [] - with open(filename, "r") as f: + with io.open(str(filename), "r", encoding="utf-8", errors="replace") as f: for line in f: before.append(line) if line.find("SPLICE") != -1: diff --git a/src/funfuzz/util/fork_join.py b/src/funfuzz/util/fork_join.py index a5650fd23..f2a50169a 100644 --- a/src/funfuzz/util/fork_join.py +++ b/src/funfuzz/util/fork_join.py @@ -7,14 +7,19 @@ """Functions dealing with multiple processes. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io import multiprocessing -import os import sys from past.builtins import range +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + # Call |fun| in a bunch of separate processes, then wait for them all to finish. # fun is called with someArgs, plus an additional argument with a numeric ID. @@ -23,7 +28,7 @@ def forkJoin(logDir, numProcesses, fun, *someArgs): # pylint: disable=invalid-n def showFile(fn): # pylint: disable=invalid-name,missing-docstring print("==== %s ====" % fn) print() - with open(fn) as f: + with io.open(str(fn), "r", encoding="utf-8", errors="replace") as f: for line in f: print(line.rstrip()) print() @@ -44,20 +49,29 @@ def showFile(fn): # pylint: disable=invalid-name,missing-docstring p.join() print("=== Child process #%d exited with code %d ===" % (i, p.exitcode)) print() - showFile(logFileName(logDir, i, "out")) - showFile(logFileName(logDir, i, "err")) + showFile(log_name(logDir, i, "out")) + showFile(log_name(logDir, i, "err")) print() # Functions used by forkJoin are top-level so they can be "pickled" (required on Windows) -def logFileName(logDir, i, t): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - return os.path.join(logDir, "forkjoin-" + str(i) + "-" + t + ".txt") +def log_name(log_dir, i, log_type): + """Returns the path of the forkjoin log file as a string. + + Args: + log_dir (str): Directory of the log file + i (int): Log number + log_type (str): Log type + + Returns: + str: The forkjoin log file path + """ + return str(Path(log_dir) / ("forkjoin-%s-%s.txt" % (i, log_type))) def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring - sys.stdout = open(logFileName(logDir, i, "out"), "w", buffering=0) - sys.stderr = open(logFileName(logDir, i, "err"), "w", buffering=0) + sys.stdout = io.open(log_name(logDir, i, "out"), "wb", buffering=0) + sys.stderr = io.open(log_name(logDir, i, "err"), "wb", buffering=0) fun(*(someArgs + (i,))) @@ -65,19 +79,19 @@ def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=inval # * "Green Chairs" from the first few processes # * A pause and error (with stack trace) from process 5 # * "Green Chairs" again from the rest. -def test_forkJoin(): # pylint: disable=invalid-name,missing-docstring - forkJoin(".", 8, test_forkJoin_inner, "Green", "Chairs") +# def test_forkJoin(): # pylint: disable=invalid-name,missing-docstring +# forkJoin(".", 8, test_forkJoin_inner, "Green", "Chairs") -def test_forkJoin_inner(adj, noun, forkjoin_id): # pylint: disable=invalid-name,missing-docstring - import time - print("%s %s" % (adj, noun)) - print(forkjoin_id) - if forkjoin_id == 5: - time.sleep(1) - raise NameError() +# def test_forkJoin_inner(adj, noun, forkjoin_id): # pylint: disable=invalid-name,missing-docstring +# import time +# print("%s %s" % (adj, noun)) +# print(forkjoin_id) +# if forkjoin_id == 5: +# time.sleep(1) +# raise NameError() -if __name__ == "__main__": - print("test_forkJoin():") - test_forkJoin() +# if __name__ == "__main__": +# print("test_forkJoin():") +# test_forkJoin() diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py index 3ab1a49a4..24400e53b 100644 --- a/src/funfuzz/util/hg_helpers.py +++ b/src/funfuzz/util/hg_helpers.py @@ -7,64 +7,91 @@ """Helper functions involving Mercurial (hg). """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import input # pylint: disable=redefined-builtin import configparser import os import re -import subprocess import sys from . import subprocesses as sps +if sys.version_info.major == 2: + from pathlib2 import Path + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error +else: + from pathlib import Path # pylint: disable=import-error + import subprocess -def destroyPyc(repoDir): # pylint: disable=invalid-name,missing-docstring + +def destroyPyc(repo_dir): # pylint: disable=invalid-name,missing-docstring # This is roughly equivalent to ["hg", "purge", "--all", "--include=**.pyc"]) # but doesn't run into purge's issues (incompatbility with -R, requiring an hg extension) - for root, dirs, files in os.walk(repoDir): + for root, dirs, files in os.walk(str(repo_dir)): for fn in files: # pylint: disable=invalid-name if fn.endswith(".pyc"): - os.remove(os.path.join(root, fn)) + (Path(root) / fn).unlink() if ".hg" in dirs: # Don't visit .hg dir dirs.remove(".hg") -def ensureMqEnabled(): # pylint: disable=invalid-name,missing-raises-doc - """Ensure that mq is enabled in the ~/.hgrc file.""" - user_hgrc = os.path.join(os.path.expanduser("~"), ".hgrc") - assert os.path.isfile(user_hgrc) +def ensure_mq_enabled(): + """Ensure that mq is enabled in the ~/.hgrc file. + + Raises: + NoOptionError: Raises if an mq entry is not found in [extensions] + """ + user_hgrc = Path.home() / ".hgrc" + assert user_hgrc.is_file() # pylint: disable=no-member user_hgrc_cfg = configparser.SafeConfigParser() - user_hgrc_cfg.read(user_hgrc) + user_hgrc_cfg.read(str(user_hgrc)) try: user_hgrc_cfg.get("extensions", "mq") except configparser.NoOptionError: - raise Exception('Please first enable mq in ~/.hgrc by having "mq =" in [extensions].') + print('Please first enable mq in ~/.hgrc by having "mq =" in [extensions].') + raise -def findCommonAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-docstring,missing-return-doc +def findCommonAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - return sps.captureStdout(["hg", "-R", repoDir, "log", "-r", "ancestor(" + a + "," + b + ")", - "--template={node|short}"])[0] + return subprocess.run( + ["hg", "-R", str(repo_dir), "log", "-r", "ancestor(" + a + "," + b + ")", "--template={node|short}"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=999 + ).stdout.decode("utf-8", errors="replace") -def isAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc +def isAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist.""" - return sps.captureStdout(["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", - "--template={node|short}"])[0] != "" + return subprocess.run( + ["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=999 + ).stdout.decode("utf-8", errors="replace") != "" -def existsAndIsAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc +def existsAndIsAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Return true iff |a| exists and is an ancestor of |b|.""" # Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error - out = sps.captureStdout(["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", - "--template={node|short}"], combineStderr=True, ignoreExitCode=True)[0] - return out != "" and out.decode("utf-8", errors="replace").find("abort: unknown revision") < 0 + out = subprocess.run( + ["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=999 + ).stdout.decode("utf-8", errors="replace") + return out != "" and out.find("abort: unknown revision") < 0 def get_cset_hash_from_bisect_msg(msg): @@ -86,16 +113,31 @@ def get_cset_hash_from_bisect_msg(msg): raise ValueError("Bisection output format required for hash extraction unavailable. The variable msg is: %s" % msg) -def getRepoHashAndId(repoDir, repoRev="parents() and default"): # pylint: disable=invalid-name,missing-param-doc - # pylint: disable=missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc +def get_repo_hash_and_id(repo_dir, repo_rev="parents() and default"): """Return the repository hash and id, and whether it is on default. It will also ask what the user would like to do, should the repository not be on default. + + Args: + repo_dir (Path): Full path to the repository + repo_rev (str): Intended Mercurial changeset details to retrieve + + Raises: + ValueError: Raises if the input is invalid + + Returns: + tuple: Changeset hash, local numerical ID, boolean on whether the repository is on default tip """ # This returns null if the repository is not on default. - hg_log_template_cmds = ["hg", "-R", repoDir, "log", "-r", repoRev, + hg_log_template_cmds = ["hg", "-R", str(repo_dir), "log", "-r", repo_rev, "--template", "{node|short} {rev}"] - hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] + hg_id_full = subprocess.run( + hg_log_template_cmds, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=99 + ).stdout.decode("utf-8", errors="replace") is_on_default = bool(hg_id_full) if not is_on_default: update_default = input("Not on default tip! " @@ -105,80 +147,117 @@ def getRepoHashAndId(repoDir, repoRev="parents() and default"): # pylint: disab print("Aborting...") sys.exit(0) elif update_default == "d": - subprocess.check_call(["hg", "-R", repoDir, "update", "default"]) + subprocess.run(["hg", "-R", str(repo_dir), "update", "default"], check=True) is_on_default = True elif update_default == "u": - hg_log_template_cmds = ["hg", "-R", repoDir, "log", "-r", "parents()", "--template", + hg_log_template_cmds = ["hg", "-R", str(repo_dir), "log", "-r", "parents()", "--template", "{node|short} {rev}"] else: - raise Exception("Invalid choice.") - hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] + raise ValueError("Invalid choice.") + hg_id_full = subprocess.run( + hg_log_template_cmds, + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stdout=subprocess.PIPE, + timeout=99 + ).stdout.decode("utf-8", errors="replace") assert hg_id_full != "" - (hg_id_hash, hg_id_local_num) = hg_id_full.decode("utf-8", errors="replace").split(" ") + (hg_id_hash, hg_id_local_num) = hg_id_full.split(" ") sps.vdump("Finished getting the hash and local id number of the repository.") return hg_id_hash, hg_id_local_num, is_on_default -def getRepoNameFromHgrc(repoDir): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Look in the hgrc file in the .hg directory of the repository and return the name.""" - assert isRepoValid(repoDir) +def hgrc_repo_name(repo_dir): + """Look in the hgrc file in the .hg directory of the Mercurial repository and return the name. + + Args: + repo_dir (Path): Mercurial repository directory + + Returns: + str: Returns the name of the Mercurial repository as indicated in the .hgrc + """ hgrc_cfg = configparser.SafeConfigParser() - hgrc_cfg.read(sps.normExpUserPath(os.path.join(repoDir, ".hg", "hgrc"))) + hgrc_cfg.read(str(repo_dir / ".hg" / "hgrc")) # Not all default entries in [paths] end with "/". return [i for i in hgrc_cfg.get("paths", "default").split("/") if i][-1] -def isRepoValid(repo): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Check that a repository is valid by ensuring that the hgrc file is around.""" - return os.path.isfile(sps.normExpUserPath(os.path.join(repo, ".hg", "hgrc"))) +def patch_hg_repo_with_mq(patch_file, repo_dir=None): + """Use mq to patch the Mercurial repository + Args: + patch_file (Path): Full path to the patch + repo_dir (Path): Working directory path -def patchHgRepoUsingMq(patchFile, workingDir=None): # pylint: disable=invalid-name,missing-docstring,missing-return-doc - # pylint: disable=missing-return-type-doc - workingDir = workingDir or ( + Raises: + OSError: Raises when `hg qimport` or `hg qpush` did not return a return code of 0 + + Returns: + str: Returns the name of the patch file + """ + repo_dir = str(repo_dir) or ( os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member # We may have passed in the patch with or without the full directory. - patch_abs_path = os.path.abspath(sps.normExpUserPath(patchFile)) - pname = os.path.basename(patch_abs_path) - assert pname != "" - qimport_output, qimport_return_code = sps.captureStdout(["hg", "-R", workingDir, "qimport", patch_abs_path], - combineStderr=True, ignoreStderr=True, - ignoreExitCode=True) + patch_abs_path = patch_file.resolve() + pname = patch_abs_path.name + qimport_result = subprocess.run( + ["hg", "-R", str(repo_dir), "qimport", patch_abs_path], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=99) + qimport_output, qimport_return_code = (qimport_result.stdout.decode("utf-8", errors="replace"), + qimport_result.returncode) if qimport_return_code != 0: if "already exists" in qimport_output: print("A patch with the same name has already been qpush'ed. Please qremove it first.") - raise Exception("Return code from `hg qimport` is: " + str(qimport_return_code)) + raise OSError("Return code from `hg qimport` is: " + str(qimport_return_code)) print("Patch qimport'ed...", end=" ") - qpush_output, qpush_return_code = sps.captureStdout(["hg", "-R", workingDir, "qpush", pname], - combineStderr=True, ignoreStderr=True) + qpush_result = subprocess.run( + ["hg", "-R", str(repo_dir), "qpush", pname], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + check=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=99) + qpush_output, qpush_return_code = qpush_result.stdout.decode("utf-8", errors="replace"), qpush_result.returncode assert " is empty" not in qpush_output, "Patch to be qpush'ed should not be empty." if qpush_return_code != 0: - hgQpopQrmAppliedPatch(patchFile, workingDir) + qpop_qrm_applied_patch(patch_file, repo_dir) print("You may have untracked .rej or .orig files in the repository.") - print("`hg status` output of the repository of interesting files in %s :" % workingDir) - subprocess.check_call(["hg", "-R", workingDir, "status", "--modified", "--added", - "--removed", "--deleted"]) - raise Exception("Return code from `hg qpush` is: " + str(qpush_return_code)) + print("`hg status` output of the repository of interesting files in %s :" % repo_dir) + subprocess.run(["hg", "-R", str(repo_dir), "status", "--modified", "--added", + "--removed", "--deleted"], check=True) + raise OSError("Return code from `hg qpush` is: " + str(qpush_return_code)) print("Patch qpush'ed. Continuing...", end=" ") return pname -def hgQpopQrmAppliedPatch(patchFile, repoDir): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc - # pylint: disable=missing-type-doc - """Remove applied patch using `hg qpop` and `hg qdelete`.""" - qpop_output, qpop_return_code = sps.captureStdout(["hg", "-R", repoDir, "qpop"], - combineStderr=True, ignoreStderr=True, - ignoreExitCode=True) +def qpop_qrm_applied_patch(patch_file, repo_dir): + """Remove applied patch using `hg qpop` and `hg qdelete`. + + Args: + patch_file (Path): Full path to the patch + repo_dir (Path): Working directory path + + Raises: + OSError: Raises when `hg qpop` did not return a return code of 0 + """ + qpop_result = subprocess.run( + ["hg", "-R", str(repo_dir), "qpop"], + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=99) + qpop_output, qpop_return_code = qpop_result.stdout.decode("utf-8", errors="replace"), qpop_result.returncode if qpop_return_code != 0: print("`hg qpop` output is: " + qpop_output) - raise Exception("Return code from `hg qpop` is: " + str(qpop_return_code)) + raise OSError("Return code from `hg qpop` is: " + str(qpop_return_code)) print("Patch qpop'ed...", end=" ") - subprocess.check_call(["hg", "-R", repoDir, "qdelete", os.path.basename(patchFile)]) + subprocess.run(["hg", "-R", str(repo_dir), "qdelete", patch_file.name], check=True) print("Patch qdelete'd.") diff --git a/src/funfuzz/util/link_js.py b/src/funfuzz/util/link_js.py deleted file mode 100644 index dfa88c5d0..000000000 --- a/src/funfuzz/util/link_js.py +++ /dev/null @@ -1,39 +0,0 @@ -# coding=utf-8 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -"""Functions to concatenate files, with one specially for js files. -""" - -from __future__ import absolute_import, print_function # isort:skip - -import os - - -def link_js(target_fn, file_list_fn, source_base, prologue="", module_dirs=None): - # pylint: disable=missing-docstring - module_dirs = module_dirs or [] - with open(target_fn, "w") as target: - target.write(prologue) - - # Add files listed in file_list_fn - with open(file_list_fn) as file_list: - for source_fn in file_list: - source_fn = source_fn.replace("/", os.path.sep).strip() - if source_fn and source_fn[0] != "#": - add_contents(os.path.join(source_base, source_fn), target) - - # Add all *.js files in module_dirs - for module_base in module_dirs: - for module_fn in os.listdir(module_base): - if module_fn.endswith(".js"): - add_contents(os.path.join(module_base, module_fn), target) - - -def add_contents(source_fn, target): # pylint: disable=missing-docstring - target.write("\n\n// " + source_fn + "\n\n") - with open(source_fn) as source: - for line in source: - target.write(line) diff --git a/src/funfuzz/util/lithium_helpers.py b/src/funfuzz/util/lithium_helpers.py index c24df6ee6..26d6dc8a2 100644 --- a/src/funfuzz/util/lithium_helpers.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -7,12 +7,12 @@ """Helper functions to use the Lithium reducer. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip +import io import os import re import shutil -import subprocess import sys import tempfile @@ -25,6 +25,14 @@ from ..js.js_interesting import JS_OVERALL_MISMATCH from ..js.js_interesting import JS_VG_AMISS +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + runlithiumpy = [sys.executable, "-u", "-m", "lithium"] # pylint: disable=invalid-name # Status returns for runLithium and many_timed_runs @@ -41,14 +49,14 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename (If it's not prepared to accept engineFlags, engineFlags must be empty.) """ - lithArgs = itest + [jsEngine] + engineFlags + [infilename] # pylint: disable=invalid-name + lithArgs = itest + [str(jsEngine)] + engineFlags + [str(infilename)] # pylint: disable=invalid-name - (lithResult, lithDetails) = strategicReduction( # pylint: disable=invalid-name + (lithResult, lithDetails) = reduction_strat( # pylint: disable=invalid-name logPrefix, infilename, lithArgs, targetTime, suspiciousLevel) print() print("Done running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:") - print(" ".join(quote(x) for x in [sys.executable, "-u", "-m", "lithium", "--strategy=check-only"] + lithArgs)) + print(" ".join(quote(str(x)) for x in [sys.executable, "-u", "-m", "lithium", "--strategy=check-only"] + lithArgs)) print() # pylint: disable=literal-comparison @@ -60,21 +68,22 @@ def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: dis ["-p", " ".join(engineFlags + [infilename])] + ["-i"] + itest ) - print(" ".join(quote(x) for x in autobisectCmd)) - autoBisectLogFilename = logPrefix + "-autobisect.txt" # pylint: disable=invalid-name - subprocess.call(autobisectCmd, stdout=open(autoBisectLogFilename, "w"), stderr=subprocess.STDOUT) - print("Done running autobisectjs. Log: %s" % autoBisectLogFilename) + print(" ".join(quote(str(x)) for x in autobisectCmd)) + autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") + with io.open(str(autobisect_log), "w", encoding="utf-8", errors="replace") as f: + subprocess.run(autobisectCmd, stderr=subprocess.STDOUT, stdout=f) + print("Done running autobisectjs. Log: %s" % autobisect_log) - with open(autoBisectLogFilename, "r") as f: + with io.open(str(autobisect_log), "r", encoding="utf-8", errors="replace") as f: lines = f.readlines() - autoBisectLog = file_manipulation.truncateMid(lines, 50, ["..."]) # pylint: disable=invalid-name + autobisect_log_trunc = file_manipulation.truncateMid(lines, 50, ["..."]) else: - autoBisectLog = [] # pylint: disable=invalid-name + autobisect_log_trunc = [] # pylint: disable=invalid-name - return (lithResult, lithDetails, autoBisectLog) + return (lithResult, lithDetails, autobisect_log_trunc) -def runLithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc +def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level. @@ -88,24 +97,25 @@ def runLithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name lithArgs = ["--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp] + lithArgs else: # loop is being run standalone - lithtmp = logPrefix + "-lith-tmp" - os.mkdir(lithtmp) - lithArgs = ["--tempdir=" + lithtmp] + lithArgs - lithlogfn = logPrefix + "-lith-out.txt" + lithtmp = logPrefix.parent / (logPrefix.stem + "-lith-tmp") + Path.mkdir(lithtmp) + lithArgs = ["--tempdir=" + str(lithtmp)] + lithArgs + lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) - print(" ".join(quote(x) for x in runlithiumpy + lithArgs)) - subprocess.call(runlithiumpy + lithArgs, stdout=open(lithlogfn, "w"), stderr=subprocess.STDOUT) + print(" ".join(quote(str(x)) for x in runlithiumpy + lithArgs)) + with io.open(str(lithlogfn), "w", encoding="utf-8", errors="replace") as f: + subprocess.run(runlithiumpy + lithArgs, stderr=subprocess.STDOUT, stdout=f) print("Done running Lithium") if deletableLithTemp: shutil.rmtree(deletableLithTemp) r = readLithiumResult(lithlogfn) # pylint: disable=invalid-name - subprocess.call(["gzip", "-f", lithlogfn]) + subprocess.run(["gzip", "-f", str(lithlogfn)], check=True) return r def readLithiumResult(lithlogfn): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc - with open(lithlogfn) as f: + with io.open(str(lithlogfn), "r", encoding="utf-8", errors="replace") as f: for line in f: if line.startswith("Lithium result"): print(line.rstrip()) @@ -125,43 +135,51 @@ def readLithiumResult(lithlogfn): # pylint: disable=invalid-name,missing-docstr return (LITH_BUSTED, None) -def strategicReduction(logPrefix, infilename, lithArgs, targetTime, lev): # pylint: disable=invalid-name +def reduction_strat(logPrefix, infilename, lithArgs, targetTime, lev): # pylint: disable=invalid-name # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc,too-complex # pylint: disable=too-many-branches,too-many-locals,too-many-statements """Reduce jsfunfuzz output files using Lithium by using various strategies.""" + # This is an array because Python does not like assigning to upvars. reductionCount = [0] # pylint: disable=invalid-name - backupFilename = infilename + "-backup" # pylint: disable=invalid-name + backup_file = (logPrefix.parent / (logPrefix.stem + "-backup")) - def lithReduceCmd(strategy): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc - """Lithium reduction commands accepting various strategies.""" + """Lithium reduction commands accepting various strategies. + + Args: + strategy (str): Intended strategy to use + + Returns: + (tuple): The finished Lithium run result and details + """ reductionCount[0] += 1 # Remove empty elements - fullLithArgs = [x for x in (strategy + lithArgs) if x] # pylint: disable=invalid-name - print(" ".join(quote(x) for x in [sys.executable, "-u", "-m", "lithium"] + fullLithArgs)) + full_lith_args = [x for x in (strategy + lithArgs) if x] + print(" ".join(quote(str(x)) for x in [sys.executable, "-u", "-m", "lithium"] + full_lith_args)) desc = "-chars" if strategy == "--char" else "-lines" - (lithResult, lithDetails) = runLithium( # pylint: disable=invalid-name - fullLithArgs, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) - if lithResult == LITH_FINISHED: - shutil.copy2(infilename, backupFilename) + (lith_result, lith_details) = run_lithium( # pylint: disable=invalid-name + full_lith_args, (logPrefix.parent / ("%s-%s%s" % (logPrefix.stem, reductionCount[0], desc))), targetTime) + if lith_result == LITH_FINISHED: + shutil.copy2(str(infilename), str(backup_file)) - return lithResult, lithDetails + return lith_result, lith_details print() print("Running the first line reduction...") print() # Step 1: Run the first instance of line reduction. - lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce([]) # pylint: disable=invalid-name - if lithDetails is not None: # lithDetails can be None if testcase no longer becomes interesting - origNumOfLines = int(lithDetails.split()[0]) # pylint: disable=invalid-name + if lith_details is not None: # lith_details can be None if testcase no longer becomes interesting + origNumOfLines = int(lith_details.split()[0]) # pylint: disable=invalid-name hasTryItOut = False # pylint: disable=invalid-name hasTryItOutRegex = re.compile(r'count=[0-9]+; tryItOut\("') # pylint: disable=invalid-name - with open(infilename, "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: for line in file_manipulation.linesWith(f, '; tryItOut("'): # Checks if testcase came from jsfunfuzz or compare_jit. # Do not use .match here, it only matches from the start of the line: @@ -171,29 +189,29 @@ def lithReduceCmd(strategy): # pylint: disable=invalid-name,missing-param-doc,m break # Step 2: Run 1 instance of 1-line reduction after moving tryItOut and count=X around. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: tryItOutAndCountRegex = re.compile(r'"\);\ncount=([0-9]+); tryItOut\("', # pylint: disable=invalid-name re.MULTILINE) - with open(infilename, "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: infileContents = f.read() # pylint: disable=invalid-name infileContents = re.sub(tryItOutAndCountRegex, # pylint: disable=invalid-name ';\\\n"); count=\\1; tryItOut("\\\n', infileContents) - with open(infilename, "w") as f: + with io.open(str(infilename), "w", encoding="utf-8", errors="replace") as f: f.write(infileContents) print() print("Running 1 instance of 1-line reduction after moving tryItOut and count=X...") print() # --chunksize=1: Reduce only individual lines, for only 1 round. - lithResult, lithDetails = lithReduceCmd(["--chunksize=1"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--chunksize=1"]) # pylint: disable=invalid-name # Step 3: Run 1 instance of 2-line reduction after moving count=X to its own line and add a # 1-line offset. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: intendedLines = [] # pylint: disable=invalid-name - with open(infilename, "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: for line in f: # The testcase is likely to already be partially reduced. if "dumpln(cookie" not in line: # jsfunfuzz-specific line ignore # This should be simpler than re.compile. @@ -202,61 +220,61 @@ def lithReduceCmd(strategy): # pylint: disable=invalid-name,missing-param-doc,m # The 1-line offset is added here. .replace("SPLICE DDBEGIN", "SPLICE DDBEGIN\n")) - with open(infilename, "w") as f: + with io.open(str(infilename), "w", encoding="utf-8", errors="replace") as f: f.writelines(intendedLines) print() print("Running 1 instance of 2-line reduction after moving count=X to its own line...") print() - lithResult, lithDetails = lithReduceCmd(["--chunksize=2"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--chunksize=2"]) # pylint: disable=invalid-name # Step 4: Run 1 instance of 2-line reduction again, e.g. to remove pairs of STRICT_MODE lines. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: print() print("Running 1 instance of 2-line reduction again...") print() - lithResult, lithDetails = lithReduceCmd(["--chunksize=2"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--chunksize=2"]) # pylint: disable=invalid-name isLevOverallMismatchAsmJsAvailable = (lev == JS_OVERALL_MISMATCH and # pylint: disable=invalid-name - file_contains_str(infilename, "isAsmJSCompilationAvailable")) + file_contains_str(str(infilename), "isAsmJSCompilationAvailable")) # Step 5 (not always run): Run character reduction within interesting lines. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and targetTime is None and \ + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and targetTime is None and \ lev >= JS_OVERALL_MISMATCH and not isLevOverallMismatchAsmJsAvailable: print() print("Running character reduction...") print() - lithResult, lithDetails = lithReduceCmd(["--char"]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce(["--char"]) # pylint: disable=invalid-name # Step 6: Run line reduction after activating SECOND DDBEGIN with a 1-line offset. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: infileContents = [] # pylint: disable=invalid-name - with open(infilename, "r") as f: + with io.open(str(infilename), "r", encoding="utf-8", errors="replace") as f: for line in f: if "NIGEBDD" in line: infileContents.append(line.replace("NIGEBDD", "DDBEGIN")) infileContents.append("\n") # The 1-line offset is added here. continue infileContents.append(line) - with open(infilename, "w") as f: + with io.open(str(infilename), "w", encoding="utf-8", errors="replace") as f: f.writelines(infileContents) print() print("Running line reduction with a 1-line offset...") print() - lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce([]) # pylint: disable=invalid-name # Step 7: Run line reduction for a final time. - if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: + if lith_result == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: print() print("Running the final line reduction...") print() - lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name + lith_result, lith_details = lith_reduce([]) # pylint: disable=invalid-name # Restore from backup if testcase can no longer be reproduced halfway through reduction. - if lithResult != LITH_FINISHED and lithResult != LITH_PLEASE_CONTINUE: + if lith_result != LITH_FINISHED and lith_result != LITH_PLEASE_CONTINUE: # Probably can move instead of copy the backup, once this has stabilised. - if os.path.isfile(backupFilename): - shutil.copy2(backupFilename, infilename) + if backup_file.is_file(): + shutil.copy2(str(backup_file), str(infilename)) else: - print("DEBUG! backupFilename is supposed to be: %s" % backupFilename) + print("DEBUG! backup_file is supposed to be: %s" % backup_file) - return lithResult, lithDetails + return lith_result, lith_details diff --git a/src/funfuzz/util/lock_dir.py b/src/funfuzz/util/lock_dir.py index edf9e6c39..405e155b4 100644 --- a/src/funfuzz/util/lock_dir.py +++ b/src/funfuzz/util/lock_dir.py @@ -8,18 +8,20 @@ released. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin -import os -class LockDir(object): # pylint: disable=missing-param-doc,missing-type-doc,too-few-public-methods +class LockDir(object): # pylint: disable=too-few-public-methods """Create a filesystem-based lock while in scope. Use: with LockDir(path): # No other code is concurrently using LockDir(path) + + Args: + directory (str): Lock directory name """ def __init__(self, directory): @@ -27,10 +29,10 @@ def __init__(self, directory): def __enter__(self): try: - os.mkdir(self.directory) + self.directory.mkdir() except OSError: - print("Lock file exists: %s" % self.directory) + print("Lock directory exists: %s" % self.directory) raise def __exit__(self, exc_type, exc_val, exc_tb): - os.rmdir(self.directory) + self.directory.rmdir() diff --git a/src/funfuzz/util/os_ops.py b/src/funfuzz/util/os_ops.py new file mode 100644 index 000000000..8d3853ac9 --- /dev/null +++ b/src/funfuzz/util/os_ops.py @@ -0,0 +1,380 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Functions dealing with files and their contents. +""" + +from __future__ import absolute_import, print_function # isort:skip + +import io +import os +import platform +import shutil +import sys +import time + +from pkg_resources import parse_version +from shellescape import quote + +from . import subprocesses as sps + +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + +NO_DUMP_MSG = r""" +WARNING: Minidumps are not being generated, so all crashes will be uninteresting. +WARNING: Make sure the following key value exists in this key: +WARNING: HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps +WARNING: Name: DumpType Type: REG_DWORD +WARNING: http://msdn.microsoft.com/en-us/library/windows/desktop/bb787181%28v=vs.85%29.aspx +""" + + +def make_cdb_cmd(prog_full_path, crashed_pid): + """Construct a command that uses the Windows debugger (cdb.exe) to turn a minidump file into a stack trace. + + Args: + prog_full_path (Path): Full path to the program + crashed_pid (int): PID of the program + + Returns: + list: cdb command list + """ + assert platform.system() == "Windows" + # Look for a minidump. + dump_name = Path.home() / "AppData" / "Local" / "CrashDumps" / "%s.%s.dmp" % (prog_full_path.name, crashed_pid) + + if platform.uname()[2] == "10": # Windows 10 + win64_debugging_folder = Path(os.getenv("PROGRAMFILES(X86)")) / "Windows Kits" / "10" / "Debuggers" / "x64" + else: + win64_debugging_folder = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" + + # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. + cdb_path = win64_debugging_folder / "cdb.exe" + if cdb_path.is_file(): # pylint: disable=no-member + print() + print("WARNING: cdb.exe is not found - all crashes will be interesting.") + print() + return [] + + if is_win_dumping_to_default(): + loops = 0 + max_loops = 300 + while True: + if dump_name.is_file(): + dbggr_cmd_path = Path(__file__).parent / "cdb_cmds.txt" + assert dbggr_cmd_path.is_file() # pylint: disable=no-member + + cdb_cmd_list = [] + cdb_cmd_list.append("$<" + str(dbggr_cmd_path)) + + # See bug 902706 about -g. + return [cdb_path, "-g", "-c", ";".join(cdb_cmd_list), "-z", str(dump_name)] + + time.sleep(0.200) + loops += 1 + if loops > max_loops: + # Windows may take some time to generate the dump. + print("make_cdb_cmd waited a long time, but %s never appeared!" % str(dump_name)) + return [] + else: + return [] + + +def make_gdb_cmd(prog_full_path, crashed_pid): + """Construct a command that uses the POSIX debugger (gdb) to turn a minidump file into a stack trace. + + Args: + prog_full_path (Path): Full path to the program + crashed_pid (int): PID of the program + + Returns: + list: gdb command list + """ + assert os.name == "posix" + # On Mac and Linux, look for a core file. + core_name = None + if platform.system() == "Darwin": + # Core files will be generated if you do: + # mkdir -p /cores/ + # ulimit -c 2147483648 (or call resource.setrlimit from a preexec_fn hook) + core_name = "/cores/core." + str(crashed_pid) + elif platform.system() == "Linux": + is_pid_used = False + core_uses_pid_path = Path("/proc/sys/kernel/core_uses_pid") + if core_uses_pid_path.is_file(): + with io.open(str(core_uses_pid_path), "r", encoding="utf-8", errors="replace") as f: + is_pid_used = bool(int(f.read()[0])) # Setting [0] turns the input to a str. + core_name = "core." + str(crashed_pid) if is_pid_used else "core" + core_name_path = Path.cwd() / core_name + if not core_name_path.is_file(): + core_name_path = Path.home() / core_name # try the home dir + + if core_name and core_name_path.is_file(): + dbggr_cmd_path = Path(__file__).parent / "gdb_cmds.txt" + assert dbggr_cmd_path.is_file() # pylint: disable=no-member + + # Run gdb and move the core file. Tip: gdb gives more info for: + # (debug with intact build dir > debug > opt with frame pointers > opt) + return ["gdb", "-n", "-batch", "-x", str(dbggr_cmd_path), str(prog_full_path), str(core_name)] + return [] + + +def disable_corefile(): + """When called as a preexec_fn, sets appropriate resource limits for the JS shell. Must only be called on POSIX.""" + import resource # module only available on POSIX pylint: disable=import-error + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # pylint: disable=no-member + + +def get_core_limit(): + """Returns the maximum core file size that the current process can create. + + Returns: + int: Maximum size (in bytes) of a core file that the current process can create. + """ + import resource # module only available on POSIX pylint: disable=import-error + return resource.getrlimit(resource.RLIMIT_CORE) # pylint: disable=no-member + + +# pylint: disable=inconsistent-return-statements +def grab_crash_log(prog_full_path, crashed_pid, log_prefix, want_stack): + # pylint: disable=too-complex,too-many-branches + """Return the crash log if found. + + Args: + prog_full_path (Path): Full path to the program + crashed_pid (int): PID of the crashed program + log_prefix (str): Log prefix + want_stack (bool): Boolean on whether a stack is desired + + Returns: + str: Returns the path to the crash log + """ + progname = prog_full_path.name + + use_logfiles = isinstance(log_prefix, ("".__class__, u"".__class__)) + crash_log = (log_prefix.parent / (log_prefix.stem + "-crash")).with_suffix(".txt") + core_file = (log_prefix.parent / (log_prefix.stem + "-core")) + + if use_logfiles: + if crash_log.is_file(): + crash_log.unlink() + if core_file.is_file(): + core_file.unlink() + + if not want_stack or progname == "valgrind": + return "" + + # This has only been tested on 64-bit Windows 7 and higher + if platform.system() == "Windows": + dbggr_cmd = make_cdb_cmd(prog_full_path, crashed_pid) + elif os.name == "posix": + dbggr_cmd = make_gdb_cmd(prog_full_path, crashed_pid) + else: + dbggr_cmd = None + + if dbggr_cmd: + sps.vdump(" ".join(dbggr_cmd)) + core_file = Path(dbggr_cmd[-1]) + assert core_file.is_file() + dbbgr_exit_code = subprocess.run( + dbggr_cmd, + stdin=None, + stderr=subprocess.STDOUT, + stdout=io.open(str(crash_log), "w", encoding="utf-8", errors="replace") if use_logfiles else None, + # It would be nice to use this everywhere, but it seems to be broken on Windows + # (http://docs.python.org/library/subprocess.html) + close_fds=(os.name == "posix"), + # Do not generate a core_file if gdb crashes in Linux + preexec_fn=(disable_corefile if platform.system() == "Linux" else None) + ) + if dbbgr_exit_code != 0: + print("Debugger exited with code %d : %s" % (dbbgr_exit_code, " ".join(quote(str(x)) for x in dbggr_cmd))) + if use_logfiles: + if core_file.is_file(): + shutil.move(str(core_file), str(core_file)) + subprocess.run(["gzip", "-f", str(core_file)], check=True) + # chmod here, else the uploaded -core.gz files do not have sufficient permissions. + subprocess.run(["chmod", "og+r", "%s.gz" % core_file], check=True) + return str(crash_log) + else: + print("I don't know what to do with a core file when log_prefix is null") + + # On Mac, look for a crash log generated by Mac OS X Crash Reporter + elif platform.system() == "Darwin": + loops = 0 + max_loops = 500 if progname.startswith("firefox") else 450 + while True: + crash_log_found = grab_mac_crash_log(crashed_pid, log_prefix, use_logfiles) + if crash_log_found is not None: + return crash_log_found + + # print "[grab_crash_log] Waiting for the crash log to appear..." + time.sleep(0.200) + loops += 1 + if loops > max_loops: + # I suppose this might happen if the process corrupts itself so much that + # the crash reporter gets confused about the process name, for example. + print("grab_crash_log waited a long time, but a crash log for %s [%s] never appeared!" % ( + progname, crashed_pid)) + break + + elif platform.system() == "Linux": + print("Warning: grab_crash_log() did not find a core file for PID %d." % crashed_pid) + print("Note: Your soft limit for core file sizes is currently %d. " + # pylint: disable=indexing-exception + 'You can increase it with "ulimit -c" in bash.' % get_core_limit()[0]) + + +def grab_mac_crash_log(crash_pid, log_prefix, use_log_files): + """Find the required crash log in the given crash reporter directory. + + Args: + crash_pid (str): PID value of the crashed process + log_prefix (str): Prefix (may include dirs) of the log file + use_log_files (bool): Boolean that decides whether *-crash.txt log files should be used + + Returns: + str: Absolute (if use_log_files is False) or relative (if use_log_files is True) path to crash log file + """ + assert parse_version(platform.mac_ver()[0]) >= parse_version("10.6") + + for base_dir in [Path.home(), Path("/")]: + # Sometimes the crash reports end up in the root directory. + # This possibly happens when the value of : + # defaults write com.apple.CrashReporter DialogType + # is none, instead of server, or some other option. + # It also happens when ssh'd into a computer. + # And maybe when the computer is under heavy load. + # See http://en.wikipedia.org/wiki/Crash_Reporter_%28Mac_OS_X%29 + reports_dir = base_dir / "Library" / "Logs" / "DiagnosticReports" + # Find a crash log for the right process name and pid, preferring + # newer crash logs (which sort last). + if reports_dir.is_dir(): # pylint: disable=no-member + crash_logs = [x for x in reports_dir.iterdir()].sort(reverse=True) # pylint: disable=no-member + else: + crash_logs = [] + + for file_name in crash_logs: + full_report_path = reports_dir / file_name + try: + with io.open(str(full_report_path), "r", encoding="utf-8", errors="replace") as f: + first_line = f.readline() + if first_line.rstrip().endswith("[%s]" % crash_pid): + if use_log_files: + # Copy, don't rename, because we might not have permissions + # (especially for the system rather than user crash log directory) + # Use copyfile, as we do not want to copy the permissions metadata over + crash_log = (log_prefix.parent / (log_prefix.stem + "-crash")).with_suffix(".txt") + shutil.copyfile(str(full_report_path), str(crash_log)) + subprocess.run(["chmod", "og+r", str(crash_log)], + # pylint: disable=no-member + cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), + check=True, + timeout=9) + return str(crash_log) + return str(full_report_path) + + except OSError: + # Maybe the log was rotated out between when we got the list + # of files and when we tried to open this file. If so, it's + # clearly not The One. + pass + return None + + +def is_win_dumping_to_default(): # pylint: disable=too-complex,too-many-branches + """Check whether Windows minidumps are enabled and set to go to Windows' default location. + + Raises: + OSError: Raises if querying for the DumpType key throws and it is unrelated to various issues, + e.g. the key not being present. + + Returns: + bool: Returns True when Windows has dumping enabled, and is dumping to the default location, otherwise False + """ + if sys.version_info.major == 2: + import _winreg as winreg # pylint: disable=import-error + else: + import winreg # pylint: disable=import-error + # For now, this code does not edit the Windows Registry because we tend to be in a 32-bit + # version of Python and if one types in regedit in the Run dialog, opens up the 64-bit registry. + # If writing a key, we most likely need to flush. For the moment, no keys are written. + try: + with winreg.OpenKey(winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE), + r"Software\Microsoft\Windows\Windows Error Reporting\LocalDumps", + # Read key from 64-bit registry, which also works for 32-bit + 0, (winreg.KEY_WOW64_64KEY + winreg.KEY_READ)) as key: + + try: + dump_type_reg_value = winreg.QueryValueEx(key, "DumpType") + if not (dump_type_reg_value[0] == 1 and dump_type_reg_value[1] == winreg.REG_DWORD): + print(NO_DUMP_MSG) + return False + except OSError as ex: + if ex.errno == 2: + print(NO_DUMP_MSG) + return False + else: + raise + + try: + dump_folder_reg_value = winreg.QueryValueEx(key, "DumpFolder") + # %LOCALAPPDATA%\CrashDumps is the default location. + if not (dump_folder_reg_value[0] == r"%LOCALAPPDATA%\CrashDumps" and + dump_folder_reg_value[1] == winreg.REG_EXPAND_SZ): + print() + print("WARNING: Dumps are instead appearing at: %s - " + "all crashes will be uninteresting." % dump_folder_reg_value[0]) + print() + return False + except OSError as ex: + # If the key value cannot be found, the dumps will be put in the default location + if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": + return True + else: + raise + + return True + except OSError as ex: + # If the LocalDumps registry key cannot be found, dumps will be put in the default location. + if ex.errno == 2 and ex.strerror == "The system cannot find the file specified": + print() + print("WARNING: The registry key HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\" + "Windows\\Windows Error Reporting\\LocalDumps cannot be found.") + print() + return False + else: + raise + + +def make_wtmp_dir(base_dir): + """Create wtmp directory, incrementing the number if one is already found. + + Args: + base_dir (Path): Base directory to create the wtmp directories + + Returns: + Path: Full path to the numbered wtmp directory + """ + assert isinstance(base_dir, Path) + + i = 1 + while True: + numbered_tmp_dir = "wtmp%s" % i + full_dir = base_dir / numbered_tmp_dir + try: + full_dir.mkdir() # To avoid race conditions, we use try/except instead of exists/create + break # break out of the while loop + except OSError: + i += 1 + + return full_dir diff --git a/src/funfuzz/util/repos_update.py b/src/funfuzz/util/repos_update.py index 4d71243e4..15090407c 100644 --- a/src/funfuzz/util/repos_update.py +++ b/src/funfuzz/util/repos_update.py @@ -10,7 +10,7 @@ Assumes that the repositories are located in ../../trees/*. """ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # isort:skip from copy import deepcopy import logging @@ -19,12 +19,12 @@ import sys import time -from . import subprocesses as sps - if sys.version_info.major == 2: + from pathlib2 import Path if os.name == "posix": import subprocess32 as subprocess # pylint: disable=import-error else: + from pathlib import Path # pylint: disable=import-error import subprocess @@ -37,16 +37,16 @@ if platform.system() == "Windows": # pylint: disable=invalid-name - git_64bit_path = os.path.normpath(os.path.join(os.getenv("PROGRAMFILES"), "Git", "bin", "git.exe")) - git_32bit_path = os.path.normpath(os.path.join(os.getenv("PROGRAMFILES(X86)"), "Git", "bin", "git.exe")) - if os.path.isfile(git_64bit_path): - GITBINARY = git_64bit_path - elif os.path.isfile(git_32bit_path): - GITBINARY = git_32bit_path + git_64bit_path = Path(os.getenv("PROGRAMFILES")) / "Git" / "bin" / "git.exe" + git_32bit_path = Path(os.getenv("PROGRAMFILES(X86)")) / "Git" / "bin" / "git.exe" + if git_64bit_path.is_file(): # pylint: disable=no-member + GITBINARY = str(git_64bit_path) + elif git_32bit_path.is_file(): # pylint: disable=no-member + GITBINARY = str(git_32bit_path) else: raise OSError("Git binary not found") else: - GITBINARY = "git" + GITBINARY = str("git") def time_cmd(cmd, cwd=None, env=None, timeout=None): @@ -77,7 +77,7 @@ def typeOfRepo(r): # pylint: disable=invalid-name,missing-param-doc,missing-rai repo_types.append(".hg") repo_types.append(".git") for rtype in repo_types: - if os.path.isdir(os.path.join(r, rtype)): + if (r / rtype).is_dir(): return rtype[1:] raise Exception("Type of repository located at " + r + " cannot be determined.") @@ -85,20 +85,20 @@ def typeOfRepo(r): # pylint: disable=invalid-name,missing-param-doc,missing-rai def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Update a repository. Return False if missing; return True if successful; raise an exception if updating fails.""" - assert os.path.isdir(repo) + repo.is_dir() repo_type = typeOfRepo(repo) if repo_type == "hg": hg_pull_cmd = ["hg", "--time", "pull", "-u"] logger.info("\nRunning `%s` now..\n", " ".join(hg_pull_cmd)) - out_hg_pull = subprocess.run(hg_pull_cmd, check=True, cwd=repo, stderr=subprocess.PIPE) + out_hg_pull = subprocess.run(hg_pull_cmd, check=True, cwd=str(repo), stderr=subprocess.PIPE) logger.info('"%s" had the above output and took - %s', subprocess.list2cmdline(out_hg_pull.args), out_hg_pull.stderr) hg_log_default_cmd = ["hg", "--time", "log", "-r", "default"] logger.info("\nRunning `%s` now..\n", " ".join(hg_log_default_cmd)) - out_hg_log_default = subprocess.run(hg_log_default_cmd, check=True, cwd=repo, + out_hg_log_default = subprocess.run(hg_log_default_cmd, check=True, cwd=str(repo), stderr=subprocess.PIPE) logger.info('"%s" had the above output and took - %s', subprocess.list2cmdline(out_hg_log_default.args), @@ -108,7 +108,7 @@ def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing- gitenv = deepcopy(os.environ) if platform.system() == "Windows": gitenv["GIT_SSH_COMMAND"] = "~/../../mozilla-build/msys/bin/ssh.exe -F ~/.ssh/config" - time_cmd([GITBINARY, "pull"], cwd=repo, env=gitenv) + time_cmd([GITBINARY, "pull"], cwd=str(repo), env=gitenv) else: raise Exception("Unknown repository type: " + repo_type) @@ -117,15 +117,15 @@ def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing- def updateRepos(): # pylint: disable=invalid-name """Update Mercurial and Git repositories located in ~ and ~/trees .""" - home_dir = sps.normExpUserPath("~") + home_dir = Path.home() trees = [ - os.path.normpath(os.path.join(home_dir)), - os.path.normpath(os.path.join(home_dir, "trees")) + home_dir, + home_dir / "trees" ] for tree in trees: - for name in sorted(os.listdir(tree)): - name_path = os.path.join(tree, name) - if os.path.isdir(name_path) and (name in REPOS or (name.startswith("funfuzz") and "-" in name)): + for name in sorted(os.listdir(str(tree))): + name_path = Path(tree) / name + if name_path.is_dir() and (name in REPOS or (name.startswith("funfuzz") and "-" in name)): logger.info("Updating %s ...", name) updateRepo(name_path) diff --git a/src/funfuzz/util/s3cache.py b/src/funfuzz/util/s3cache.py index ce3d12124..bc7c1e25a 100644 --- a/src/funfuzz/util/s3cache.py +++ b/src/funfuzz/util/s3cache.py @@ -7,7 +7,7 @@ """Functions here interact with Amazon EC2 using boto. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip from builtins import object # pylint: disable=redefined-builtin import os diff --git a/src/funfuzz/util/sm_compile_helpers.py b/src/funfuzz/util/sm_compile_helpers.py new file mode 100644 index 000000000..06fbeef9e --- /dev/null +++ b/src/funfuzz/util/sm_compile_helpers.py @@ -0,0 +1,183 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Helper functions to compile SpiderMonkey shells. +""" + +from __future__ import absolute_import, print_function, unicode_literals # isort:skip + +import io +import os +import platform +import sys +import traceback + +from shellescape import quote +from whichcraft import which # Once we are fully on Python 3.5+, whichcraft can be removed in favour of shutil.which + +if sys.version_info.major == 2: + if os.name == "posix": + import subprocess32 as subprocess # pylint: disable=import-error + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import subprocess + + +def ensure_cache_dir(base_dir): + """Retrieve a cache directory for compiled shells to live in, and create one if needed. + + Args: + base_dir (Path): Base directory to create the cache directory in + + Returns: + Path: Returns the full shell-cache path + """ + if not base_dir: + base_dir = Path.home() + cache_dir = base_dir / "shell-cache" + cache_dir.mkdir(exist_ok=True) + return cache_dir + + +def autoconf_run(working_dir): + """Run autoconf binaries corresponding to the platform. + + Args: + working_dir (Path): Directory to be set as the current working directory + """ + if platform.system() == "Darwin": + autoconf213_mac_bin = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" if which("brew") else "autoconf213" + # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 + if not Path(which(autoconf213_mac_bin)).is_file(): + autoconf213_mac_bin = "autoconf213" + subprocess.run([autoconf213_mac_bin], check=True, cwd=str(working_dir)) + elif platform.system() == "Linux": + if which("autoconf2.13"): + subprocess.run(["autoconf2.13"], check=True, cwd=str(working_dir)) + elif which("autoconf-2.13"): + subprocess.run(["autoconf-2.13"], check=True, cwd=str(working_dir)) + elif which("autoconf213"): + subprocess.run(["autoconf213"], check=True, cwd=str(working_dir)) + elif platform.system() == "Windows": + # Windows needs to call sh to be able to find autoconf. + subprocess.run(["sh", "autoconf-2.13"], check=True, cwd=str(working_dir)) + + +def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Create a .busted file with the exception message and backtrace included.""" + with io.open(str(filename), "w", encoding="utf-8", errors="replace") as f: + f.write("Caught exception %r (%s)\n" % (e, e)) + f.write("Backtrace:\n") + f.write(traceback.format_exc() + "\n") + + print("Compilation failed (%s) (details in %s)" % (e, filename)) + + +def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Dump environment to a .fuzzmanagerconf file.""" + # Platform and OS detection for the spec, part of which is in: + # https://wiki.mozilla.org/Security/CrashSignatures + fmconf_platform = "x86" if shell.build_opts.enable32 else "x86-64" + + if platform.system() == "Linux": + fmconf_os = "linux" + elif platform.system() == "Darwin": + fmconf_os = "macosx" + elif platform.system() == "Windows": + fmconf_os = "windows" + + with io.open(str(log), "a", encoding="utf-8", errors="replace") as f: + f.write("# Information about shell:\n# \n") + + f.write("# Create another shell in shell-cache like this one:\n") + f.write('# python -u -m %s -b "%s" -r %s\n# \n' % ("funfuzz.js.compile_shell", + shell.build_opts.build_options_str, shell.get_hg_hash())) + + f.write("# Full environment is:\n") + f.write("# %s\n# \n" % str(shell.get_env_full())) + + f.write("# Full configuration command with needed environment variables is:\n") + f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.get_env_added()), + " ".join(quote(str(x)) for x in shell.get_cfg_cmd_excl_env()))) + + # .fuzzmanagerconf details + f.write("\n") + f.write("[Main]\n") + f.write("platform = %s\n" % fmconf_platform) + f.write("product = %s\n" % shell.get_repo_name()) + f.write("product_version = %s\n" % shell.get_hg_hash()) + f.write("os = %s\n" % fmconf_os) + + f.write("\n") + f.write("[Metadata]\n") + f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) + f.write("majorVersion = %s\n" % shell.get_version().split(".")[0]) + f.write("pathPrefix = %s/\n" % shell.get_repo_dir()) + f.write("version = %s\n" % shell.get_version()) + + +def extract_vers(objdir): # pylint: disable=inconsistent-return-statements + """Extract the version from js.pc and put it into *.fuzzmanagerconf. + + Args: + objdir (Path): Full path to the objdir + + Raises: + OSError: Raises when js.pc is not found + + Returns: + str: Version number of the compiled js shell + """ + jspc_file_path = objdir / "js" / "src" / "js.pc" + # Moved to /js/src/build/, see bug 1262241, Fx55 m-c rev 351194:2159959522f4 + jspc_new_file_path = objdir / "js" / "src" / "build" / "js.pc" + + if jspc_file_path.is_file(): + actual_path = jspc_file_path + elif jspc_new_file_path.is_file(): + actual_path = jspc_new_file_path + else: + raise OSError("js.pc file not found - needed to extract the version number") + + with io.open(str(actual_path), mode="r", encoding="utf-8", errors="replace") as f: + for line in f: + if line.startswith("Version: "): # Sample line: "Version: 47.0a2" + return line.split(": ")[1].rstrip() + + +def get_lock_dir_path(cache_dir_base, repo_dir, tbox_id=""): + """Return the name of the lock directory. + + Args: + cache_dir_base (Path): Base directory where the cache directory is located + repo_dir (Path): Full path to the repository + tbox_id (str): Tinderbox entry id + + Returns: + Path: Full path to the shell cache lock directory + """ + lockdir_name = "shell-%s-lock" % repo_dir.name + if tbox_id: + lockdir_name += "-%s" % tbox_id + return ensure_cache_dir(cache_dir_base) / lockdir_name + + +def verify_full_win_pageheap(shell_path): + """Turn on full page heap verification on Windows. + + Args: + shell_path (Path): Path to the compiled js shell + """ + # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx + # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ + gflags_bin_path = Path(os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" / "gflags.exe" + if gflags_bin_path.is_file() and shell_path.is_file(): # pylint: disable=no-member + print(subprocess.run([str(gflags_bin_path).decode("utf-8", errors="replace"), + "-p", "/enable", str(shell_path).decode("utf-8", errors="replace"), "/full"], + check=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE).stdout) diff --git a/src/funfuzz/util/subprocesses.py b/src/funfuzz/util/subprocesses.py index ec04ac337..48e938d8f 100644 --- a/src/funfuzz/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -7,441 +7,63 @@ """Miscellaneous helper functions. """ -from __future__ import absolute_import, print_function # isort:skip +from __future__ import absolute_import, print_function, unicode_literals # isort:skip import errno import os import platform import shutil import stat -import subprocess -import sys -import time - -from pkg_resources import parse_version -from shellescape import quote verbose = False # pylint: disable=invalid-name -# pylint: disable=invalid-name -noMinidumpMsg = r""" -WARNING: Minidumps are not being generated, so all crashes will be uninteresting. -WARNING: Make sure the following key value exists in this key: -WARNING: HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps -WARNING: Name: DumpType Type: REG_DWORD -WARNING: http://msdn.microsoft.com/en-us/library/windows/desktop/bb787181%28v=vs.85%29.aspx -""" - - -# pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc -# pylint: disable=missing-type-doc,too-complex,too-many-arguments,too-many-branches,too-many-statements -def captureStdout(inputCmd, ignoreStderr=False, combineStderr=False, ignoreExitCode=False, currWorkingDir=None, - env="NOTSET", verbosity=False): - """Capture standard output, return the output as a string, along with the return value.""" - currWorkingDir = currWorkingDir or ( - os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member - if env == "NOTSET": - vdump(" ".join(quote(x) for x in inputCmd)) - env = os.environ - else: - # There is no way yet to only print the environment variables that were added by the harness - # We could dump all of os.environ but it is too much verbose output. - vdump("ENV_VARIABLES_WERE_ADDED_HERE " + " ".join(quote(x) for x in inputCmd)) - cmd = [] - for el in inputCmd: - if el.startswith('"') and el.endswith('"'): - cmd.append(str(el[1:-1])) - else: - cmd.append(str(el)) - assert cmd != [] - try: - p = subprocess.Popen( - cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT if combineStderr else subprocess.PIPE, - cwd=currWorkingDir, - env=env) - (stdout, stderr) = p.communicate() - except OSError as e: - raise Exception(repr(e.strerror) + " error calling: " + " ".join(quote(x) for x in cmd)) - if p.returncode != 0: - oomErrorOutput = stdout if combineStderr else stderr - if (platform.system() == "Linux" or platform.system() == "Darwin") and oomErrorOutput: - if "internal compiler error: Killed (program cc1plus)" in oomErrorOutput: - raise Exception("GCC running out of memory") - elif "error: unable to execute command: Killed" in oomErrorOutput: - raise Exception("Clang running out of memory") - if not ignoreExitCode: - # Potential problem area: Note that having a non-zero exit code does not mean that the - # operation did not succeed, for example when compiling a shell. A non-zero exit code - # can appear even though a shell compiled successfully. - # Pymake in builds earlier than revision 232553f741a0 did not support the "-s" option. - if "no such option: -s" not in stdout: - print("Nonzero exit code from: ") - print(" %s" % " ".join(quote(x) for x in cmd)) - print("stdout is:") - print(stdout) - if stderr is not None: - print("stderr is:") - print(stderr) - # Pymake in builds earlier than revision 232553f741a0 did not support the "-s" option. - if "hg pull: option --rebase not recognized" not in stdout and "no such option: -s" not in stdout: - if platform.system() == "Windows" and stderr and "Permission denied" in stderr and \ - "configure: error: installation or configuration problem: " + \ - "C++ compiler cannot create executables." in stderr: - raise Exception("Windows conftest.exe configuration permission problem") - else: - raise Exception("Nonzero exit code") - if not combineStderr and not ignoreStderr and stderr: - # Ignore hg color mode throwing an error in console on Windows platforms. - if not (platform.system() == "Windows" and "warning: failed to set color mode to win32" in stderr): - print("Unexpected output on stderr from: ") - print(" %s" % " ".join(quote(x) for x in cmd)) - print("%s %s" % (stdout, stderr)) - raise Exception("Unexpected output on stderr") - if stderr and ignoreStderr and stderr and p.returncode != 0: - # During configure, there will always be stderr. Sometimes this stderr causes configure to - # stop the entire script, especially on Windows. - print("Return code not zero, and unexpected output on stderr from: ") - print(" %s" % " ".join(quote(x) for x in cmd)) - print("%s %s" % (stdout, stderr)) - raise Exception("Return code not zero, and unexpected output on stderr") - if verbose or verbosity: - print(stdout) - if stderr is not None: - print(stderr) - return stdout.rstrip(), p.returncode - - -def createWtmpDir(tmpDirBase): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Create wtmp directory, incrementing the number if one is already found.""" - i = 1 - while True: - tmpDirWithNum = "wtmp" + str(i) - tmpDir = os.path.join(tmpDirBase, tmpDirWithNum) - try: - os.mkdir(tmpDir) # To avoid race conditions, we use try/except instead of exists/create - break - except OSError: - i += 1 - vdump(tmpDirWithNum + os.sep) # Even if not verbose, wtmp is also dumped: wtmp1/w1: NORMAL - return tmpDirWithNum - - -def disableCorefile(): - """When called as a preexec_fn, sets appropriate resource limits for the JS shell. Must only be called on POSIX.""" - import resource # module only available on POSIX pylint: disable=import-error - resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) - - -def getCoreLimit(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - import resource # module only available on POSIX pylint: disable=import-error - return resource.getrlimit(resource.RLIMIT_CORE) +def rm_tree_incl_readonly(dir_tree): + """Remove a directory tree including all read-only files. -def grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles): # pylint: disable=invalid-name,missing-param-doc - # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc - """Find the required crash log in the given crash reporter directory.""" - assert parse_version(platform.mac_ver()[0]) >= parse_version("10.6") - reportDirList = [os.path.expanduser("~"), "/"] - for baseDir in reportDirList: - # Sometimes the crash reports end up in the root directory. - # This possibly happens when the value of : - # defaults write com.apple.CrashReporter DialogType - # is none, instead of server, or some other option. - # It also happens when ssh'd into a computer. - # And maybe when the computer is under heavy load. - # See http://en.wikipedia.org/wiki/Crash_Reporter_%28Mac_OS_X%29 - reportDir = os.path.join(baseDir, "Library/Logs/DiagnosticReports/") - # Find a crash log for the right process name and pid, preferring - # newer crash logs (which sort last). - if os.path.exists(reportDir): - crashLogs = os.listdir(reportDir) - else: - crashLogs = [] - # Firefox sometimes still runs as firefox-bin, at least on Mac (likely bug 658850) - crashLogs = [x for x in crashLogs - if x.startswith(progname + "_") or x.startswith(progname + "-bin_")] - crashLogs.sort(reverse=True) - for fn in crashLogs: - fullfn = os.path.join(reportDir, fn) - try: - with open(fullfn) as c: - firstLine = c.readline() - if firstLine.rstrip().endswith("[" + str(crashedPID) + "]"): - if useLogFiles: - # Copy, don't rename, because we might not have permissions - # (especially for the system rather than user crash log directory) - # Use copyfile, as we do not want to copy the permissions metadata over - shutil.copyfile(fullfn, logPrefix + "-crash.txt") - captureStdout(["chmod", "og+r", logPrefix + "-crash.txt"]) - return logPrefix + "-crash.txt" - return fullfn - # return open(fullfn).read() + Args: + dir_tree (Path): Directory tree of files to be removed + """ + shutil.rmtree(str(dir_tree), onerror=handle_rm_readonly if platform.system() == "Windows" else None) - except (OSError, IOError): # pylint: disable=overlapping-except - # Maybe the log was rotated out between when we got the list - # of files and when we tried to open this file. If so, it's - # clearly not The One. - pass - return None +# This test needs updates for the move to pathlib, and needs to move to pytest +# def test_rm_tree_incl_readonly(): # pylint: disable=invalid-name +# """Run this function in the same directory as subprocesses to test.""" +# test_dir = "test_rm_tree_incl_readonly" +# os.mkdir(test_dir) +# read_only_dir = os.path.join(test_dir, "nestedReadOnlyDir") +# os.mkdir(read_only_dir) +# filename = os.path.join(read_only_dir, "test.txt") +# with io.open(filename, "w", encoding="utf-8", errors="replace") as f: +# f.write("testing\n") -def grabCrashLog(progfullname, crashedPID, logPrefix, wantStack): # pylint: disable=inconsistent-return-statements - # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc - # pylint: disable=too-complex,too-many-branches - """Return the crash log if found.""" - progname = os.path.basename(progfullname) +# os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) +# os.chmod(read_only_dir, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - useLogFiles = isinstance(logPrefix, ("".__class__, u"".__class__)) - if useLogFiles: - if os.path.exists(logPrefix + "-crash.txt"): - os.remove(logPrefix + "-crash.txt") - if os.path.exists(logPrefix + "-core"): - os.remove(logPrefix + "-core") +# rm_tree_incl_readonly(test_dir) # Should pass here - if not wantStack or progname == "valgrind": - return - # This has only been tested on 64-bit Windows 7 and higher - if platform.system() == "Windows": - debuggerCmd = constructCdbCommand(progfullname, crashedPID) - elif os.name == "posix": - debuggerCmd = constructGdbCommand(progfullname, crashedPID) - else: - debuggerCmd = None - - if debuggerCmd: - vdump(" ".join(debuggerCmd)) - coreFile = debuggerCmd[-1] - assert os.path.isfile(coreFile) - debuggerExitCode = subprocess.call( - debuggerCmd, - stdin=None, - stderr=subprocess.STDOUT, - stdout=open(logPrefix + "-crash.txt", "w") if useLogFiles else None, - # It would be nice to use this everywhere, but it seems to be broken on Windows - # (http://docs.python.org/library/subprocess.html) - close_fds=(os.name == "posix"), - # Do not generate a corefile if gdb crashes in Linux - preexec_fn=(disableCorefile if platform.system() == "Linux" else None) - ) - if debuggerExitCode != 0: - print("Debugger exited with code %d : %s" % (debuggerExitCode, " ".join(quote(x) for x in debuggerCmd))) - if useLogFiles: - if os.path.isfile(coreFile): - shutil.move(coreFile, logPrefix + "-core") - subprocess.call(["gzip", "-f", logPrefix + "-core"]) - # chmod here, else the uploaded -core.gz files do not have sufficient permissions. - subprocess.check_call(["chmod", "og+r", logPrefix + "-core.gz"]) - return logPrefix + "-crash.txt" - else: - print("I don't know what to do with a core file when logPrefix is null") - - # On Mac, look for a crash log generated by Mac OS X Crash Reporter - elif platform.system() == "Darwin": - loops = 0 - maxLoops = 500 if progname.startswith("firefox") else 450 - while True: - cLogFound = grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles) - if cLogFound is not None: - return cLogFound +def handle_rm_readonly(func, path, exc): + """Handle read-only files on Windows. Adapted from http://stackoverflow.com/q/1213706 and some docs adapted from + Python 2.7 official docs. - # print "[grabCrashLog] Waiting for the crash log to appear..." - time.sleep(0.200) - loops += 1 - if loops > maxLoops: - # I suppose this might happen if the process corrupts itself so much that - # the crash reporter gets confused about the process name, for example. - print("grabCrashLog waited a long time, but a crash log for %s [%s] never appeared!" % ( - progname, crashedPID)) - break + Args: + func (function): Function which raised the exception + path (str): Path name passed to function + exc (exception): Exception information returned by sys.exc_info() - elif platform.system() == "Linux": - print("Warning: grabCrashLog() did not find a core file for PID %d." % crashedPID) - print("Note: Your soft limit for core file sizes is currently %d. " - 'You can increase it with "ulimit -c" in bash.' % getCoreLimit()[0]) - - -def constructCdbCommand(progfullname, crashedPID): # pylint: disable=invalid-name - # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc - """Construct a command that uses the Windows debugger (cdb.exe) to turn a minidump file into a stack trace.""" + Raises: + OSError: Raised if the read-only files are unable to be handled + """ assert platform.system() == "Windows" - # Look for a minidump. - dumpFilename = normExpUserPath(os.path.join( - "~", "AppData", "Local", "CrashDumps", os.path.basename(progfullname) + "." + str(crashedPID) + ".dmp")) - if platform.uname()[2] == "10": # Windows 10 - win64bitDebuggerFolder = os.path.join(os.getenv("PROGRAMFILES(X86)"), "Windows Kits", "10", "Debuggers", "x64") - else: - win64bitDebuggerFolder = os.path.join(os.getenv("PROGRAMW6432"), "Debugging Tools for Windows (x64)") - # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. - cdbPath = os.path.join(win64bitDebuggerFolder, "cdb.exe") - if not os.path.exists(cdbPath): - print() - print("WARNING: cdb.exe is not found - all crashes will be interesting.") - print() - return None - - if isWinDumpingToDefaultLocation(): - loops = 0 - maxLoops = 300 - while True: - if os.path.exists(dumpFilename): - debuggerCmdPath = getAbsPathForAdjacentFile("cdb_cmds.txt") - assert os.path.exists(debuggerCmdPath) - - cdbCmdList = [] - cdbCmdList.append("$<" + debuggerCmdPath) - - # See bug 902706 about -g. - return [cdbPath, "-g", "-c", ";".join(cdbCmdList), "-z", dumpFilename] - - time.sleep(0.200) - loops += 1 - if loops > maxLoops: - # Windows may take some time to generate the dump. - print("constructCdbCommand waited a long time, but %s never appeared!" % dumpFilename) - return None - else: - return None - - -def isWinDumpingToDefaultLocation(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc - # pylint: disable=too-complex,too-many-branches - """Check whether Windows minidumps are enabled and set to go to Windows' default location.""" - if sys.version_info.major == 2: - import _winreg as winreg # pylint: disable=import-error - else: - import winreg # pylint: disable=import-error - # For now, this code does not edit the Windows Registry because we tend to be in a 32-bit - # version of Python and if one types in regedit in the Run dialog, opens up the 64-bit registry. - # If writing a key, we most likely need to flush. For the moment, no keys are written. - try: - with winreg.OpenKey(winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE), - r"Software\Microsoft\Windows\Windows Error Reporting\LocalDumps", - # Read key from 64-bit registry, which also works for 32-bit - 0, (winreg.KEY_WOW64_64KEY + winreg.KEY_READ)) as key: - - try: - dumpTypeRegValue = winreg.QueryValueEx(key, "DumpType") - if not (dumpTypeRegValue[0] == 1 and dumpTypeRegValue[1] == winreg.REG_DWORD): - print(noMinidumpMsg) - return False - except WindowsError as e: # pylint: disable=undefined-variable - if e.errno == 2: - print(noMinidumpMsg) - return False - else: - raise - - try: - dumpFolderRegValue = winreg.QueryValueEx(key, "DumpFolder") - # %LOCALAPPDATA%\CrashDumps is the default location. - if not (dumpFolderRegValue[0] == r"%LOCALAPPDATA%\CrashDumps" and - dumpFolderRegValue[1] == winreg.REG_EXPAND_SZ): - print() - print("WARNING: Dumps are instead appearing at: %s - " - "all crashes will be uninteresting." % dumpFolderRegValue[0]) - print() - return False - except WindowsError as e: # pylint: disable=undefined-variable - # If the key value cannot be found, the dumps will be put in the default location - if e.errno == 2 and e.strerror == "The system cannot find the file specified": - return True - else: - raise - - return True - except WindowsError as e: # pylint: disable=undefined-variable - # If the LocalDumps registry key cannot be found, dumps will be put in the default location. - if e.errno == 2 and e.strerror == "The system cannot find the file specified": - print() - print("WARNING: The registry key HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\" - "Windows\\Windows Error Reporting\\LocalDumps cannot be found.") - print() - return None - else: - raise - - -def constructGdbCommand(progfullname, crashedPID): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Construct a command that uses the POSIX debugger (gdb) to turn a minidump file into a stack trace.""" - assert os.name == "posix" - # On Mac and Linux, look for a core file. - core_name = None - if platform.system() == "Darwin": - # Core files will be generated if you do: - # mkdir -p /cores/ - # ulimit -c 2147483648 (or call resource.setrlimit from a preexec_fn hook) - core_name = "/cores/core." + str(crashedPID) - elif platform.system() == "Linux": - is_pid_used = False - if os.path.exists("/proc/sys/kernel/core_uses_pid"): - with open("/proc/sys/kernel/core_uses_pid") as f: - is_pid_used = bool(int(f.read()[0])) # Setting [0] turns the input to a str. - core_name = "core." + str(crashedPID) if is_pid_used else "core" # relative path - if not os.path.isfile(core_name): - core_name = normExpUserPath(os.path.join("~", core_name)) # try the home dir - - if core_name and os.path.exists(core_name): - debuggerCmdPath = getAbsPathForAdjacentFile("gdb_cmds.txt") # pylint: disable=invalid-name - assert os.path.exists(debuggerCmdPath) - - # Run gdb and move the core file. Tip: gdb gives more info for: - # (debug with intact build dir > debug > opt with frame pointers > opt) - return ["gdb", "-n", "-batch", "-x", debuggerCmdPath, progfullname, core_name] - return None - - -def getAbsPathForAdjacentFile(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc - # pylint: disable=missing-return-type-doc,missing-type-doc - """Get the absolute path of a particular file, given its base directory and filename.""" - return os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) - - -def rmTreeIncludingReadOnly(dirTree): # pylint: disable=invalid-name,missing-docstring - shutil.rmtree(dirTree, onerror=handleRemoveReadOnly if platform.system() == "Windows" else None) - - -def test_rmTreeIncludingReadOnly(): # pylint: disable=invalid-name - """Run this function in the same directory as subprocesses to test.""" - test_dir = "test_rmTreeIncludingReadOnly" - os.mkdir(test_dir) - read_only_dir = os.path.join(test_dir, "nestedReadOnlyDir") - os.mkdir(read_only_dir) - filename = os.path.join(read_only_dir, "test.txt") - with open(filename, "w") as f: - f.write("testing\n") - - os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - os.chmod(read_only_dir, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - - rmTreeIncludingReadOnly(test_dir) # Should pass here - - -def handleRemoveReadOnly(func, path, exc): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc - # pylint: disable=missing-type-doc - """Handle read-only files. Adapted from http://stackoverflow.com/q/1213706 .""" if func in (os.rmdir, os.remove) and exc[1].errno == errno.EACCES: - if os.name == "posix": - # Ensure parent directory is also writeable. - pardir = os.path.abspath(os.path.join(path, os.path.pardir)) - if not os.access(pardir, os.W_OK): - os.chmod(pardir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - elif os.name == "nt": - os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 func(path) else: raise OSError("Unable to handle read-only files.") -def normExpUserPath(p): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc - return os.path.normpath(os.path.expanduser(p)) - - def vdump(inp): # pylint: disable=missing-param-doc,missing-type-doc """Append the word "DEBUG" to any verbose output.""" if verbose: diff --git a/tests/js/test_build_options.py b/tests/js/test_build_options.py new file mode 100644 index 000000000..33121a6c2 --- /dev/null +++ b/tests/js/test_build_options.py @@ -0,0 +1,54 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import logging +import sys +import unittest + +from _pytest.monkeypatch import MonkeyPatch +import pytest + +from funfuzz.js import build_options + +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +def mock_chance(i): + """Overwrite the chance function to return True or False depending on a specific condition. + + Args: + i (float): Intended probability between 0 < i < 1 + + Returns: + bool: True if i > 0, False otherwise. + """ + return True if i > 0 else False + + +class BuildOptionsTests(unittest.TestCase): + """"TestCase class for functions in build_options.py""" + monkeypatch = MonkeyPatch() + trees_location = Path.home() / "trees" + + # pylint: disable=no-member + @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), + reason="requires a Mozilla Mercurial repository") + def test_get_random_valid_repo(self): + """Test that a valid repository can be obtained.""" + BuildOptionsTests.monkeypatch.setattr(build_options, "chance", mock_chance) + self.assertEqual(build_options.get_random_valid_repo(self.trees_location), + self.trees_location / "mozilla-central") diff --git a/tests/js/test_compile_shell.py b/tests/js/test_compile_shell.py index 2c8febb95..a5caee9fa 100644 --- a/tests/js/test_compile_shell.py +++ b/tests/js/test_compile_shell.py @@ -10,6 +10,7 @@ import logging import os +import platform import sys import unittest @@ -20,8 +21,10 @@ if sys.version_info.major == 2: from functools32 import lru_cache # pylint: disable=import-error + from pathlib2 import Path else: from functools import lru_cache # pylint: disable=no-name-in-module + from pathlib import Path # pylint: disable=import-error FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") logging.basicConfig(level=logging.DEBUG) @@ -30,15 +33,19 @@ class CompileShellTests(unittest.TestCase): """"TestCase class for functions in compile_shell.py""" + # Paths + mc_hg_repo = Path.home() / "trees" / "mozilla-central" + shell_cache = Path.home() / "shell-cache" + @pytest.mark.slow @lru_cache(maxsize=None) def test_shell_compile(self): """Test compilation of shells depending on the specified environment variable. Returns: - str: Path to the compiled shell. + Path: Path to the compiled shell. """ - self.assertTrue(os.path.isdir(os.path.join(os.path.expanduser("~"), "trees", "mozilla-central"))) + self.assertTrue(self.mc_hg_repo.is_dir()) # pylint: disable=no-member # Change the repository location by uncommenting this line and specifying the right one # "-R ~/trees/mozilla-central/") @@ -47,8 +54,8 @@ def test_shell_compile(self): # Remember to update the corresponding BUILD build parameters in .travis.yml as well build_opts = os.environ.get("BUILD", default_parameters_debug) - opts_parsed = js.build_options.parseShellOptions(build_opts) - hg_hash_of_default = util.hg_helpers.getRepoHashAndId(opts_parsed.repoDir)[0] + opts_parsed = js.build_options.parse_shell_opts(build_opts) + hg_hash_of_default = util.hg_helpers.get_repo_hash_and_id(opts_parsed.repo_dir)[0] # Ensure exit code is 0 self.assertTrue(not js.compile_shell.CompiledShell(opts_parsed, hg_hash_of_default).run(["-b", build_opts])) @@ -60,7 +67,9 @@ def test_shell_compile(self): # This set of builds should also have the following: 32-bit with ARM, with asan, and with clang file_name = "js-64-profDisabled-intlDisabled-linux-" + hg_hash_of_default - compiled_bin = os.path.join(os.path.expanduser("~"), "shell-cache", file_name, file_name) - self.assertTrue(os.path.isfile(compiled_bin)) + js_bin_path = self.shell_cache / file_name / file_name + if platform.system() == "Windows": + js_bin_path.with_suffix(".exe") + self.assertTrue(js_bin_path.is_file()) - return compiled_bin + return js_bin_path diff --git a/tests/js/test_link_fuzzer.py b/tests/js/test_link_fuzzer.py new file mode 100644 index 000000000..14e5cb4f0 --- /dev/null +++ b/tests/js/test_link_fuzzer.py @@ -0,0 +1,47 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import io +import logging +import sys +import unittest + +from funfuzz.js import link_fuzzer + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class LinkFuzzerTests(unittest.TestCase): + """"TestCase class for functions in link_fuzzer.py""" + def test_link_fuzzer(self): + """Test that a full jsfunfuzz file can be created.""" + with tempfile.TemporaryDirectory(suffix="link_fuzzer_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + jsfunfuzz_tmp = tmp_dir / "jsfunfuzz.js" + + link_fuzzer.link_fuzzer(jsfunfuzz_tmp) + + found = False + with io.open(str(jsfunfuzz_tmp), "r", encoding="utf-8", errors="replace") as f: + for line in f: + if "It's looking good" in line: + found = True + break + + self.assertTrue(found) diff --git a/tests/util/test_fork_join.py b/tests/util/test_fork_join.py new file mode 100644 index 000000000..47d309c90 --- /dev/null +++ b/tests/util/test_fork_join.py @@ -0,0 +1,41 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import io +import logging +import sys +import unittest + +from funfuzz.util import fork_join + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class ForkJoinTests(unittest.TestCase): + """"TestCase class for functions in fork_join.py""" + def test_log_name(self): + """Test that incrementally numbered wtmp directories can be created""" + with tempfile.TemporaryDirectory(suffix="make_wtmp_dir_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + log_path = tmp_dir / "forkjoin-1-out.txt" + + with io.open(str(log_path), "w", encoding="utf-8", errors="replace") as f: + f.writelines("test") + + self.assertEqual(fork_join.log_name(tmp_dir, 1, "out"), str(log_path)) diff --git a/tests/util/test_hg_helpers.py b/tests/util/test_hg_helpers.py index 2fccf731b..3d3228b0d 100644 --- a/tests/util/test_hg_helpers.py +++ b/tests/util/test_hg_helpers.py @@ -12,7 +12,14 @@ import sys import unittest -import funfuzz +import pytest + +from funfuzz.util import hg_helpers + +if sys.version_info.major == 2: + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") logging.basicConfig(level=logging.DEBUG) @@ -30,14 +37,23 @@ def assertRaisesRegex(self, *args, **kwds): # pylint: disable=arguments-differ, class HgHelpersTests(TestCase): """"TestCase class for functions in hg_helpers.py""" + trees_location = Path.home() / "trees" + def test_get_cset_hash_in_bisectmsg(self): """Test that we are able to extract the changeset hash from bisection output.""" - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("x 12345:abababababab"), "abababababab") - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("x 12345:123412341234"), "123412341234") - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("12345:abababababab y"), "abababababab") - self.assertEqual(funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg( + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg("x 12345:abababababab"), "abababababab") + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg("x 12345:123412341234"), "123412341234") + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg("12345:abababababab y"), "abababababab") + self.assertEqual(hg_helpers.get_cset_hash_from_bisect_msg( "Testing changeset 41831:4f4c01fb42c3 (2 changesets remaining, ~1 tests)"), "4f4c01fb42c3") with self.assertRaisesRegex(ValueError, (r"^Bisection output format required for hash extraction unavailable. " "The variable msg is:")): - funfuzz.util.hg_helpers.get_cset_hash_from_bisect_msg("1a2345 - abababababab") + hg_helpers.get_cset_hash_from_bisect_msg("1a2345 - abababababab") + + # pylint: disable=no-member + @pytest.mark.skipif(not (trees_location / "mozilla-central" / ".hg" / "hgrc").is_file(), + reason="requires a Mozilla Mercurial repository") + def test_hgrc_repo_name(self): + """Test that we are able to extract the repository name from the hgrc file.""" + self.assertEqual(hg_helpers.hgrc_repo_name(self.trees_location / "mozilla-central"), "mozilla-central") diff --git a/tests/util/test_os_ops.py b/tests/util/test_os_ops.py new file mode 100644 index 000000000..929680c6c --- /dev/null +++ b/tests/util/test_os_ops.py @@ -0,0 +1,50 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import logging +import sys +import unittest + +from funfuzz.util import os_ops + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class OsOpsTests(unittest.TestCase): + """"TestCase class for functions in os_ops.py""" + def test_make_wtmp_dir(self): + """Test that incrementally numbered wtmp directories can be created""" + with tempfile.TemporaryDirectory(suffix="make_wtmp_dir_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + + wtmp_dir_1 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_1.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_1.name.endswith("1")) + + wtmp_dir_2 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_2.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_2.name.endswith("2")) + + wtmp_dir_3 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_3.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_3.name.endswith("3")) + + wtmp_dir_4 = os_ops.make_wtmp_dir(tmp_dir) + self.assertTrue(wtmp_dir_4.is_dir()) # pylint: disable=no-member + self.assertTrue(wtmp_dir_4.name.endswith("4")) diff --git a/tests/util/test_sm_compile_helpers.py b/tests/util/test_sm_compile_helpers.py new file mode 100644 index 000000000..7de5eb8b0 --- /dev/null +++ b/tests/util/test_sm_compile_helpers.py @@ -0,0 +1,43 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Test the compile_shell.py file.""" + +from __future__ import absolute_import, unicode_literals # isort:skip + +import logging +import sys +import unittest + +from funfuzz import util + +if sys.version_info.major == 2: + import backports.tempfile as tempfile # pylint: disable=import-error,no-name-in-module + from pathlib2 import Path +else: + from pathlib import Path # pylint: disable=import-error + import tempfile + +FUNFUZZ_TEST_LOG = logging.getLogger("funfuzz_test") +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("flake8").setLevel(logging.WARNING) + + +class SmCompileHelpersTests(unittest.TestCase): + """"TestCase class for functions in sm_compile_helpers.py""" + def test_autoconf_run(self): # pylint: disable=no-self-use + """Test the autoconf runs properly.""" + with tempfile.TemporaryDirectory(suffix="autoconf_run_test") as tmp_dir: + tmp_dir = Path(tmp_dir) + + # configure.in is required by autoconf2.13 + (tmp_dir / "configure.in").touch() # pylint: disable=no-member + util.sm_compile_helpers.autoconf_run(tmp_dir) + + def test_ensure_cache_dir(self): + """Test the shell-cache dir is created properly if it does not exist, and things work even though it does.""" + self.assertTrue(util.sm_compile_helpers.ensure_cache_dir(None).is_dir()) + self.assertTrue(util.sm_compile_helpers.ensure_cache_dir(Path.home()).is_dir()) # pylint: disable=no-member