From d175840386477bd04ca8a7a7cb1c779f76dbed3e Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Fri, 12 Aug 2016 08:25:37 +0200 Subject: [PATCH 01/10] Add missing newlines before class TestBasedir --- integrationtests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integrationtests.py b/integrationtests.py index 39ae2f75..8e73be0b 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -604,6 +604,8 @@ def testHitsViaMpConcurrent(self): self.assertEqual(stats.numCacheHits(), 2) self.assertEqual(stats.numCacheMisses(), 2) self.assertEqual(stats.numCacheEntries(), 2) + + class TestBasedir(unittest.TestCase): def testBasedir(self): with cd(os.path.join(ASSETS_DIR, "basedir")), tempfile.TemporaryDirectory() as tempDir: From 4a37bebf8059fbdeffab68d583b5c0a9835e3275 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Sun, 31 Jul 2016 17:35:11 +0200 Subject: [PATCH 02/10] Add testRequiredHeaderDisappears() --- integrationtests.py | 20 +++++++++++++++++++ tests/integrationtests/header-miss/.gitignore | 2 ++ tests/integrationtests/header-miss/main.cpp | 8 ++++++++ 3 files changed, 30 insertions(+) create mode 100644 tests/integrationtests/header-miss/.gitignore create mode 100644 tests/integrationtests/header-miss/main.cpp diff --git a/integrationtests.py b/integrationtests.py index 8e73be0b..3cb4e05d 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -312,6 +312,26 @@ def testNoDirect(self): self.assertEqual(output, "2") +class TestHeaderMiss(unittest.TestCase): + # When a required header disappears, we must fall back to real compiler + # complaining about the miss + def testRequiredHeaderDisappears(self): + with cd(os.path.join(ASSETS_DIR, "header-miss")): + compileCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", "main.cpp"] + + with open("info.h", "w") as header: + header.write("#define INFO 1337\n") + subprocess.check_call(compileCmd) + + os.remove("info.h") + + # real compiler fails + process = subprocess.Popen(compileCmd, stdout=subprocess.PIPE) + stdout, _ = process.communicate() + self.assertEqual(process.returncode, 2) + self.assertTrue("C1083" in stdout.decode(clcache.CL_DEFAULT_CODEC)) + + class TestRunParallel(unittest.TestCase): def _zeroStats(self): subprocess.check_call(CLCACHE_CMD + ["-z"]) diff --git a/tests/integrationtests/header-miss/.gitignore b/tests/integrationtests/header-miss/.gitignore new file mode 100644 index 00000000..278b3d7d --- /dev/null +++ b/tests/integrationtests/header-miss/.gitignore @@ -0,0 +1,2 @@ +# This file is auto-generated by tests +info.h diff --git a/tests/integrationtests/header-miss/main.cpp b/tests/integrationtests/header-miss/main.cpp new file mode 100644 index 00000000..b6d0bb6e --- /dev/null +++ b/tests/integrationtests/header-miss/main.cpp @@ -0,0 +1,8 @@ +#include +#include "info.h" + +int main() +{ + std::cout << INFO << std::endl; + return 0; +} From ff81bdde0673350b859717141d30215af06853a4 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Sun, 31 Jul 2016 21:39:37 +0200 Subject: [PATCH 03/10] Add testObsoleteHeaderDisappears() --- integrationtests.py | 22 +++++++++++++++++++ .../header-miss-obsolete/.gitignore | 5 +++++ .../header-miss-obsolete/main.cpp | 8 +++++++ 3 files changed, 35 insertions(+) create mode 100644 tests/integrationtests/header-miss-obsolete/.gitignore create mode 100644 tests/integrationtests/header-miss-obsolete/main.cpp diff --git a/integrationtests.py b/integrationtests.py index 3cb4e05d..cdc32c71 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -331,6 +331,28 @@ def testRequiredHeaderDisappears(self): self.assertEqual(process.returncode, 2) self.assertTrue("C1083" in stdout.decode(clcache.CL_DEFAULT_CODEC)) + # When a header included by another header becomes obsolete and disappers, + # we must fall back to real compiler. + def testObsoleteHeaderDisappears(self): + # A includes B + with cd(os.path.join(ASSETS_DIR, "header-miss-obsolete")): + compileCmd = CLCACHE_CMD + ["/I.", "/nologo", "/EHsc", "/c", "main.cpp"] + + with open("A.h", "w") as header: + header.write('#define INFO 1337\n') + header.write('#include "B.h"\n') + with open("B.h", "w") as header: + header.write('#define SOMETHING 1\n') + + subprocess.check_call(compileCmd) + + with open("A.h", "w") as header: + header.write('#define INFO 1337\n') + header.write('\n') + os.remove("B.h") + + subprocess.check_call(compileCmd) + class TestRunParallel(unittest.TestCase): def _zeroStats(self): diff --git a/tests/integrationtests/header-miss-obsolete/.gitignore b/tests/integrationtests/header-miss-obsolete/.gitignore new file mode 100644 index 00000000..3d1cf01d --- /dev/null +++ b/tests/integrationtests/header-miss-obsolete/.gitignore @@ -0,0 +1,5 @@ +*.obj + +# headers auto-generated by tests +A.h +B.h diff --git a/tests/integrationtests/header-miss-obsolete/main.cpp b/tests/integrationtests/header-miss-obsolete/main.cpp new file mode 100644 index 00000000..f6983593 --- /dev/null +++ b/tests/integrationtests/header-miss-obsolete/main.cpp @@ -0,0 +1,8 @@ +#include +#include "A.h" + +int main() +{ + std::cout << INFO << std::endl; + return 0; +} From e480d93ca92472d5f9fd7b58165e99744e2b9f87 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Thu, 4 Aug 2016 23:50:12 +0200 Subject: [PATCH 04/10] Raise IncludeNotFoundException when include file is missing --- clcache.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index cd299de8..416c55ff 100644 --- a/clcache.py +++ b/clcache.py @@ -91,6 +91,10 @@ def normalizeBaseDir(baseDir): return None +class IncludeNotFoundException(Exception): + pass + + class CacheLockException(Exception): pass @@ -185,7 +189,10 @@ def getManifestHash(compilerBinary, commandLine, sourceFile): @staticmethod def getIncludesContentHashForFiles(listOfIncludesAbsolute): - listOfIncludesHashes = [getFileHash(filepath) for filepath in listOfIncludesAbsolute] + try: + listOfIncludesHashes = [getFileHash(filepath) for filepath in listOfIncludesAbsolute] + except FileNotFoundError as e: + raise IncludeNotFoundException(e.filename) return ManifestRepository.getIncludesContentHashForHashes(listOfIncludesHashes) @staticmethod From 2b3068fb47f7ae510cf62b5e31c0185c83579ba5 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Thu, 11 Aug 2016 21:48:17 +0200 Subject: [PATCH 05/10] Rename parseIncludesList to parseIncludesSet and let it return a set --- clcache.py | 15 +++++++++------ unittests.py | 8 ++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/clcache.py b/clcache.py index 416c55ff..24defc7b 100644 --- a/clcache.py +++ b/clcache.py @@ -1231,12 +1231,14 @@ def clearCache(cache): print('Cache cleared') -# Returns pair - list of includes and new compiler output. +# Returns pair: +# 1. set of include filepaths +# 2. new compiler output # Output changes if strip is True in that case all lines with include # directives are stripped from it -def parseIncludesList(compilerOutput, sourceFile, strip): +def parseIncludesSet(compilerOutput, sourceFile, strip): newOutput = [] - includesSet = set([]) + includesSet = set() # Example lines # Note: including file: C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\INCLUDE\limits.h @@ -1263,9 +1265,9 @@ def parseIncludesList(compilerOutput, sourceFile, strip): elif strip: newOutput.append(line) if strip: - return sorted(includesSet), ''.join(newOutput) + return includesSet, ''.join(newOutput) else: - return sorted(includesSet), compilerOutput + return includesSet, compilerOutput def addObjectToCache(stats, cache, cachekey, artifacts): @@ -1326,12 +1328,13 @@ def postprocessHeaderChangedMiss( def postprocessNoManifestMiss( cache, objectFile, manifestSection, manifestHash, baseDir, sourceFile, compilerResult, stripIncludes): returnCode, compilerOutput, compilerStderr = compilerResult - listOfIncludes, compilerOutput = parseIncludesList(compilerOutput, sourceFile, stripIncludes) + includePaths, compilerOutput = parseIncludesSet(compilerOutput, sourceFile, stripIncludes) manifest = None cachekey = None if returnCode == 0 and os.path.exists(objectFile): + listOfIncludes = sorted(includePaths) # Store compile output and manifest if baseDir: relocatableIncludePaths = [collapseBasedirToPlaceholder(path, baseDir) for path in listOfIncludes] diff --git a/unittests.py b/unittests.py index 3b5596a6..08bb6e50 100644 --- a/unittests.py +++ b/unittests.py @@ -812,7 +812,7 @@ def _readSampleFileNoIncludes(self): def testParseIncludesNoStrip(self): sample = self._readSampleFileDefault() - includesSet, newCompilerOutput = clcache.parseIncludesList( + includesSet, newCompilerOutput = clcache.parseIncludesSet( sample['CompilerOutput'], r'C:\Projects\test\smartsqlite\src\version.cpp', strip=False) @@ -826,7 +826,7 @@ def testParseIncludesNoStrip(self): def testParseIncludesStrip(self): sample = self._readSampleFileDefault() - includesSet, newCompilerOutput = clcache.parseIncludesList( + includesSet, newCompilerOutput = clcache.parseIncludesSet( sample['CompilerOutput'], r'C:\Projects\test\smartsqlite\src\version.cpp', strip=True) @@ -841,7 +841,7 @@ def testParseIncludesStrip(self): def testParseIncludesNoIncludes(self): sample = self._readSampleFileNoIncludes() for stripIncludes in [True, False]: - includesSet, newCompilerOutput = clcache.parseIncludesList( + includesSet, newCompilerOutput = clcache.parseIncludesSet( sample['CompilerOutput'], r"C:\Projects\test\myproject\main.cpp", strip=stripIncludes) @@ -851,7 +851,7 @@ def testParseIncludesNoIncludes(self): def testParseIncludesGerman(self): sample = self._readSampleFileDefault(lang="de") - includesSet, _ = clcache.parseIncludesList( + includesSet, _ = clcache.parseIncludesSet( sample['CompilerOutput'], r"C:\Projects\test\smartsqlite\src\version.cpp", strip=False) From 1b3f3c148e609fa997a00e9f35347aac98a5ebb4 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Thu, 11 Aug 2016 22:00:21 +0200 Subject: [PATCH 06/10] Implement handling of changed and missing includes --- clcache.py | 56 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/clcache.py b/clcache.py index 24defc7b..59fe377d 100644 --- a/clcache.py +++ b/clcache.py @@ -95,6 +95,10 @@ class IncludeNotFoundException(Exception): pass +class IncludeChangedException(Exception): + pass + + class CacheLockException(Exception): pass @@ -122,7 +126,7 @@ def setManifest(self, manifestHash, manifest): ensureDirectoryExists(self.manifestSectionDir) with open(self.manifestPath(manifestHash), 'w') as outFile: # Converting namedtuple to JSON via OrderedDict preserves key names and keys order - json.dump(manifest._asdict(), outFile, indent=2) + json.dump(manifest._asdict(), outFile, sort_keys=True, indent=2) def getManifest(self, manifestHash): fileName = self.manifestPath(manifestHash) @@ -142,7 +146,7 @@ class ManifestRepository(object): # invalidation, such that a manifest that was stored using the old format is not # interpreted using the new format. Instead the old file will not be touched # again due to a new manifest hash and is cleaned away after some time. - MANIFEST_FILE_FORMAT_VERSION = 3 + MANIFEST_FILE_FORMAT_VERSION = 4 def __init__(self, manifestsRootDir): self._manifestsRootDir = manifestsRootDir @@ -188,11 +192,22 @@ def getManifestHash(compilerBinary, commandLine, sourceFile): return getFileHash(sourceFile, additionalData) @staticmethod - def getIncludesContentHashForFiles(listOfIncludesAbsolute): - try: - listOfIncludesHashes = [getFileHash(filepath) for filepath in listOfIncludesAbsolute] - except FileNotFoundError as e: - raise IncludeNotFoundException(e.filename) + def getIncludesContentHashForFiles(includes): + listOfIncludesHashes = [] + includeMissing = False + + for path in sorted(includes.keys()): + try: + fileHash = getFileHash(path) + if fileHash != includes[path]: + raise IncludeChangedException() + listOfIncludesHashes.append(fileHash) + except FileNotFoundError: + includeMissing = True + + if includeMissing: + raise IncludeNotFoundException() + return ManifestRepository.getIncludesContentHashForHashes(listOfIncludesHashes) @staticmethod @@ -1334,15 +1349,19 @@ def postprocessNoManifestMiss( cachekey = None if returnCode == 0 and os.path.exists(objectFile): - listOfIncludes = sorted(includePaths) - # Store compile output and manifest + includes = {path:getFileHash(path) for path in includePaths} + includesContentHash = ManifestRepository.getIncludesContentHashForFiles(includes) + cachekey = Cache.getDirectCacheKey(manifestHash, includesContentHash) + + # Create new manifest if baseDir: - relocatableIncludePaths = [collapseBasedirToPlaceholder(path, baseDir) for path in listOfIncludes] + relocatableIncludePaths = { + collapseBasedirToPlaceholder(path, baseDir):contentHash + for path, contentHash in includes.items() + } manifest = Manifest(relocatableIncludePaths, {}) else: - manifest = Manifest(listOfIncludes, {}) - includesContentHash = ManifestRepository.getIncludesContentHashForFiles(listOfIncludes) - cachekey = Cache.getDirectCacheKey(manifestHash, includesContentHash) + manifest = Manifest(includes, {}) manifest.includesContentToObjectMap[includesContentHash] = cachekey with cache.lock, cache.statistics as stats: @@ -1470,6 +1489,11 @@ def processCompileRequest(cache, compiler, args): except CalledForPreprocessingError: printTraceStatement("Cannot cache invocation as {}: called for preprocessing".format(cmdLine)) updateCacheStatistics(cache, Statistics.registerCallForPreprocessing) + except IncludeChangedException: + updateCacheStatistics(cache, Statistics.registerHeaderChangedMiss) + except IncludeNotFoundException: + # register nothing. This is probably just a compile error + pass return invokeRealCompiler(compiler, args[1:]) @@ -1482,8 +1506,10 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): manifest = manifestSection.getManifest(manifestHash) if manifest is not None: # NOTE: command line options already included in hash for manifest name - includesContentHash = ManifestRepository.getIncludesContentHashForFiles( - [expandBasedirPlaceholder(include, baseDir) for include in manifest.includeFiles]) + includesContentHash = ManifestRepository.getIncludesContentHashForFiles({ + expandBasedirPlaceholder(path, baseDir):contentHash + for path, contentHash in manifest.includeFiles.items() + }) cachekey = manifest.includesContentToObjectMap.get(includesContentHash) if cachekey is not None: if cache.compilerArtifactsRepository.section(cachekey).hasEntry(cachekey): From dcfe16d7de4a5fa95fa5b3d1666c1f95fb24e056 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Fri, 12 Aug 2016 08:33:31 +0200 Subject: [PATCH 07/10] Add hit count test to testObsoleteHeaderDisappears --- integrationtests.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/integrationtests.py b/integrationtests.py index cdc32c71..c062b9e9 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -337,6 +337,7 @@ def testObsoleteHeaderDisappears(self): # A includes B with cd(os.path.join(ASSETS_DIR, "header-miss-obsolete")): compileCmd = CLCACHE_CMD + ["/I.", "/nologo", "/EHsc", "/c", "main.cpp"] + cache = clcache.Cache() with open("A.h", "w") as header: header.write('#define INFO 1337\n') @@ -346,6 +347,12 @@ def testObsoleteHeaderDisappears(self): subprocess.check_call(compileCmd) + with cache.statistics as stats: + headerChangedMisses1 = stats.numHeaderChangedMisses() + hits1 = stats.numCacheHits() + misses1 = stats.numCacheMisses() + + # Make include B.h obsolete with open("A.h", "w") as header: header.write('#define INFO 1337\n') header.write('\n') @@ -353,6 +360,27 @@ def testObsoleteHeaderDisappears(self): subprocess.check_call(compileCmd) + with cache.statistics as stats: + headerChangedMisses2 = stats.numHeaderChangedMisses() + hits2 = stats.numCacheHits() + misses2 = stats.numCacheMisses() + + self.assertEqual(headerChangedMisses2, headerChangedMisses1+1) + self.assertEqual(misses2, misses1+1) + self.assertEqual(hits2, hits1) + + # Ensure the new manifest was stored + subprocess.check_call(compileCmd) + + with cache.statistics as stats: + headerChangedMisses3 = stats.numHeaderChangedMisses() + hits3 = stats.numCacheHits() + misses3 = stats.numCacheMisses() + + self.assertEqual(headerChangedMisses3, headerChangedMisses2) + self.assertEqual(misses3, misses2) + self.assertEqual(hits3, hits2+1) + class TestRunParallel(unittest.TestCase): def _zeroStats(self): From ded249c786e62cfb14ad87a0b6047e73afdf168e Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Fri, 12 Aug 2016 08:51:30 +0200 Subject: [PATCH 08/10] postprocessHeaderChangedMiss on IncludeChangedException --- clcache.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/clcache.py b/clcache.py index 59fe377d..f35ac4cd 100644 --- a/clcache.py +++ b/clcache.py @@ -1489,11 +1489,6 @@ def processCompileRequest(cache, compiler, args): except CalledForPreprocessingError: printTraceStatement("Cannot cache invocation as {}: called for preprocessing".format(cmdLine)) updateCacheStatistics(cache, Statistics.registerCallForPreprocessing) - except IncludeChangedException: - updateCacheStatistics(cache, Statistics.registerHeaderChangedMiss) - except IncludeNotFoundException: - # register nothing. This is probably just a compile error - pass return invokeRealCompiler(compiler, args[1:]) @@ -1506,20 +1501,25 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): manifest = manifestSection.getManifest(manifestHash) if manifest is not None: # NOTE: command line options already included in hash for manifest name - includesContentHash = ManifestRepository.getIncludesContentHashForFiles({ - expandBasedirPlaceholder(path, baseDir):contentHash - for path, contentHash in manifest.includeFiles.items() - }) - cachekey = manifest.includesContentToObjectMap.get(includesContentHash) - if cachekey is not None: + try: + includesContentHash = ManifestRepository.getIncludesContentHashForFiles({ + expandBasedirPlaceholder(path, baseDir):contentHash + for path, contentHash in manifest.includeFiles.items() + }) + + cachekey = manifest.includesContentToObjectMap.get(includesContentHash) + assert cachekey is not None if cache.compilerArtifactsRepository.section(cachekey).hasEntry(cachekey): return processCacheHit(cache, objectFile, cachekey) else: postProcessing = lambda compilerResult: postprocessObjectEvicted( cache, objectFile, cachekey, compilerResult) - else: + except IncludeChangedException: postProcessing = lambda compilerResult: postprocessHeaderChangedMiss( cache, objectFile, manifestSection, manifest, manifestHash, includesContentHash, compilerResult) + except IncludeNotFoundException: + # register nothing. This is probably just a compile error + postProcessing = None else: origCmdLine = cmdLine stripIncludes = False @@ -1530,7 +1530,8 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): cache, objectFile, manifestSection, manifestHash, baseDir, sourceFile, compilerResult, stripIncludes) compilerResult = invokeRealCompiler(compiler, cmdLine, captureOutput=True) - compilerResult = postProcessing(compilerResult) + if postProcessing: + compilerResult = postProcessing(compilerResult) printTraceStatement("Finished. Exit code {0:d}".format(compilerResult[0])) return compilerResult From 39765c24fdd74d17d344403614bc3aff68a6baca Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Fri, 12 Aug 2016 09:52:50 +0200 Subject: [PATCH 09/10] Recreate manifest on HeaderChangedMiss --- clcache.py | 64 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/clcache.py b/clcache.py index f35ac4cd..ccc72a89 100644 --- a/clcache.py +++ b/clcache.py @@ -1318,17 +1318,34 @@ def postprocessObjectEvicted(cache, objectFile, cachekey, compilerResult): return compilerResult -def postprocessHeaderChangedMiss( - cache, objectFile, manifestSection, manifest, manifestHash, includesContentHash, compilerResult): +def createManifest(manifestHash, includePaths): + baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) + + includes = {path:getFileHash(path) for path in includePaths} + includesContentHash = ManifestRepository.getIncludesContentHashForFiles(includes) cachekey = Cache.getDirectCacheKey(manifestHash, includesContentHash) + + # Create new manifest + if baseDir: + relocatableIncludePaths = { + collapseBasedirToPlaceholder(path, baseDir):contentHash + for path, contentHash in includes.items() + } + manifest = Manifest(relocatableIncludePaths, {}) + else: + manifest = Manifest(includes, {}) + manifest.includesContentToObjectMap[includesContentHash] = cachekey + return manifest, cachekey + + +def postprocessHeaderChangedMiss( + cache, objectFile, manifestSection, manifestHash, sourceFile, compilerResult, stripIncludes): returnCode, compilerOutput, compilerStderr = compilerResult + includePaths, compilerOutput = parseIncludesSet(compilerOutput, sourceFile, stripIncludes) removedItems = [] if returnCode == 0 and os.path.exists(objectFile): - while len(manifest.includesContentToObjectMap) >= MAX_MANIFEST_HASHES: - _, objectHash = manifest.includesContentToObjectMap.popitem() - removedItems.append(objectHash) - manifest.includesContentToObjectMap[includesContentHash] = cachekey + manifest, cachekey = createManifest(manifestHash, includePaths) with cache.lock, cache.statistics as stats: stats.registerHeaderChangedMiss() @@ -1341,7 +1358,7 @@ def postprocessHeaderChangedMiss( def postprocessNoManifestMiss( - cache, objectFile, manifestSection, manifestHash, baseDir, sourceFile, compilerResult, stripIncludes): + cache, objectFile, manifestSection, manifestHash, sourceFile, compilerResult, stripIncludes): returnCode, compilerOutput, compilerStderr = compilerResult includePaths, compilerOutput = parseIncludesSet(compilerOutput, sourceFile, stripIncludes) @@ -1349,20 +1366,7 @@ def postprocessNoManifestMiss( cachekey = None if returnCode == 0 and os.path.exists(objectFile): - includes = {path:getFileHash(path) for path in includePaths} - includesContentHash = ManifestRepository.getIncludesContentHashForFiles(includes) - cachekey = Cache.getDirectCacheKey(manifestHash, includesContentHash) - - # Create new manifest - if baseDir: - relocatableIncludePaths = { - collapseBasedirToPlaceholder(path, baseDir):contentHash - for path, contentHash in includes.items() - } - manifest = Manifest(relocatableIncludePaths, {}) - else: - manifest = Manifest(includes, {}) - manifest.includesContentToObjectMap[includesContentHash] = cachekey + manifest, cachekey = createManifest(manifestHash, includePaths) with cache.lock, cache.statistics as stats: stats.registerSourceChangedMiss() @@ -1498,6 +1502,7 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): manifestHash = ManifestRepository.getManifestHash(compiler, cmdLine, sourceFile) manifestSection = cache.manifestRepository.section(manifestHash) with cache.lock: + createNewManifest = False manifest = manifestSection.getManifest(manifestHash) if manifest is not None: # NOTE: command line options already included in hash for manifest name @@ -1515,19 +1520,22 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): postProcessing = lambda compilerResult: postprocessObjectEvicted( cache, objectFile, cachekey, compilerResult) except IncludeChangedException: + createNewManifest = True postProcessing = lambda compilerResult: postprocessHeaderChangedMiss( - cache, objectFile, manifestSection, manifest, manifestHash, includesContentHash, compilerResult) + cache, objectFile, manifestSection, manifestHash, sourceFile, compilerResult, stripIncludes) except IncludeNotFoundException: # register nothing. This is probably just a compile error postProcessing = None else: - origCmdLine = cmdLine - stripIncludes = False - if '/showIncludes' not in cmdLine: - cmdLine = ['/showIncludes'] + origCmdLine - stripIncludes = True + createNewManifest = True postProcessing = lambda compilerResult: postprocessNoManifestMiss( - cache, objectFile, manifestSection, manifestHash, baseDir, sourceFile, compilerResult, stripIncludes) + cache, objectFile, manifestSection, manifestHash, sourceFile, compilerResult, stripIncludes) + + if createNewManifest: + stripIncludes = False + if '/showIncludes' not in cmdLine: + cmdLine.insert(0, '/showIncludes') + stripIncludes = True compilerResult = invokeRealCompiler(compiler, cmdLine, captureOutput=True) if postProcessing: From 8e16bcd74488bb4e892d4e8a947146b988831986 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Tue, 16 Aug 2016 19:29:39 +0200 Subject: [PATCH 10/10] Return updated compilerOutput in postprocessHeaderChangedMiss() --- clcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index ccc72a89..45c6eb30 100644 --- a/clcache.py +++ b/clcache.py @@ -1354,7 +1354,7 @@ def postprocessHeaderChangedMiss( cache.compilerArtifactsRepository.removeObjects(stats, removedItems) manifestSection.setManifest(manifestHash, manifest) - return compilerResult + return returnCode, compilerOutput, compilerStderr def postprocessNoManifestMiss(