Calls to non-existing includes should force a cache miss to display compiler error

clcache.py

@@ -186,6 +186,8 @@ def getManifestHash(compilerBinary, commandLine, sourceFile):
     @staticmethod
     def getIncludesContentHashForFiles(listOfIncludesAbsolute):
         listOfIncludesHashes = [getFileHash(filepath) for filepath in listOfIncludesAbsolute]
+        if None in listOfIncludesHashes:
+            return None
         return ManifestRepository.getIncludesContentHashForHashes(listOfIncludesHashes)
 
     @staticmethod
@@ -425,6 +427,8 @@ def clean(self, stats, maximumSize):
 
     @staticmethod
     def getDirectCacheKey(manifestHash, includesContentHash):
+        if includesContentHash is None:
+            return None
         # We must take into account manifestHash to avoid
         # collisions when different source files use the same
         # set of includes.
@@ -685,8 +689,11 @@ def getCompilerHash(compilerBinary):
 
 def getFileHash(filePath, additionalData=None):
     hasher = HashAlgorithm()
-    with open(filePath, 'rb') as inFile:
-        hasher.update(inFile.read())
+    try:
+        with open(filePath, 'rb') as inFile:
+            hasher.update(inFile.read())
+    except FileNotFoundError:
+        return None
     if additionalData is not None:
         # Encoding of this additional data does not really matter
         # as long as we keep it fixed, otherwise hashes change.
@@ -1474,7 +1481,10 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile):
             # NOTE: command line options already included in hash for manifest name
             includesContentHash = ManifestRepository.getIncludesContentHashForFiles(
                 [expandBasedirPlaceholder(include, baseDir) for include in manifest.includeFiles])
-            cachekey = manifest.includesContentToObjectMap.get(includesContentHash)
+            if includesContentHash is None:
+                cachekey = None
+            else:
+                cachekey = manifest.includesContentToObjectMap.get(includesContentHash)
             if cachekey is not None:
                 if cache.compilerArtifactsRepository.section(cachekey).hasEntry(cachekey):
                     return processCacheHit(cache, objectFile, cachekey)

unittests.py

@@ -22,6 +22,7 @@
     Manifest,
     ManifestRepository,
     Statistics,
+    getFileHash
 )
 from clcache import (
     AnalysisError,
@@ -175,14 +176,16 @@ def filesize(path, filename):
 
         return size
 
+    def setUp(self):
+        self.manifestsRootDir = os.path.join(ASSETS_DIR, "manifests")
+        self.mm = ManifestRepository(self.manifestsRootDir)
+
     def testPaths(self):
-        manifestsRootDir = os.path.join(ASSETS_DIR, "manifests")
-        mm = ManifestRepository(manifestsRootDir)
-        ms = mm.section("fdde59862785f9f0ad6e661b9b5746b7")
+        ms = self.mm.section("fdde59862785f9f0ad6e661b9b5746b7")
 
-        self.assertEqual(ms.manifestSectionDir, os.path.join(manifestsRootDir, "fd"))
+        self.assertEqual(ms.manifestSectionDir, os.path.join(self.manifestsRootDir, "fd"))
         self.assertEqual(ms.manifestPath("fdde59862785f9f0ad6e661b9b5746b7"),
-                         os.path.join(manifestsRootDir, "fd", "fdde59862785f9f0ad6e661b9b5746b7.json"))
+                         os.path.join(self.manifestsRootDir, "fd", "fdde59862785f9f0ad6e661b9b5746b7.json"))
 
     def testIncludesContentHash(self):
         self.assertEqual(
@@ -223,18 +226,15 @@ def testIncludesContentHash(self):
         )
 
     def testStoreAndGetManifest(self):
-        manifestsRootDir = os.path.join(ASSETS_DIR, "manifests")
-        mm = ManifestRepository(manifestsRootDir)
-
         manifest1 = Manifest([r'somepath\myinclude.h'], {
             "fdde59862785f9f0ad6e661b9b5746b7": "a649723940dc975ebd17167d29a532f8"
         })
         manifest2 = Manifest([r'somepath\myinclude.h', 'moreincludes.h'], {
             "474e7fc26a592d84dfa7416c10f036c6": "8771d7ebcf6c8bd57a3d6485f63e3a89"
         })
 
-        ms1 = mm.section("8a33738d88be7edbacef48e262bbb5bc")
-        ms2 = mm.section("0623305942d216c165970948424ae7d1")
+        ms1 = self.mm.section("8a33738d88be7edbacef48e262bbb5bc")
+        ms2 = self.mm.section("0623305942d216c165970948424ae7d1")
 
         ms1.setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1)
         ms2.setManifest("0623305942d216c165970948424ae7d1", manifest2)
@@ -250,16 +250,10 @@ def testStoreAndGetManifest(self):
                          "8771d7ebcf6c8bd57a3d6485f63e3a89")
 
     def testNonExistingManifest(self):
-        manifestsRootDir = os.path.join(ASSETS_DIR, "manifests")
-        mm = ManifestRepository(manifestsRootDir)
-
-        retrieved = mm.section("ffffffffffffffffffffffffffffffff").getManifest("ffffffffffffffffffffffffffffffff")
+        retrieved = self.mm.section("ffffffffffffffffffffffffffffffff").getManifest("ffffffffffffffffffffffffffffffff")
         self.assertIsNone(retrieved)
 
     def testClean(self):
-        manifestsRootDir = os.path.join(ASSETS_DIR, "manifests")
-        mm = ManifestRepository(manifestsRootDir)
-
         # Size in (120, 240] bytes
         manifest1 = Manifest([r'somepath\myinclude.h'], {
             "fdde59862785f9f0ad6e661b9b5746b7": "a649723940dc975ebd17167d29a532f8"
@@ -268,26 +262,35 @@ def testClean(self):
         manifest2 = Manifest([r'somepath\myinclude.h', 'moreincludes.h'], {
             "474e7fc26a592d84dfa7416c10f036c6": "8771d7ebcf6c8bd57a3d6485f63e3a89"
         })
-        mm.section("8a33738d88be7edbacef48e262bbb5bc").setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1)
-        mm.section("0623305942d216c165970948424ae7d1").setManifest("0623305942d216c165970948424ae7d1", manifest2)
+        self.mm.section("8a33738d88be7edbacef48e262bbb5bc").setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1)
+        self.mm.section("0623305942d216c165970948424ae7d1").setManifest("0623305942d216c165970948424ae7d1", manifest2)
 
-        cleaningResultSize = mm.clean(240)
+        cleaningResultSize = self.mm.clean(240)
         # Only one of those manifests can be left
         self.assertLessEqual(cleaningResultSize, 240)
-        self.assertLessEqual(self._getDirectorySize(manifestsRootDir), 240)
+        self.assertLessEqual(self._getDirectorySize(self.manifestsRootDir), 240)
 
-        cleaningResultSize = mm.clean(240)
+        cleaningResultSize = self.mm.clean(240)
         # The one remaining is remains alive
         self.assertLessEqual(cleaningResultSize, 240)
         self.assertGreaterEqual(cleaningResultSize, 120)
-        self.assertLessEqual(self._getDirectorySize(manifestsRootDir), 240)
-        self.assertGreaterEqual(self._getDirectorySize(manifestsRootDir), 120)
+        self.assertLessEqual(self._getDirectorySize(self.manifestsRootDir), 240)
+        self.assertGreaterEqual(self._getDirectorySize(self.manifestsRootDir), 120)
 
-        cleaningResultSize = mm.clean(0)
+        cleaningResultSize = self.mm.clean(0)
         # All manifest are gone
         self.assertEqual(cleaningResultSize, 0)
-        self.assertEqual(self._getDirectorySize(manifestsRootDir), 0)
+        self.assertEqual(self._getDirectorySize(self.manifestsRootDir), 0)
+
+    def testNonExistingInclude(self):
+        try:
+            self.mm.getIncludesContentHashForFiles(["include/removedInclude.h"])
+        except Exception as e:
+            self.fail('Failed to handle non-existing includes: {0} {1} '.format(type(e).__name__,e))
 
+class TestGlobalFunctions(unittest.TestCase):
+    def testgetFileHash(self):
+        self.assertEqual(getFileHash("nonexisting.h"), None)
 
 class TestCompilerArtifactsRepository(unittest.TestCase):
     def testPaths(self):