Add support for loading config from a clcache.conf file

This is useful and the GCC version of ccache also implements a similar concept.

It might be useful in situations where some default settings could be
specified or when environment is not passed correctly down to the compiler
chain.

This change introduces support for clcache.conf file to allow clcache
to read settings from there.

The precedence is as follows:
 1. If present, Environment variable('s) is used, or
 2. [current_working_dir]/clcache.conf is loaded, or, if not found
 3. [%USERPROFILE% or ~/].clcache/clcache.conf is be loaded, or if not found
 4. [%ALLUSERSPROFILE% or /etc/].clcache/clcache.conf is loaded.

In each case, once a clcache.conf file is found, no other conf file is
considered and values only from this file are used. It is also loaded
only once (and all its values then cached) - all to avoid unnecessary
performance penalties.

Author: formiaczek

README.asciidoc

@@ -46,8 +46,13 @@ Options
     Sets the maximum size of the cache in bytes.
     The default value is 1073741824 (1 GiB).
 
-Environment Variables
-~~~~~~~~~~~~~~~~~~~~~
+Configuration
+~~~~~~~~~~~~~
+
+Following values are read from Environment variables, and if absent, from either (in that order):
+  - (current-working-dir)\clcache.conf
+  - %HOME%\.clcache\clcache.conf
+  - %ALLUSERSPROFILE%\.clcache\clcache.conf
 
 CLCACHE_DIR::
     If set, points to the directory within which all the cached object files

clcache/__main__.py

@@ -308,6 +308,49 @@ def getIncludesContentHashForHashes(listOfHashes):
         return HashAlgorithm(','.join(listOfHashes).encode()).hexdigest()
 
 
+class GlobalSettings(object):
+    @staticmethod
+    def getValue(settingName, defaultValue = None):
+        value = os.environ.get(settingName, None)
+        if value is None: # compare to None to allow empty values
+            value = GlobalSettings.__getFromCache(settingName)
+        return value if value is not None else defaultValue
+
+    # serves as a cache to only read the config file once
+    __settings = None
+
+    @staticmethod
+    def __getFromCache(settingName):
+        if not GlobalSettings.__settings:
+            GlobalSettings.__readFromFile()
+        return GlobalSettings.__settings.get(settingName, None)
+
+    @staticmethod
+    def __readFromFile():
+        GlobalSettings.__settings = dict()
+
+        # prefer config in current directory
+        filename = os.path.join(os.getcwd(), "clcache.conf")
+
+        # ..or in home directory..
+        if not os.path.exists(filename):
+            filename = os.path.join(os.path.expanduser("~"), ".clcache", "clcache.conf")
+
+        # or in "sysconfdir" (%ALLUSERSPROFILE%)
+        if not os.path.exists(filename):
+            dir =  os.environ.get('ALLUSERSPROFILE', None)
+            filename = os.path.join(dir if dir else "C:\\Users", ".clcache", "clcache.conf")
+        try:
+            with open(filename) as f:
+                for line in f.readlines():
+                    kv = line.split("=")
+                    if len(kv) != 2 or kv[0].startswith("#"):
+                        continue
+                    GlobalSettings.__settings[kv[0].strip()] = kv[1].split("#")[0].strip()
+        except IOError:
+            pass # only ignore file access errors (including not-existing path)
+
+
 class CacheLock(object):
     """ Implements a lock for the object cache which
     can be used in 'with' statements. """
@@ -359,7 +402,7 @@ def release(self):
 
     @staticmethod
     def forPath(path):
-        timeoutMs = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000))
+        timeoutMs = int(GlobalSettings.getValue('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000))
         lockName = path.replace(':', '-').replace('\\', '-')
         return CacheLock(lockName, timeoutMs)
 
@@ -505,10 +548,8 @@ class CacheFileStrategy(object):
     def __init__(self, cacheDirectory=None):
         self.dir = cacheDirectory
         if not self.dir:
-            try:
-                self.dir = os.environ["CLCACHE_DIR"]
-            except KeyError:
-                self.dir = os.path.join(os.path.expanduser("~"), "clcache")
+            self.dir = GlobalSettings.getValue("CLCACHE_DIR",
+                                               os.path.join(os.path.expanduser("~"), "clcache"))
 
         manifestsRootDir = os.path.join(self.dir, "manifests")
         ensureDirectoryExists(manifestsRootDir)
@@ -593,9 +634,10 @@ def clean(self, stats, maximumSize):
 
 class Cache(object):
     def __init__(self, cacheDirectory=None):
-        if os.environ.get("CLCACHE_MEMCACHED"):
+        memcached = GlobalSettings.getValue("CLCACHE_MEMCACHED")
+        if memcached:
             from .storage import CacheFileWithMemcacheFallbackStrategy
-            self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"),
+            self.strategy = CacheFileWithMemcacheFallbackStrategy(memcached,
                                                                   cacheDirectory=cacheDirectory)
         else:
             self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory)
@@ -900,7 +942,7 @@ def getCompilerHash(compilerBinary):
 
 
 def getFileHashes(filePaths):
-    if 'CLCACHE_SERVER' in os.environ:
+    if GlobalSettings.getValue('CLCACHE_SERVER') not in ['0', 'false', 'False']:
         pipeName = r'\\.\pipe\clcache_srv'
         while True:
             try:
@@ -939,7 +981,7 @@ def getStringHash(dataString):
 
 
 def expandBasedirPlaceholder(path):
-    baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR'))
+    baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR'))
     if path.startswith(BASEDIR_REPLACEMENT):
         if not baseDir:
             raise LogicException('No CLCACHE_BASEDIR set, but found relative path ' + path)
@@ -949,7 +991,7 @@ def expandBasedirPlaceholder(path):
 
 
 def collapseBasedirToPlaceholder(path):
-    baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR'))
+    baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR'))
     if baseDir is None:
         return path
     else:
@@ -972,7 +1014,7 @@ def ensureDirectoryExists(path):
 def copyOrLink(srcFilePath, dstFilePath):
     ensureDirectoryExists(os.path.dirname(os.path.abspath(dstFilePath)))
 
-    if "CLCACHE_HARDLINK" in os.environ:
+    if GlobalSettings.getValue("CLCACHE_HARDLINK") not in ['0', 'false', 'False']:
         ret = windll.kernel32.CreateHardLinkW(str(dstFilePath), str(srcFilePath), None)
         if ret != 0:
             # Touch the time stamp of the new link so that the build system
@@ -998,11 +1040,10 @@ def myExecutablePath():
 
 
 def findCompilerBinary():
-    if "CLCACHE_CL" in os.environ:
-        path = os.environ["CLCACHE_CL"]
+    path = GlobalSettings.getValue("CLCACHE_CL")
+    if path:
         if os.path.basename(path) == path:
             path = which(path)
-
         return path if os.path.exists(path) else None
 
     frozenByPy2Exe = hasattr(sys, "frozen")
@@ -1020,7 +1061,7 @@ def findCompilerBinary():
 
 
 def printTraceStatement(msg: str) -> None:
-    if "CLCACHE_LOG" in os.environ:
+    if GlobalSettings.getValue("CLCACHE_LOG") not in ['0', 'false', 'False']:
         scriptDir = os.path.realpath(os.path.dirname(sys.argv[0]))
         with OUTPUT_LOCK:
             print(os.path.join(scriptDir, "clcache.py") + " " + msg)
@@ -1571,7 +1612,7 @@ def main():
     printTraceStatement("Found real compiler binary at '{0!s}'".format(compiler))
     printTraceStatement("Arguments we care about: '{}'".format(sys.argv))
 
-    if "CLCACHE_DISABLE" in os.environ:
+    if GlobalSettings.getValue("CLCACHE_DISABLE") not in ['0', 'false', 'False']:
         return invokeRealCompiler(compiler, sys.argv[1:])[0]
     try:
         return processCompileRequest(cache, compiler, sys.argv)
@@ -1671,7 +1712,7 @@ def processSingleSource(compiler, cmdLine, sourceFile, objectFile, environment):
         assert objectFile is not None
         cache = Cache()
 
-        if 'CLCACHE_NODIRECT' in os.environ:
+        if GlobalSettings.getValue('CLCACHE_NODIRECT') not in ['0', 'false', 'False']:
             return processNoDirect(cache, objectFile, compiler, cmdLine, environment)
         else:
             return processDirect(cache, objectFile, compiler, cmdLine, sourceFile)
@@ -1770,7 +1811,7 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex
 
 
 if __name__ == '__main__':
-    if 'CLCACHE_PROFILE' in os.environ:
+    if GlobalSettings.getValue('CLCACHE_PROFILE') not in ['0', 'false', 'False']:
         INVOCATION_HASH = getStringHash(','.join(sys.argv))
         cProfile.run('main()', filename='clcache-{}.prof'.format(INVOCATION_HASH))
     else:

tests/test_integration.py

@@ -47,6 +47,28 @@ def cd(targetDirectory):
         os.chdir(oldDirectory)
 
 
+def executeStatsCommand(customEnv = None):
+    cmd = CLCACHE_CMD +  ["-s"]
+    if customEnv:
+        out = subprocess.check_output(cmd, env=customEnv)
+    else:
+        out = subprocess.check_output(cmd)
+    return extractStatsOutput(out.decode("ascii").strip())
+
+
+def extractStatsOutput(outputLines):
+    stats = dict()
+    print(outputLines)
+    for line in outputLines.splitlines():
+        kv = line.split(":", 1)
+        if len(kv) != 2 or not kv[1]:
+            continue
+        stats[kv[0].strip()] = kv[1].strip()
+    # special case to avoid duplication: Update 'Disc cache at X:\\blah\\ccache' => 'X:\\blah\\ccache'
+    stats["current cache dir"] = stats["current cache dir"].split("cache at")[1].strip()
+    return stats
+
+
 class TestCommandLineArguments(unittest.TestCase):
     def testValidMaxSize(self):
         with tempfile.TemporaryDirectory() as tempDir:
@@ -74,6 +96,87 @@ def testPrintStatistics(self):
                 0,
                 "Command must be able to print statistics")
 
+
+class TestGlobalSettings(unittest.TestCase):
+    def testSettingsDefault(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            customEnv = dict(os.environ, HOME=tempDir)
+            stats = executeStatsCommand(customEnv)
+            print (stats)
+            self.assertEqual(stats["current cache dir"], os.path.join(tempDir, "clcache"))
+
+    def testSettingsEnvironmentVariables(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+            stats = executeStatsCommand(customEnv)
+            print (stats)
+            self.assertEqual(stats["current cache dir"], os.path.join(tempDir))
+
+    def testSettingsLocalConfigFile(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            with cd(tempDir):
+                confFileName = os.path.join(tempDir, "clcache.conf")
+                clcacheDir = os.path.join(tempDir, "clcache")
+                self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+                stats = executeStatsCommand()
+                self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testSettingsConfigFileInHomeDir(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+            customEnv = dict(os.environ, HOME=tempDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testSettingsConfigFileInHomeDirOverridenByEnvironment(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR="this should be ignored")
+            customEnv = dict(os.environ, HOME=tempDir, CLCACHE_DIR=clcacheDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testSettingsConfigFileInProfiles(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+            customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testSettingsConfigFileInProfilesOverridenByEnvironment(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR="should be ignored")
+            customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir, CLCACHE_DIR=clcacheDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testSettingsConfigFileInProfilesOverridenByHomeDirSettings(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            clcacheDir = os.path.join(tempDir, "clcache")
+            homeDir = os.path.join(tempDir, "home")
+            self._createConfFile(os.path.join(homeDir, ".clcache", "clcache.conf"), CLCACHE_DIR=clcacheDir)
+            profilesDir = os.path.join(tempDir, "allusersprofile")
+            self._createConfFile(os.path.join(profilesDir, ".clcache", "clcache.conf"), CLCACHE_DIR="ignored")
+            customEnv = dict(os.environ, HOME=homeDir, ALLUSERSPROFILE=profilesDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def _createConfFile(self, filename, **settings):
+        dirname = os.path.dirname(filename)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(filename, "w") as f:
+            for k, v in settings.items():
+                f.write("{0} = {1}\n\r".format(k, v))
+
+
 class TestDistutils(unittest.TestCase):
     @pytest.mark.skipif(not MONKEY_LOADED, reason="Monkeypatch not loaded")
     @pytest.mark.skipif(CLCACHE_MEMCACHED, reason="Fails with memcached")