ShaderCompiler: Polishing

Author: alichraghi

include/nbl/asset/utils/IShaderCompiler.h

@@ -33,7 +33,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 				{
 					system::path absolutePath = {};
 					std::string contents = {};
-					std::array<uint64_t, 4> hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future
+					std::array<uint8_t, 32> hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future
 					// Could be used in the future for early rejection of cache hit
 					//nbl::system::IFileBase::time_point_t lastWriteTime = {};
 
@@ -185,7 +185,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 				// Used to check compatibility of Caches before reading
 				constexpr static inline std::string_view VERSION = "1.0.0";
 
-				using hash_t = std::array<uint64_t,4>;
+				using hash_t = std::array<uint8_t,32>;
 				static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8
 
 				struct SEntry
@@ -196,11 +196,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 					{
 						public:
 							// Perf note: hashing while preprocessor lexing is likely to be slower than just hashing the whole array like this 
-							inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, const std::string_view& _contents, bool _standardInclude, std::array<uint64_t, 4> _hash) :
-								requestingSourceDir(_requestingSourceDir), identifier(_identifier), contents(_contents), standardInclude(_standardInclude), hash(_hash)
-							{
-								assert(!_contents.empty());
-							}
+							inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, bool _standardInclude, std::array<uint8_t, 32> _hash) :
+								requestingSourceDir(_requestingSourceDir), identifier(_identifier), standardInclude(_standardInclude), hash(_hash)
+							{}
 
 							inline SPreprocessingDependency(SPreprocessingDependency&) = default;
 							inline SPreprocessingDependency& operator=(SPreprocessingDependency&) = delete;
@@ -218,11 +216,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 							// path or identifier
 							system::path requestingSourceDir = "";
 							std::string identifier = "";
-							// file contents
-							// TODO: change to `core::vector<uint8_t>` a compressed blob of LZMA, and store all contents together in the `SEntry`
-							std::string contents = ""; 
 							// hash of the contents - used to check against a found_t
-							std::array<uint64_t, 4> hash = {};
+							std::array<uint8_t, 32> hash = {};
 							// If true, then `getIncludeStandard` was used to find, otherwise `getIncludeRelative`
 							bool standardInclude = false;
 					};
@@ -351,9 +346,13 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 						// Now add the mainFileContents and produce both lookup and early equality rejection hashes
 						hashable.insert(hashable.end(), mainFileContents.begin(), mainFileContents.end());
-						hash = nbl::core::XXHash_256(hashable.data(), hashable.size());
-						lookupHash = hash[0];
-						for (auto i = 1u; i < 4; i++) {
+
+						core::blake3_hasher hasher;
+						hasher.update(hashable.data(), hashable.size());
+						hash = { *static_cast<core::blake3_hash_t>(hasher).data };
+						// ALI:TODO
+						lookupHash = std::bit_cast<uint64_t, uint8_t[8]>({hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7]});
+						for (auto i = 8u; i < 32; i++) {
 							core::hash_combine<uint64_t>(lookupHash, hash[i]);
 						}
 					}
@@ -374,7 +373,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 					// TODO: make some of these private
 					std::string mainFileContents;
 					SCompilerArgs compilerArgs;
-					std::array<uint64_t,4> hash;
+					std::array<uint8_t,32> hash;
 					size_t lookupHash;
 					dependency_container_t dependencies;
 					core::smart_refctd_ptr<asset::ICPUShader> cpuShader;
@@ -429,7 +428,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 				};
 
-				using EntrySet = core::unordered_multiset<SEntry, Hash, KeyEqual>;
+				using EntrySet = core::unordered_set<SEntry, Hash, KeyEqual>;
 				EntrySet m_container;
 
 				NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const;

src/nbl/asset/utils/IShaderCompiler.cpp

@@ -116,7 +116,10 @@ auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& req
         retVal = std::move(contents);
     else retVal = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), includeName);
 
-    retVal.hash = nbl::core::XXHash_256((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t)));
+
+    core::blake3_hasher hasher;
+    hasher.update((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t)));
+    retVal.hash = { *static_cast<core::blake3_hash_t>(hasher).data };
     return retVal;
 }
 
@@ -129,7 +132,10 @@ auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& req
     if (auto contents = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), includeName))
         retVal = std::move(contents);
     else retVal = std::move(trySearchPaths(includeName));
-    retVal.hash = nbl::core::XXHash_256((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t)));
+
+    core::blake3_hasher hasher;
+    hasher.update((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t)));
+    retVal.hash = { *static_cast<core::blake3_hash_t>(hasher).data };
     return retVal;
 }
 
@@ -238,7 +244,7 @@ IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_
             else
                 header = finder->getIncludeRelative(dependency.requestingSourceDir, dependency.identifier);
 
-            if (header.hash != dependency.hash || header.contents != dependency.contents)
+            if (header.hash != dependency.hash)
             {
                 allDependenciesMatch = false;
                 break;

src/nbl/asset/utils/shaderCompiler_serialization.h

@@ -116,7 +116,6 @@ inline void to_json(json& j, const SEntry::SPreprocessingDependency& dependency)
     j = json{
         { "requestingSourceDir", dependency.requestingSourceDir },
         { "identifier", dependency.identifier },
-        { "contents", dependency.contents },
         { "hash", dependency.hash },
         { "standardInclude", dependency.standardInclude },
     };
@@ -126,7 +125,6 @@ inline void from_json(const json& j, SEntry::SPreprocessingDependency& dependenc
 {
     j.at("requestingSourceDir").get_to(dependency.requestingSourceDir);
     j.at("identifier").get_to(dependency.identifier);
-    j.at("contents").get_to(dependency.contents);
     j.at("hash").get_to(dependency.hash);
     j.at("standardInclude").get_to(dependency.standardInclude);
 }

src/nbl/asset/utils/waveContext.h

@@ -533,7 +533,7 @@ template<> inline bool boost::wave::impl::pp_iterator_functor<nbl::wave::context
 
     // If caching was requested, push a new SDependency onto dependencies
     if (ctx.cachingRequested) {
-        ctx.dependencies.emplace_back(ctx.get_current_directory(), file_path, result.contents, standardInclude, std::move(result.hash));
+        ctx.dependencies.emplace_back(ctx.get_current_directory(), file_path, standardInclude, std::move(result.hash));
     }
 
     ctx.located_include_content = std::move(result.contents);