[Runtime] Change SimpleGlobalCache to use ConcurrentReadableHashMap instead of ConcurrentMap.

This gives us faster lookups and a small advantage in memory usage. Most of these maps need stable addresses for their entries, so we add a level of indirection to ConcurrentReadableHashMap for these cases to accommodate that. This costs some extra memory, but it's still a net win.

A new StableAddressConcurrentReadableHashMap type handles this indirection and adds a convenience getOrInsert to take advantage of it.

ConcurrentReadableHashMap is tweaked to avoid any global constructors or destructors when using it as a global variable.

ForeignWitnessTables does not need stable addresses and it now uses ConcurrentReadableHashMap directly.

rdar://problem/70056398

Author: mikeash

include/swift/Runtime/Concurrent.h

@@ -579,6 +579,11 @@ using llvm::hash_value;
 /// ensure that readers which started before the clear see valid (pre-clear)
 /// data. Readers which see any array as empty will produce no results, thus
 /// providing valid post-clear data.
+///
+/// This is intended to be used for tables that exist for the life of the
+/// process. It has no destructor, to avoid generating useless global destructor
+/// calls. The memory it allocates can be freed by calling clear() with no
+/// outstanding readers, but this won't destroy the static mutex it uses.
 template <class ElemTy> struct ConcurrentReadableHashMap {
   // We use memcpy and don't call destructors. Make sure the elements will put
   // up with this.
@@ -724,7 +729,7 @@ template <class ElemTy> struct ConcurrentReadableHashMap {
   std::atomic<IndexStorage *> Indices{nullptr};
 
   /// The writer lock, which must be taken before any mutation of the table.
-  Mutex WriterLock;
+  StaticMutex WriterLock;
 
   /// The maximum number of elements that the current elements array can hold.
   uint32_t ElementCapacity{0};
@@ -841,20 +846,19 @@ template <class ElemTy> struct ConcurrentReadableHashMap {
   }
 
 public:
+  // Implicitly trivial constructor/destructor.
+  ConcurrentReadableHashMap() = default;
+  ~ConcurrentReadableHashMap() = default;
+
   // This type cannot be safely copied or moved.
   ConcurrentReadableHashMap(const ConcurrentReadableHashMap &) = delete;
   ConcurrentReadableHashMap(ConcurrentReadableHashMap &&) = delete;
   ConcurrentReadableHashMap &
   operator=(const ConcurrentReadableHashMap &) = delete;
 
-  ConcurrentReadableHashMap()
-      : ReaderCount(0), ElementCount(0), Elements(nullptr), Indices(nullptr),
-        ElementCapacity(0) {}
-
-  ~ConcurrentReadableHashMap() {
-    assert(ReaderCount.load(std::memory_order_acquire) == 0 &&
-           "deallocating ConcurrentReadableHashMap with outstanding snapshots");
-    FreeListNode::freeAll(&FreeList);
+  /// Returns whether there are outstanding readers. For testing purposes only.
+  bool hasActiveReaders() {
+    return ReaderCount.load(std::memory_order_relaxed) > 0;
   }
 
   /// Readers take a snapshot of the hash map, then work with the snapshot.
@@ -945,7 +949,7 @@ template <class ElemTy> struct ConcurrentReadableHashMap {
   /// The return value is ignored when `created` is `false`.
   template <class KeyTy, typename Call>
   void getOrInsert(KeyTy key, const Call &call) {
-    ScopedLock guard(WriterLock);
+    StaticScopedLock guard(WriterLock);
 
     auto *indices = Indices.load(std::memory_order_relaxed);
     if (!indices)
@@ -955,7 +959,7 @@ template <class ElemTy> struct ConcurrentReadableHashMap {
     auto elementCount = ElementCount.load(std::memory_order_relaxed);
     auto *elements = Elements.load(std::memory_order_relaxed);
 
-    auto found = find(key, indices, elementCount, elements);
+    auto found = this->find(key, indices, elementCount, elements);
     if (found.first) {
       call(found.first, false);
       deallocateFreeListIfSafe();
@@ -996,7 +1000,7 @@ template <class ElemTy> struct ConcurrentReadableHashMap {
   /// Clear the hash table, freeing (when safe) all memory currently used for
   /// indices and elements.
   void clear() {
-    ScopedLock guard(WriterLock);
+    StaticScopedLock guard(WriterLock);
 
     auto *indices = Indices.load(std::memory_order_relaxed);
     auto *elements = Elements.load(std::memory_order_relaxed);
@@ -1015,6 +1019,66 @@ template <class ElemTy> struct ConcurrentReadableHashMap {
   }
 };
 
+/// A wrapper type for indirect hash map elements. Stores a pointer to the real
+/// element and forwards key matching and hashing.
+template <class ElemTy> struct HashMapElementWrapper {
+  ElemTy *Ptr;
+
+  template <class KeyTy> bool matchesKey(const KeyTy &key) {
+    return Ptr->matchesKey(key);
+  }
+
+  friend llvm::hash_code hash_value(const HashMapElementWrapper &wrapper) {
+    return hash_value(*wrapper.Ptr);
+  }
+};
+
+/// A ConcurrentReadableHashMap that provides stable addresses for the elements
+/// by allocating them separately and storing pointers to them. The elements of
+/// the hash table are instances of HashMapElementWrapper. A new getOrInsert
+/// method is provided that directly returns the stable element pointer.
+template <class ElemTy, class Allocator>
+struct StableAddressConcurrentReadableHashMap
+    : public ConcurrentReadableHashMap<HashMapElementWrapper<ElemTy>> {
+  // Implicitly trivial destructor.
+  ~StableAddressConcurrentReadableHashMap() = default;
+
+  /// Get or insert an element for the given key and arguments. Returns the
+  /// pointer to the existing or new element, and a bool indicating whether the
+  /// element was created. When false, the element already existed before the
+  /// call.
+  template <class KeyTy, class... ArgTys>
+  std::pair<ElemTy *, bool> getOrInsert(KeyTy key, ArgTys &&...args) {
+    // Optimize for the case where the value already exists.
+    if (auto wrapper = this->snapshot().find(key))
+      return {wrapper->Ptr, false};
+
+    // No such element. Insert if needed. Note: another thread may have inserted
+    // it in the meantime, so we still have to handle both cases!
+    ElemTy *ptr = nullptr;
+    bool outerCreated = false;
+    ConcurrentReadableHashMap<HashMapElementWrapper<ElemTy>>::getOrInsert(
+        key, [&](HashMapElementWrapper<ElemTy> *wrapper, bool created) {
+          if (created) {
+            // Created the indirect entry. Allocate the actual storage.
+            size_t allocSize =
+                sizeof(ElemTy) + ElemTy::getExtraAllocationSize(key, args...);
+            void *memory = Allocator().Allocate(allocSize, alignof(ElemTy));
+            new (memory) ElemTy(key, std::forward<ArgTys>(args)...);
+            wrapper->Ptr = reinterpret_cast<ElemTy *>(memory);
+          }
+          ptr = wrapper->Ptr;
+          outerCreated = created;
+          return true; // Keep the new entry.
+        });
+    return {ptr, outerCreated};
+  }
+
+private:
+  // Clearing would require deallocating elements, which we don't support.
+  void clear() = delete;
+};
+
 } // end namespace swift
 
 #endif // SWIFT_RUNTIME_CONCURRENTUTILS_H

stdlib/public/runtime/HeapObject.cpp

@@ -229,8 +229,10 @@ class BoxCacheEntry {
     return reinterpret_cast<intptr_t>(Data.BoxedType);
   }
 
-  int compareWithKey(const Metadata *type) const {
-    return comparePointers(type, Data.BoxedType);
+  bool matchesKey(const Metadata *type) const { return type == Data.BoxedType; }
+
+  friend llvm::hash_code hash_value(const BoxCacheEntry &value) {
+    return llvm::hash_value(value.Data.BoxedType);
   }
 
   static size_t getExtraAllocationSize(const Metadata *key) {