feat: add kvcache event uploading for host block.

Author: Kang-Meng

xllm/core/framework/block/block_manager_impl.cpp

@@ -171,7 +171,6 @@ void BlockManagerImpl::get_merged_kvcache_event(KvCacheEvent* event) const {
   if (events != nullptr) {
     event->removed_cache.merge(events->removed_cache);
     event->stored_cache.merge(events->stored_cache);
-    event->offload_cache.merge(events->offload_cache);
     events->clear();
   }
 }

xllm/core/framework/block/block_manager_impl.h

@@ -77,7 +77,7 @@ class BlockManagerImpl : public BlockManager {
   }
 
   float get_gpu_cache_usage_perc() const override {
-    return 1.0 - num_free_blocks_ * 1.0 / num_total_blocks();
+    return 1 - static_cast<float>(num_free_blocks_) / num_total_blocks();
   }
 
   // call BlockManager to free block used by Block.

xllm/core/framework/block/multi_tier_block_manager_pool.cpp

@@ -265,4 +265,15 @@ void MultiTierBlockManagerPool::transfer_blocks(std::vector<Batch>* batches) {
   saved_device_blocks_.resize(host_block_managers_.size());
 }
 
+void MultiTierBlockManagerPool::get_merged_kvcache_event(
+    KvCacheEvent* event) const {
+  if (host_block_managers_.empty()) {
+    BlockManagerPool::get_merged_kvcache_event(event);
+  } else {
+    for (int32_t i = 0; i < host_block_managers_.size(); ++i) {
+      host_block_managers_[i]->get_merged_kvcache_event(event);
+    }
+  }
+}
+
 }  // namespace xllm

xllm/core/framework/block/multi_tier_block_manager_pool.h

@@ -40,6 +40,8 @@ class MultiTierBlockManagerPool : public BlockManagerPool {
   bool update_prefetch_result(std::shared_ptr<Request>& request,
                               const uint32_t timeout) override;
 
+  void get_merged_kvcache_event(KvCacheEvent* event) const override;
+
  private:
   void allocate_host_shared(Sequence* sequence);
 

xllm/core/framework/kv_cache/kv_cache_event.h

@@ -25,13 +25,10 @@ struct KvCacheEvent {
       stored_cache;
   std::unordered_set<Murmur3Key, FixedStringKeyHash, FixedStringKeyEqual>
       removed_cache;
-  std::unordered_set<Murmur3Key, FixedStringKeyHash, FixedStringKeyEqual>
-      offload_cache;
 
   void clear() {
     stored_cache.clear();
     removed_cache.clear();
-    offload_cache.clear();
   }
 };
 

xllm/core/framework/request/sequence.h

@@ -19,7 +19,6 @@ limitations under the License.
 #include <absl/time/time.h>
 #include <folly/futures/Future.h>
 
-#include <chrono>
 #include <cstdint>
 #include <vector>
 
@@ -249,7 +248,7 @@ class Sequence final {
     return &prefetch_results_;
   }
 
-  bool update_prefetch_result(uint32_t timeout = 30);
+  bool update_prefetch_result(uint32_t timeout);
 
   void reset();
 

xllm/core/runtime/xservice_client.cpp

@@ -330,15 +330,6 @@ void XServiceClient::heartbeat() {
                                            sizeof(hash_key.data));
           }
         }
-
-        if (event.offload_cache.size()) {
-          cache_event->mutable_offload_cache()->Reserve(
-              event.offload_cache.size());
-          for (auto& hash_key : event.offload_cache) {
-            cache_event->add_offload_cache(hash_key.data,
-                                           sizeof(hash_key.data));
-          }
-        }
       }
 
       req.mutable_load_metrics()->set_gpu_cache_usage_perc(

xllm/proto/xservice.proto

@@ -42,7 +42,6 @@ message InstanceMetaInfo {
 message KvCacheEvent {
     repeated bytes stored_cache = 1;
     repeated bytes removed_cache = 2;
-    repeated bytes offload_cache = 3;
 }
 
 message LoadMetrics {
@@ -55,7 +54,6 @@ message LatencyMetrics {
   int64 recent_max_tbt = 2;
 }
 
-// TODO: add metainfo/metrics
 message HeartbeatRequest {
   string name = 1;
   KvCacheEvent cache_event = 2;