From 7f5aa0a17b57557f23c88137f57385a1728aaabc Mon Sep 17 00:00:00 2001 From: ashwinagrl Date: Sat, 14 Sep 2024 03:43:55 +0530 Subject: [PATCH 1/6] Made Changes for PT Integration --- CMakeLists.txt | 7 +- MLModelRunner/C/CMakeLists.txt | 2 + MLModelRunner/CMakeLists.txt | 4 +- MLModelRunner/ONNXModelRunner/CMakeLists.txt | 3 + MLModelRunner/PTModelRunner.cpp | 39 +++++ MLModelRunner/gRPCModelRunner/CMakeLists.txt | 3 +- SerDes/CMakeLists.txt | 7 +- SerDes/pytorchSerDes.cpp | 156 +++++++++++++++++++ include/MLModelRunner/MLModelRunner.h | 1 + include/MLModelRunner/PTModelRunner.h | 76 +++++++++ include/SerDes/pytorchSerDes.h | 53 +++++++ 11 files changed, 347 insertions(+), 4 deletions(-) create mode 100644 MLModelRunner/PTModelRunner.cpp create mode 100644 SerDes/pytorchSerDes.cpp create mode 100644 include/MLModelRunner/PTModelRunner.h create mode 100644 include/SerDes/pytorchSerDes.h diff --git a/CMakeLists.txt b/CMakeLists.txt index dfe2959..5d1c1be 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) +find_package(Torch REQUIRED) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" @@ -65,7 +66,9 @@ if(LLVM_MLBRIDGE) LINK_LIBS ModelRunnerLib SerDesLib - ) + ) + + target_link_libraries(LLVMMLBridge PRIVATE "${TORCH_LIBRARIES}") target_include_directories(LLVMMLBridge SYSTEM PUBLIC ${Protobuf_INCLUDE_DIRS} ${TENSORFLOW_AOT_PATH}/include) target_include_directories(LLVMMLBridge PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -87,6 +90,8 @@ else() add_library(MLCompilerBridgeC STATIC $) target_link_libraries(MLCompilerBridgeC PUBLIC SerDesCLib ModelRunnerCLib ONNXModelRunnerLib ${llvm_libs}) + target_link_libraries(MLCompilerBridge PUBLIC "${TORCH_LIBRARIES}") + target_include_directories(MLCompilerBridgeC PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${LLVM_INCLUDE_DIRS}) target_compile_features(MLCompilerBridgeC PRIVATE cxx_std_17) target_compile_definitions(MLCompilerBridgeC PRIVATE C_LIBRARY) diff --git a/MLModelRunner/C/CMakeLists.txt b/MLModelRunner/C/CMakeLists.txt index bcfa357..1283815 100644 --- a/MLModelRunner/C/CMakeLists.txt +++ b/MLModelRunner/C/CMakeLists.txt @@ -1,2 +1,4 @@ +find_package(Torch REQUIRED) add_library(ModelRunnerCWrapper OBJECT PipeModelRunnerCWrapper.cpp ONNXModelRunnerCWrapper.cpp) target_include_directories(ModelRunnerCWrapper PUBLIC ${TENSORFLOW_AOT_PATH}/include) +target_link_libraries(ModelRunnerCWrapper PUBLIC "${TORCH_LIBRARIES}") diff --git a/MLModelRunner/CMakeLists.txt b/MLModelRunner/CMakeLists.txt index 57a1cde..fd20923 100755 --- a/MLModelRunner/CMakeLists.txt +++ b/MLModelRunner/CMakeLists.txt @@ -1,3 +1,4 @@ +find_package(Torch REQUIRED) add_subdirectory(Utils) set(PROTOS_DIRECTORY "" CACHE PATH "Path to the directory containing the proto files") if(NOT PROTOS_DIRECTORY STREQUAL "") @@ -44,7 +45,8 @@ else() add_library(ModelRunnerLib OBJECT PipeModelRunner.cpp) endif(LLVM_MLBRIDGE) -target_link_libraries(ModelRunnerLib PUBLIC ModelRunnerUtils ONNXModelRunnerLib) +target_link_libraries(ModelRunnerLib PUBLIC ModelRunnerUtils ONNXModelRunnerLib "${TORCH_LIBRARIES}") + if(NOT PROTOS_DIRECTORY STREQUAL "") target_link_libraries(ModelRunnerLib PUBLIC gRPCModelRunnerLib) diff --git a/MLModelRunner/ONNXModelRunner/CMakeLists.txt b/MLModelRunner/ONNXModelRunner/CMakeLists.txt index d8f197c..501c163 100755 --- a/MLModelRunner/ONNXModelRunner/CMakeLists.txt +++ b/MLModelRunner/ONNXModelRunner/CMakeLists.txt @@ -2,6 +2,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -DORT_NO_EXCEPTIONS") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra") find_package(Onnxruntime REQUIRED) +find_package(Torch REQUIRED) #onnxruntime providers option(onnxruntime_USE_CUDA "Build with CUDA support" OFF) @@ -52,5 +53,7 @@ add_library(ONNXModelRunnerLib OBJECT onnx.cpp ) endif(LLVM_MLBRIDGE) +target_link_libraries(ONNXModelRunnerLib PRIVATE "${TORCH_LIBRARIES}") target_link_libraries(ONNXModelRunnerLib PRIVATE Onnxruntime::Onnxruntime) target_include_directories(ONNXModelRunnerLib PUBLIC ${TENSORFLOW_AOT_PATH}/include) + diff --git a/MLModelRunner/PTModelRunner.cpp b/MLModelRunner/PTModelRunner.cpp new file mode 100644 index 0000000..b0ed092 --- /dev/null +++ b/MLModelRunner/PTModelRunner.cpp @@ -0,0 +1,39 @@ +//=== PTModelRunner.cpp - PTModelRunner Implementation ---*- C++ -*-===// +// +// Part of the MLCompilerBridge Project +// +//===------------------===// + +#include "MLModelRunner/PTModelRunner.h" +#include "SerDes/pytorchSerDes.h" +#include "llvm/Support/ErrorHandling.h" + +using namespace torch::inductor; + +namespace MLBridge { + +template +void *PTModelRunner::evaluateUntyped() { + if (CompiledModel->inputTensors.empty()) { + llvm::errs() << "Input vector is empty.\n"; + return nullptr; + } + + try { + // Run the model with the input tensors + auto outputs = CompiledModel->CompiledModel->run(CompiledModel->inputTensors); + + // Store the outputs + CompiledModel->outputTensors = outputs; + + // Convert to raw data format using deserializeUntyped + void *rawData = CompiledModel->deserializeUntyped(&CompiledModel->outputTensors); + + return rawData; + } catch (const c10::Error &e) { + llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; + return nullptr; + } +} + +} // namespace MLBridge diff --git a/MLModelRunner/gRPCModelRunner/CMakeLists.txt b/MLModelRunner/gRPCModelRunner/CMakeLists.txt index 877afef..2e18d8a 100755 --- a/MLModelRunner/gRPCModelRunner/CMakeLists.txt +++ b/MLModelRunner/gRPCModelRunner/CMakeLists.txt @@ -12,7 +12,8 @@ endif() # Find gRPC installation # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. -find_package(gRPC 1.58.0 EXACT CONFIG REQUIRED) +# find_package(gRPC 1.58.0 EXACT CONFIG REQUIRED) +cmake_minimum_required(VERSION 3.10) message(STATUS "Using gRPC ${gRPC_VERSION}") set(_GRPC_GRPCPP gRPC::grpc++) diff --git a/SerDes/CMakeLists.txt b/SerDes/CMakeLists.txt index 9125409..def821a 100755 --- a/SerDes/CMakeLists.txt +++ b/SerDes/CMakeLists.txt @@ -1,5 +1,8 @@ set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) +find_package(Torch REQUIRED) + + if(LLVM_MLBRIDGE) add_llvm_library(SerDesLib @@ -7,13 +10,15 @@ if(LLVM_MLBRIDGE) jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp + pytorchSerDes.cpp tensorflowSerDes.cpp ) target_compile_definitions(SerDesLib PRIVATE LLVM_MLBRIDGE) else() - add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp tensorflowSerDes.cpp) + add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp tensorflowSerDes.cpp pytorchSerDes.cpp) add_library(SerDesCLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp) endif() target_include_directories(SerDesLib PUBLIC ${TENSORFLOW_AOT_PATH}/include) target_link_libraries(SerDesLib PRIVATE tf_xla_runtime) +target_link_libraries(SerDesLib PRIVATE "${TORCH_LIBRARIES}") diff --git a/SerDes/pytorchSerDes.cpp b/SerDes/pytorchSerDes.cpp new file mode 100644 index 0000000..c871e1c --- /dev/null +++ b/SerDes/pytorchSerDes.cpp @@ -0,0 +1,156 @@ +//=== MLCompilerBridge/SerDes/pytorchSerDes.cpp - Pytorch SerDes Implementation ---*- C++ -*-===// +// +// Part of the MLCompilerBridge Project +// +//===------------------===// + + + +#include "SerDes/pytorchSerDes.h" +#include "SerDes/baseSerDes.h" + + +namespace MLBridge { + +void PytorchSerDes::setFeature(const std::string &Name, const int &Value) { + auto tensor = torch::tensor({Value}, torch::kInt32); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const long &Value) { + auto tensor = torch::tensor({Value}, torch::kInt64); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const float &Value) { + auto tensor = torch::tensor({Value}, torch::kFloat32); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const double &Value) { + auto tensor = torch::tensor({Value}, torch::kFloat64); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const std::string &Value) { + std::vector encoded_str(Value.begin(), Value.end()); + auto tensor = torch::tensor(encoded_str, torch::kInt8); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const bool &Value) { + auto tensor = torch::tensor({Value}, torch::kBool); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { + auto tensor = torch::tensor(Value, torch::kInt32); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { + auto tensor = torch::tensor(Value, torch::kInt64); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { + auto tensor = torch::tensor(Value, torch::kFloat32); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { + auto tensor = torch::tensor(Value, torch::kFloat64); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { + std::vector flat_vec; + for (const auto& str : Value) { + flat_vec.insert(flat_vec.end(), str.begin(), str.end()); + flat_vec.push_back('\0'); // Null-terminate each string + } + auto tensor = torch::tensor(flat_vec, torch::kInt8); + inputTensors.push_back(tensor.clone()); +} + +void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { + std::vector bool_vec(Value.begin(), Value.end()); + auto tensor = torch::tensor(bool_vec, torch::kUInt8); + inputTensors.push_back(tensor.clone()); +} + +// void PytorchSerDes::setRequest(void *Request) { +// CompiledModel = reinterpret_cast(Request); +// } + +void PytorchSerDes::cleanDataStructures() { + inputTensors.clear(); // Clear the input vector +} + +void *PytorchSerDes::deserializeUntyped(void *Data) { + if (Data == nullptr) { + return nullptr; + } + + // Assume Data is a pointer to a vector of tensors + std::vector *serializedTensors = reinterpret_cast *>(Data); + + // Calculate total size for the deserialized data + size_t totalSize = 0; + for (const auto &tensor : *serializedTensors) { + totalSize += tensor.numel() * tensor.element_size(); + } + + // Giving value to the MessageLength variable + MessageLength = totalSize; + + // Allocate a contiguous block of memory for the output + auto *output = new uint8_t[totalSize]; + uint8_t *outputPtr = output; + + // Copy tensor data into the contiguous block of memory + for (const auto &tensor : *serializedTensors) { + size_t tensorSize = tensor.numel() * tensor.element_size(); + + if (tensor.dtype() == torch::kInt32) { + // Handle int32 tensors + std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); + } else if (tensor.dtype() == torch::kInt64) { + // Handle int64 tensors + std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); + } else if (tensor.dtype() == torch::kFloat32) { + // Handle float32 tensors + std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); + } else if (tensor.dtype() == torch::kFloat64) { + // Handle float64 tensors + std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); + } else if (tensor.dtype() == torch::kBool) { + // Handle bool tensors + std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); + } else if (tensor.dtype() == torch::kInt8) { + // Handle int8 tensors (e.g., for strings) + std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); + } else { + llvm::errs() << "Unsupported tensor dtype.\n"; + delete[] output; + return nullptr; + } + + outputPtr += tensorSize; + } + return static_cast(output); +} + + +//=== Implementation of the methods ===// +void *PytorchSerDes::getSerializedData() { + std::vector serializedData = outputTensors; + + // Allocate memory for the output and copy the serialized data + auto *output = new std::vector(serializedData); + return static_cast(output); +} + + + +} // namespace MLBridge diff --git a/include/MLModelRunner/MLModelRunner.h b/include/MLModelRunner/MLModelRunner.h index 77892ec..62fc698 100644 --- a/include/MLModelRunner/MLModelRunner.h +++ b/include/MLModelRunner/MLModelRunner.h @@ -49,6 +49,7 @@ #ifndef C_LIBRARY #include "SerDes/protobufSerDes.h" #include "SerDes/tensorflowSerDes.h" +#include "SerDes/pytorchSerDes.h" #endif namespace MLBridge { diff --git a/include/MLModelRunner/PTModelRunner.h b/include/MLModelRunner/PTModelRunner.h new file mode 100644 index 0000000..2467217 --- /dev/null +++ b/include/MLModelRunner/PTModelRunner.h @@ -0,0 +1,76 @@ +#ifndef PTMODELRUNNER_H +#define PTMODELRUNNER_H + +#include "MLModelRunner/MLModelRunner.h" +#include "SerDes/TensorSpec.h" +#include "SerDes/pytorchSerDes.h" +#include "llvm/Support/ErrorHandling.h" +#include +#include // or model_container_runner_cuda.h for CUDA + +#include +#include + + +namespace MLBridge { + +/// PTModelRunner - PyTorch Compiled model implementation of the +/// MLModelRunner. It uses an AOT-compiled model for efficient execution. +template class PTModelRunner final : public MLModelRunner { +public: + // New constructor that takes the model path as an input + PTModelRunner(const std::string &modelPath, llvm::LLVMContext &Ctx) + : MLModelRunner(MLModelRunner::Kind::PTAOT, BaseSerDes::Kind::Pytorch, &Ctx), + CompiledModel(std::make_unique(modelPath)) { + + SerDes->setRequest(CompiledModel.get()); + + assert(CompiledModel && "The CompiledModel should be valid"); + } + + virtual ~PTModelRunner() = default; + + virtual void requestExit() override { + llvm_unreachable("requestExit() is not supported in PTModelRunner"); + } + + static bool classof(const MLModelRunner *R) { + return R->getKind() == MLModelRunner::Kind::PTAOT; + } + +private: + void *evaluateUntyped() override; + + std::unique_ptr CompiledModel; +}; + +/// A mock class satisfying the interface expected by PTModelRunner for +/// its `TGen` parameter. Useful to avoid conditional compilation complexity, as +/// a compile-time replacement for a real AOT-ed model. +class NoopSavedModelImpl final { +#define NOOP_MODEL_ERRMSG \ + "The mock AOT-ed saved model is a compile-time stub and should not be " \ + "called." + +public: + NoopSavedModelImpl() = default; + int LookupArgIndex(const std::string &) { + llvm_unreachable(NOOP_MODEL_ERRMSG); + } + int LookupResultIndex(const std::string &) { + llvm_unreachable(NOOP_MODEL_ERRMSG); + } + void Run() { llvm_unreachable(NOOP_MODEL_ERRMSG); } + void *result_data(int) { llvm_unreachable(NOOP_MODEL_ERRMSG); } + void *arg_data(int) { llvm_unreachable(NOOP_MODEL_ERRMSG); } +#undef NOOP_MODEL_ERRMSG +}; + +template bool isEmbeddedModelEvaluatorValid() { return true; } + +template <> inline bool isEmbeddedModelEvaluatorValid() { + return false; +} +} // namespace MLBridge + +#endif // TFMODELRUNNER_H diff --git a/include/SerDes/pytorchSerDes.h b/include/SerDes/pytorchSerDes.h new file mode 100644 index 0000000..b59961c --- /dev/null +++ b/include/SerDes/pytorchSerDes.h @@ -0,0 +1,53 @@ +#ifndef PYTORCH_SERIALIZER_H +#define PYTORCH_SERIALIZER_H + +#include "SerDes/baseSerDes.h" +#include +#include + +#include +#include +#include + +namespace MLBridge { +class PytorchSerDes : public BaseSerDes { +public: + PytorchSerDes() : BaseSerDes(Kind::Pytorch) {} + + // Constructor that takes the model path and loads the model + PytorchSerDes(const std::string &modelPath) : BaseSerDes(Kind::Pytorch) { + CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); + } + + ~PytorchSerDes() { delete CompiledModel; } + + static bool classof(const BaseSerDes *S) { + return S->getKind() == BaseSerDes::Kind::Pytorch; + } + +#define SET_FEATURE(TYPE) \ + void setFeature(const std::string &, const TYPE &) override; \ + void setFeature(const std::string &, const std::vector &) override; + SUPPORTED_TYPES(SET_FEATURE) +#undef SET_FEATURE + + void setRequest(void *request) override { + CompiledModel = + reinterpret_cast(request); + } + + void *getSerializedData() override; + void cleanDataStructures() override; + void *deserializeUntyped(void *data) override; + + // void addTensorToInput(const torch::Tensor &newTensor); // Add a tensor to the + // input tensor vector + +private: + std::vector inputTensors; // Declaration of the input tensor + std::vector outputTensors; // Storage for the PyTorch output tensor + torch::inductor::AOTIModelContainerRunnerCpu *CompiledModel; +}; +} // namespace MLBridge + +#endif \ No newline at end of file From d6e3435a799510c97cc9d64dcf5fa721d757462a Mon Sep 17 00:00:00 2001 From: ashwinagrl Date: Sat, 14 Sep 2024 04:17:25 +0530 Subject: [PATCH 2/6] Fixed Some issues --- CMakeLists.txt | 5 ++++- MLModelRunner/gRPCModelRunner/CMakeLists.txt | 3 +-- SerDes/CMakeLists.txt | 2 +- SerDes/protobufSerDes.cpp | 2 ++ include/SerDes/protobufSerDes.h | 8 ++++---- include/SerDes/pytorchSerDes.h | 2 +- test/CMakeLists.txt | 3 ++- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d1c1be..e035dff 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,10 @@ message("CMAKE Module path: ${CMAKE_MODULE_PATH}") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib CACHE PATH "Output directory for static libraries") include_directories(${Protobuf_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/include) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -fPIC") + +# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -fPIC") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + set (CMAKE_CXX_STANDARD 17) option(LLVM_MLBRIDGE "MLCompilerBridge install for LLVM" OFF) diff --git a/MLModelRunner/gRPCModelRunner/CMakeLists.txt b/MLModelRunner/gRPCModelRunner/CMakeLists.txt index 2e18d8a..6ce52d6 100755 --- a/MLModelRunner/gRPCModelRunner/CMakeLists.txt +++ b/MLModelRunner/gRPCModelRunner/CMakeLists.txt @@ -12,8 +12,7 @@ endif() # Find gRPC installation # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. -# find_package(gRPC 1.58.0 EXACT CONFIG REQUIRED) -cmake_minimum_required(VERSION 3.10) +find_package(gRPC 1.34.0 EXACT CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") set(_GRPC_GRPCPP gRPC::grpc++) diff --git a/SerDes/CMakeLists.txt b/SerDes/CMakeLists.txt index def821a..2b8e70a 100755 --- a/SerDes/CMakeLists.txt +++ b/SerDes/CMakeLists.txt @@ -17,7 +17,7 @@ if(LLVM_MLBRIDGE) else() add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp tensorflowSerDes.cpp pytorchSerDes.cpp) - add_library(SerDesCLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp) + add_library(SerDesCLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp ) endif() target_include_directories(SerDesLib PUBLIC ${TENSORFLOW_AOT_PATH}/include) target_link_libraries(SerDesLib PRIVATE tf_xla_runtime) diff --git a/SerDes/protobufSerDes.cpp b/SerDes/protobufSerDes.cpp index 1e543f1..9e43e89 100644 --- a/SerDes/protobufSerDes.cpp +++ b/SerDes/protobufSerDes.cpp @@ -23,6 +23,8 @@ #include #include +using namespace google::protobuf; + namespace MLBridge { inline void ProtobufSerDes::setFeature(const std::string &name, const int value) { diff --git a/include/SerDes/protobufSerDes.h b/include/SerDes/protobufSerDes.h index 25b7e3f..d7a5f3a 100644 --- a/include/SerDes/protobufSerDes.h +++ b/include/SerDes/protobufSerDes.h @@ -18,7 +18,7 @@ #include "google/protobuf/extension_set.h" #include "google/protobuf/message.h" -using namespace google::protobuf; +// using namespace google::protobuf; namespace MLBridge { /// ProtobufSerDes - Protobuf Serialization/Deserialization to support gRPC @@ -54,12 +54,12 @@ class ProtobufSerDes : public BaseSerDes { void *getSerializedData() override; void cleanDataStructures() override; - Message *getMessage() { return Response; }; + google::protobuf::Message *getMessage() { return Response; }; private: void *deserializeUntyped(void *data) override; - Message *Response; - Message *Request; + google::protobuf::Message *Response; + google::protobuf::Message *Request; }; } // namespace MLBridge diff --git a/include/SerDes/pytorchSerDes.h b/include/SerDes/pytorchSerDes.h index b59961c..a77025d 100644 --- a/include/SerDes/pytorchSerDes.h +++ b/include/SerDes/pytorchSerDes.h @@ -25,7 +25,7 @@ class PytorchSerDes : public BaseSerDes { return S->getKind() == BaseSerDes::Kind::Pytorch; } -#define SET_FEATURE(TYPE) \ +#define SET_FEATURE(TYPE, _) \ void setFeature(const std::string &, const TYPE &) override; \ void setFeature(const std::string &, const std::vector &) override; SUPPORTED_TYPES(SET_FEATURE) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f4a83a3..a8a2680 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,5 @@ -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") +# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") add_library(MLBridgeCPPTest OBJECT MLBridgeTest.cpp) file(GLOB MODEL_OBJECTS ${CMAKE_CURRENT_SOURCE_DIR}/tf_models/*.o) From 36b19e2dcb3f73b0dd6205a35684710aba6e7501 Mon Sep 17 00:00:00 2001 From: ashwinagrl Date: Sat, 14 Sep 2024 14:01:38 +0530 Subject: [PATCH 3/6] build successful --- SerDes/pytorchSerDes.cpp | 12 ++++++------ include/SerDes/baseSerDes.h | 2 +- include/SerDes/pytorchSerDes.h | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/SerDes/pytorchSerDes.cpp b/SerDes/pytorchSerDes.cpp index c871e1c..2565ee8 100644 --- a/SerDes/pytorchSerDes.cpp +++ b/SerDes/pytorchSerDes.cpp @@ -12,33 +12,33 @@ namespace MLBridge { -void PytorchSerDes::setFeature(const std::string &Name, const int &Value) { +void PytorchSerDes::setFeature(const std::string &Name, const int Value) { auto tensor = torch::tensor({Value}, torch::kInt32); inputTensors.push_back(tensor.clone()); } -void PytorchSerDes::setFeature(const std::string &Name, const long &Value) { +void PytorchSerDes::setFeature(const std::string &Name, const long Value) { auto tensor = torch::tensor({Value}, torch::kInt64); inputTensors.push_back(tensor.clone()); } -void PytorchSerDes::setFeature(const std::string &Name, const float &Value) { +void PytorchSerDes::setFeature(const std::string &Name, const float Value) { auto tensor = torch::tensor({Value}, torch::kFloat32); inputTensors.push_back(tensor.clone()); } -void PytorchSerDes::setFeature(const std::string &Name, const double &Value) { +void PytorchSerDes::setFeature(const std::string &Name, const double Value) { auto tensor = torch::tensor({Value}, torch::kFloat64); inputTensors.push_back(tensor.clone()); } -void PytorchSerDes::setFeature(const std::string &Name, const std::string &Value) { +void PytorchSerDes::setFeature(const std::string &Name, const std::string Value) { std::vector encoded_str(Value.begin(), Value.end()); auto tensor = torch::tensor(encoded_str, torch::kInt8); inputTensors.push_back(tensor.clone()); } -void PytorchSerDes::setFeature(const std::string &Name, const bool &Value) { +void PytorchSerDes::setFeature(const std::string &Name, const bool Value) { auto tensor = torch::tensor({Value}, torch::kBool); inputTensors.push_back(tensor.clone()); } diff --git a/include/SerDes/baseSerDes.h b/include/SerDes/baseSerDes.h index 977143a..6966f64 100644 --- a/include/SerDes/baseSerDes.h +++ b/include/SerDes/baseSerDes.h @@ -43,7 +43,7 @@ namespace MLBridge { /// communication by the MLModelRunner. /// Currently, (int, float) or (long, double), char and bool are supported. /// Vectors of these types are supported as well. -enum class SerDesKind : int { Unknown, Json, Bitstream, Protobuf, Tensorflow }; +enum class SerDesKind : int { Unknown, Json, Bitstream, Protobuf, Tensorflow, Pytorch }; class BaseSerDes { public: SerDesKind getKind() const { return Type; } diff --git a/include/SerDes/pytorchSerDes.h b/include/SerDes/pytorchSerDes.h index a77025d..d74b1d0 100644 --- a/include/SerDes/pytorchSerDes.h +++ b/include/SerDes/pytorchSerDes.h @@ -12,21 +12,21 @@ namespace MLBridge { class PytorchSerDes : public BaseSerDes { public: - PytorchSerDes() : BaseSerDes(Kind::Pytorch) {} + PytorchSerDes() : BaseSerDes(SerDesKind::Pytorch) {} // Constructor that takes the model path and loads the model - PytorchSerDes(const std::string &modelPath) : BaseSerDes(Kind::Pytorch) { + PytorchSerDes(const std::string &modelPath) : BaseSerDes(SerDesKind::Pytorch) { CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); } ~PytorchSerDes() { delete CompiledModel; } static bool classof(const BaseSerDes *S) { - return S->getKind() == BaseSerDes::Kind::Pytorch; + return S->getKind() == SerDesKind::Pytorch; } #define SET_FEATURE(TYPE, _) \ - void setFeature(const std::string &, const TYPE &) override; \ + void setFeature(const std::string &, const TYPE) override; \ void setFeature(const std::string &, const std::vector &) override; SUPPORTED_TYPES(SET_FEATURE) #undef SET_FEATURE From c0e8ccc7fed83a328d60d12e3a2c3bf478596ede Mon Sep 17 00:00:00 2001 From: ashwinagrl Date: Sun, 20 Oct 2024 10:16:38 +0530 Subject: [PATCH 4/6] Add code for PT support --- CMakeLists.txt | 31 +- MLModelRunner/C/CMakeLists.txt | 4 +- MLModelRunner/CMakeLists.txt | 7 +- MLModelRunner/ONNXModelRunner/CMakeLists.txt | 32 +- .../ONNXModelRunner/ONNXModelRunner.cpp | 0 MLModelRunner/ONNXModelRunner/agent.cpp | 0 MLModelRunner/PTModelRunner.cpp | 32 -- MLModelRunner/Utils/CMakeLists.txt | 2 +- MLModelRunner/gRPCModelRunner/CMakeLists.txt | 7 +- SerDes/CMakeLists.txt | 34 +- SerDes/pytorchSerDes.cpp | 100 +++--- include/MLModelRunner/MLModelRunner.h | 15 +- .../ONNXModelRunner/ONNXModelRunner.h | 0 include/MLModelRunner/ONNXModelRunner/utils.h | 0 include/MLModelRunner/PTModelRunner.h | 75 +++-- include/MLModelRunner/PipeModelRunner.h | 0 include/SerDes/pytorchSerDes.h | 29 +- test/CMakeLists.txt | 17 +- test/MLBridgeTest.cpp | 303 +++--------------- 19 files changed, 223 insertions(+), 465 deletions(-) mode change 100755 => 100644 CMakeLists.txt mode change 100755 => 100644 MLModelRunner/CMakeLists.txt mode change 100755 => 100644 MLModelRunner/ONNXModelRunner/CMakeLists.txt mode change 100755 => 100644 MLModelRunner/ONNXModelRunner/ONNXModelRunner.cpp mode change 100755 => 100644 MLModelRunner/ONNXModelRunner/agent.cpp mode change 100755 => 100644 MLModelRunner/Utils/CMakeLists.txt mode change 100755 => 100644 MLModelRunner/gRPCModelRunner/CMakeLists.txt mode change 100755 => 100644 SerDes/CMakeLists.txt mode change 100755 => 100644 include/MLModelRunner/ONNXModelRunner/ONNXModelRunner.h mode change 100755 => 100644 include/MLModelRunner/ONNXModelRunner/utils.h mode change 100755 => 100644 include/MLModelRunner/PipeModelRunner.h diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100755 new mode 100644 index e035dff..6101ef5 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,8 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ) message("CMAKE Module path: ${CMAKE_MODULE_PATH}") + + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib CACHE PATH "Output directory for static libraries") include_directories(${Protobuf_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -73,8 +75,8 @@ if(LLVM_MLBRIDGE) target_link_libraries(LLVMMLBridge PRIVATE "${TORCH_LIBRARIES}") - target_include_directories(LLVMMLBridge SYSTEM PUBLIC ${Protobuf_INCLUDE_DIRS} ${TENSORFLOW_AOT_PATH}/include) - target_include_directories(LLVMMLBridge PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) + target_include_directories(LLVMMLBridge SYSTEM PRIVATE ${Protobuf_INCLUDE_DIRS} ${TENSORFLOW_AOT_PATH}/include) + target_include_directories(LLVMMLBridge PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) install(TARGETS LLVMMLBridge DESTINATION lib) add_custom_command(TARGET LLVMMLBridge POST_BUILD @@ -85,30 +87,35 @@ else() llvm_map_components_to_libnames(llvm_libs support core irreader analysis TransformUtils) add_library(MLCompilerBridge STATIC tools.cpp) - target_link_libraries(MLCompilerBridge PUBLIC SerDesLib ModelRunnerLib ONNXModelRunnerLib ${llvm_libs}) + target_link_libraries(MLCompilerBridge PRIVATE SerDesLib ModelRunnerLib ONNXModelRunnerLib ${llvm_libs}) + target_link_libraries(MLCompilerBridge PRIVATE "${TORCH_LIBRARIES}") set_target_properties(MLCompilerBridge PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set_property(TARGET MLCompilerBridge PROPERTY POSITION_INDEPENDENT_CODE 1) install(TARGETS MLCompilerBridge DESTINATION lib) add_library(MLCompilerBridgeC STATIC $) - target_link_libraries(MLCompilerBridgeC PUBLIC SerDesCLib ModelRunnerCLib ONNXModelRunnerLib ${llvm_libs}) - target_link_libraries(MLCompilerBridge PUBLIC "${TORCH_LIBRARIES}") + target_link_libraries(MLCompilerBridgeC PRIVATE SerDesCLib ModelRunnerCLib ONNXModelRunnerLib ${llvm_libs}) - target_include_directories(MLCompilerBridgeC PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${LLVM_INCLUDE_DIRS}) + target_include_directories(MLCompilerBridgeC PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${LLVM_INCLUDE_DIRS}) target_compile_features(MLCompilerBridgeC PRIVATE cxx_std_17) target_compile_definitions(MLCompilerBridgeC PRIVATE C_LIBRARY) set_property(TARGET MLCompilerBridgeC PROPERTY POSITION_INDEPENDENT_CODE 1) set_target_properties(MLCompilerBridgeC PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) install(TARGETS MLCompilerBridgeC DESTINATION lib) if(MLBRIDGE_ENABLE_TEST) - add_executable(MLCompilerBridgeTest $) - target_link_libraries(MLCompilerBridgeTest PUBLIC MLCompilerBridge) - set_target_properties(MLCompilerBridgeTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - install(TARGETS MLCompilerBridgeTest DESTINATION bin) - endif() -endif(LLVM_MLBRIDGE) + add_executable(MLCompilerBridgeTest $) + target_link_libraries(MLCompilerBridgeTest PRIVATE MLCompilerBridge) + set_target_properties(MLCompilerBridgeTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + install(TARGETS MLCompilerBridgeTest DESTINATION bin) +endif() +endif(LLVM_MLBRIDGE) install(DIRECTORY include/ DESTINATION include) install(DIRECTORY CompilerInterface DESTINATION MLModelRunner/CompilerInterface) file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/CompilerInterface DESTINATION ${CMAKE_BINARY_DIR}/MLModelRunner/) +set_property(TARGET MLCompilerBridge PROPERTY CXX_STANDARD 17) + + + +message(STATUS "TensorFlow HH version: ${TensorFlow_VERSION}") \ No newline at end of file diff --git a/MLModelRunner/C/CMakeLists.txt b/MLModelRunner/C/CMakeLists.txt index 1283815..d0903f7 100644 --- a/MLModelRunner/C/CMakeLists.txt +++ b/MLModelRunner/C/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(Torch REQUIRED) add_library(ModelRunnerCWrapper OBJECT PipeModelRunnerCWrapper.cpp ONNXModelRunnerCWrapper.cpp) -target_include_directories(ModelRunnerCWrapper PUBLIC ${TENSORFLOW_AOT_PATH}/include) -target_link_libraries(ModelRunnerCWrapper PUBLIC "${TORCH_LIBRARIES}") +target_include_directories(ModelRunnerCWrapper PRIVATE ${TENSORFLOW_AOT_PATH}/include) +target_link_libraries(ModelRunnerCWrapper PRIVATE "${TORCH_LIBRARIES}") diff --git a/MLModelRunner/CMakeLists.txt b/MLModelRunner/CMakeLists.txt old mode 100755 new mode 100644 index fd20923..efe3a1c --- a/MLModelRunner/CMakeLists.txt +++ b/MLModelRunner/CMakeLists.txt @@ -36,7 +36,7 @@ if (NOT TENSORFLOW_AOT_PATH STREQUAL "") endif() add_library(ModelRunnerCLib OBJECT PipeModelRunner.cpp) -target_link_libraries(ModelRunnerCLib PUBLIC ModelRunnerCUtils ONNXModelRunnerLib) +target_link_libraries(ModelRunnerCLib PRIVATE ModelRunnerCUtils ONNXModelRunnerLib) target_compile_definitions(ModelRunnerCLib PRIVATE C_LIBRARY) if(LLVM_MLBRIDGE) @@ -45,10 +45,11 @@ else() add_library(ModelRunnerLib OBJECT PipeModelRunner.cpp) endif(LLVM_MLBRIDGE) -target_link_libraries(ModelRunnerLib PUBLIC ModelRunnerUtils ONNXModelRunnerLib "${TORCH_LIBRARIES}") +target_link_libraries(ModelRunnerLib PRIVATE ModelRunnerUtils ONNXModelRunnerLib "${TORCH_LIBRARIES}") if(NOT PROTOS_DIRECTORY STREQUAL "") - target_link_libraries(ModelRunnerLib PUBLIC gRPCModelRunnerLib) + target_link_libraries(ModelRunnerLib PRIVATE gRPCModelRunnerLib) endif() set_property(TARGET ModelRunnerLib PROPERTY POSITION_INDEPENDENT_CODE 1) +set_property(TARGET ModelRunnerLib PROPERTY CXX_STANDARD 17) diff --git a/MLModelRunner/ONNXModelRunner/CMakeLists.txt b/MLModelRunner/ONNXModelRunner/CMakeLists.txt old mode 100755 new mode 100644 index 501c163..a96c57e --- a/MLModelRunner/ONNXModelRunner/CMakeLists.txt +++ b/MLModelRunner/ONNXModelRunner/CMakeLists.txt @@ -1,8 +1,7 @@ -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -DORT_NO_EXCEPTIONS") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -DORT_NO_EXCEPTIONS") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra") find_package(Onnxruntime REQUIRED) -find_package(Torch REQUIRED) #onnxruntime providers option(onnxruntime_USE_CUDA "Build with CUDA support" OFF) @@ -14,30 +13,30 @@ option(onnxruntime_USE_TENSORRT "Build with TensorRT support" OFF) set(ONNXRUNTIME_ROOTDIR "" CACHE PATH "Directory that contains ONNXRuntime" ) if(NOT ONNXRUNTIME_ROOTDIR) - message( FATAL_ERROR "Set path to Onnx runtime in -DONNXRUNTIME_ROOTDIR variable" ) +message( FATAL_ERROR "Set path to Onnx runtime in -DONNXRUNTIME_ROOTDIR variable" ) endif() if(onnxruntime_USE_CUDA) - add_definitions(-DUSE_CUDA) +add_definitions(-DUSE_CUDA) endif() if(onnxruntime_USE_OPENVINO) - add_definitions(-DUSE_OPENVINO) +add_definitions(-DUSE_OPENVINO) endif() if(onnxruntime_USE_NNAPI_BUILTIN) - add_definitions(-DUSE_NNAPI) +add_definitions(-DUSE_NNAPI) endif() if(onnxruntime_USE_DNNL) - add_definitions(-DUSE_DNNL) +add_definitions(-DUSE_DNNL) endif() if(onnxruntime_USE_NUPHAR) - add_definitions(-DUSE_NUPHAR) +add_definitions(-DUSE_NUPHAR) endif() if(onnxruntime_USE_TENSORRT) - add_definitions(-DUSE_TENSORRT) +add_definitions(-DUSE_TENSORRT) endif() if(onnxruntime_USE_DML) - message("Enabling DML") - add_definitions(-DUSE_DML) +message("Enabling DML") +add_definitions(-DUSE_DML) endif() if(LLVM_MLBRIDGE) @@ -48,12 +47,13 @@ ONNXModelRunner.cpp ) else() add_library(ONNXModelRunnerLib OBJECT onnx.cpp - agent.cpp - ONNXModelRunner.cpp - ) +agent.cpp +ONNXModelRunner.cpp +) endif(LLVM_MLBRIDGE) -target_link_libraries(ONNXModelRunnerLib PRIVATE "${TORCH_LIBRARIES}") target_link_libraries(ONNXModelRunnerLib PRIVATE Onnxruntime::Onnxruntime) -target_include_directories(ONNXModelRunnerLib PUBLIC ${TENSORFLOW_AOT_PATH}/include) +target_include_directories(ONNXModelRunnerLib PRIVATE ${TENSORFLOW_AOT_PATH}/include) +find_package(Torch REQUIRED) +target_link_libraries(ONNXModelRunnerLib PRIVATE ${TORCH_LIBRARIES}) diff --git a/MLModelRunner/ONNXModelRunner/ONNXModelRunner.cpp b/MLModelRunner/ONNXModelRunner/ONNXModelRunner.cpp old mode 100755 new mode 100644 diff --git a/MLModelRunner/ONNXModelRunner/agent.cpp b/MLModelRunner/ONNXModelRunner/agent.cpp old mode 100755 new mode 100644 diff --git a/MLModelRunner/PTModelRunner.cpp b/MLModelRunner/PTModelRunner.cpp index b0ed092..96f7939 100644 --- a/MLModelRunner/PTModelRunner.cpp +++ b/MLModelRunner/PTModelRunner.cpp @@ -5,35 +5,3 @@ //===------------------===// #include "MLModelRunner/PTModelRunner.h" -#include "SerDes/pytorchSerDes.h" -#include "llvm/Support/ErrorHandling.h" - -using namespace torch::inductor; - -namespace MLBridge { - -template -void *PTModelRunner::evaluateUntyped() { - if (CompiledModel->inputTensors.empty()) { - llvm::errs() << "Input vector is empty.\n"; - return nullptr; - } - - try { - // Run the model with the input tensors - auto outputs = CompiledModel->CompiledModel->run(CompiledModel->inputTensors); - - // Store the outputs - CompiledModel->outputTensors = outputs; - - // Convert to raw data format using deserializeUntyped - void *rawData = CompiledModel->deserializeUntyped(&CompiledModel->outputTensors); - - return rawData; - } catch (const c10::Error &e) { - llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; - return nullptr; - } -} - -} // namespace MLBridge diff --git a/MLModelRunner/Utils/CMakeLists.txt b/MLModelRunner/Utils/CMakeLists.txt old mode 100755 new mode 100644 index 0ee5d0d..2844543 --- a/MLModelRunner/Utils/CMakeLists.txt +++ b/MLModelRunner/Utils/CMakeLists.txt @@ -2,7 +2,7 @@ if(LLVM_MLBRIDGE) add_llvm_component_library(ModelRunnerUtils MLConfig.cpp ) - target_include_directories(ModelRunnerUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include) + target_include_directories(ModelRunnerUtils PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../include) else() add_library(ModelRunnerCUtils OBJECT MLConfig.cpp) diff --git a/MLModelRunner/gRPCModelRunner/CMakeLists.txt b/MLModelRunner/gRPCModelRunner/CMakeLists.txt old mode 100755 new mode 100644 index 6ce52d6..95b5b6e --- a/MLModelRunner/gRPCModelRunner/CMakeLists.txt +++ b/MLModelRunner/gRPCModelRunner/CMakeLists.txt @@ -1,5 +1,6 @@ set(protobuf_MODULE_COMPATIBLE TRUE) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +find_package(Torch REQUIRED) find_package(Protobuf CONFIG REQUIRED) set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) set(_REFLECTION gRPC::grpc++_reflection) @@ -12,7 +13,7 @@ endif() # Find gRPC installation # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. -find_package(gRPC 1.34.0 EXACT CONFIG REQUIRED) +find_package(gRPC 1.66.0 EXACT CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") set(_GRPC_GRPCPP gRPC::grpc++) @@ -96,9 +97,11 @@ target_link_libraries(gRPCModelRunnerLib PRIVATE ${_REFLECTION} ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} + "${TORCH_LIBRARIES}" ) target_include_directories(gRPCModelRunnerLib PRIVATE ${CMAKE_BINARY_DIR}/include/Service - PUBLIC ${Protobuf_INCLUDE_DIRS} + PRIVATE ${Protobuf_INCLUDE_DIRS} ) +set_property(TARGET gRPCModelRunnerLib PROPERTY CXX_STANDARD 17) diff --git a/SerDes/CMakeLists.txt b/SerDes/CMakeLists.txt old mode 100755 new mode 100644 index 2b8e70a..59deef3 --- a/SerDes/CMakeLists.txt +++ b/SerDes/CMakeLists.txt @@ -1,24 +1,28 @@ set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) -find_package(Torch REQUIRED) - +set(protobuf_MODULE_COMPATIBLE TRUE) +find_package(Protobuf CONFIG REQUIRED) if(LLVM_MLBRIDGE) - add_llvm_library(SerDesLib - TensorSpec.cpp - jsonSerDes.cpp - bitstreamSerDes.cpp - protobufSerDes.cpp - pytorchSerDes.cpp - tensorflowSerDes.cpp - ) - target_compile_definitions(SerDesLib PRIVATE LLVM_MLBRIDGE) +add_llvm_library(SerDesLib +TensorSpec.cpp +jsonSerDes.cpp +bitstreamSerDes.cpp +protobufSerDes.cpp +pytorchSerDes.cpp +tensorflowSerDes.cpp +) +target_compile_definitions(SerDesLib PRIVATE LLVM_MLBRIDGE) else() - add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp tensorflowSerDes.cpp pytorchSerDes.cpp) +# add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp tensorflowSerDes.cpp pytorchSerDes.cpp) +add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp pytorchSerDes.cpp) + - add_library(SerDesCLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp ) +add_library(SerDesCLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp ) endif() -target_include_directories(SerDesLib PUBLIC ${TENSORFLOW_AOT_PATH}/include) +target_include_directories(SerDesLib PRIVATE ${TENSORFLOW_AOT_PATH}/include) +# find_package(Torch REQUIRED) target_link_libraries(SerDesLib PRIVATE tf_xla_runtime) -target_link_libraries(SerDesLib PRIVATE "${TORCH_LIBRARIES}") +target_link_libraries(SerDesLib PRIVATE ${TORCH_LIBRARIES}) +set_property(TARGET SerDesLib PROPERTY CXX_STANDARD 17) diff --git a/SerDes/pytorchSerDes.cpp b/SerDes/pytorchSerDes.cpp index 2565ee8..eb98839 100644 --- a/SerDes/pytorchSerDes.cpp +++ b/SerDes/pytorchSerDes.cpp @@ -4,63 +4,61 @@ // //===------------------===// - - #include "SerDes/pytorchSerDes.h" #include "SerDes/baseSerDes.h" - namespace MLBridge { void PytorchSerDes::setFeature(const std::string &Name, const int Value) { auto tensor = torch::tensor({Value}, torch::kInt32); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const long Value) { auto tensor = torch::tensor({Value}, torch::kInt64); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const float Value) { auto tensor = torch::tensor({Value}, torch::kFloat32); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const double Value) { auto tensor = torch::tensor({Value}, torch::kFloat64); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::string Value) { std::vector encoded_str(Value.begin(), Value.end()); auto tensor = torch::tensor(encoded_str, torch::kInt8); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const bool Value) { auto tensor = torch::tensor({Value}, torch::kBool); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kInt32); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kInt64); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kFloat32); - inputTensors.push_back(tensor.clone()); + tensor = tensor.reshape({1, Value.size()}); + inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kFloat64); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { @@ -70,13 +68,13 @@ void PytorchSerDes::setFeature(const std::string &Name, const std::vectorinputTensors->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { std::vector bool_vec(Value.begin(), Value.end()); auto tensor = torch::tensor(bool_vec, torch::kUInt8); - inputTensors.push_back(tensor.clone()); + this->inputTensors->push_back(tensor.clone()); } // void PytorchSerDes::setRequest(void *Request) { @@ -84,7 +82,7 @@ void PytorchSerDes::setFeature(const std::string &Name, const std::vector // } void PytorchSerDes::cleanDataStructures() { - inputTensors.clear(); // Clear the input vector + this->inputTensors->clear(); // Clear the input vector } void *PytorchSerDes::deserializeUntyped(void *Data) { @@ -95,56 +93,38 @@ void *PytorchSerDes::deserializeUntyped(void *Data) { // Assume Data is a pointer to a vector of tensors std::vector *serializedTensors = reinterpret_cast *>(Data); - // Calculate total size for the deserialized data - size_t totalSize = 0; - for (const auto &tensor : *serializedTensors) { - totalSize += tensor.numel() * tensor.element_size(); + if (serializedTensors->empty()) { + return nullptr; } - // Giving value to the MessageLength variable - MessageLength = totalSize; - - // Allocate a contiguous block of memory for the output - auto *output = new uint8_t[totalSize]; - uint8_t *outputPtr = output; - - // Copy tensor data into the contiguous block of memory - for (const auto &tensor : *serializedTensors) { - size_t tensorSize = tensor.numel() * tensor.element_size(); - - if (tensor.dtype() == torch::kInt32) { - // Handle int32 tensors - std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); - } else if (tensor.dtype() == torch::kInt64) { - // Handle int64 tensors - std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); - } else if (tensor.dtype() == torch::kFloat32) { - // Handle float32 tensors - std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); - } else if (tensor.dtype() == torch::kFloat64) { - // Handle float64 tensors - std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); - } else if (tensor.dtype() == torch::kBool) { - // Handle bool tensors - std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); - } else if (tensor.dtype() == torch::kInt8) { - // Handle int8 tensors (e.g., for strings) - std::memcpy(outputPtr, tensor.data_ptr(), tensorSize); - } else { - llvm::errs() << "Unsupported tensor dtype.\n"; - delete[] output; - return nullptr; - } - - outputPtr += tensorSize; + auto type_vect = serializedTensors->at(0).dtype(); + + if (type_vect == torch::kInt32) { + return copyTensorToVect(serializedTensors); + } + else if (type_vect == torch::kInt64) { + return copyTensorToVect(serializedTensors); + } + else if (type_vect == torch::kFloat32) { + return copyTensorToVect(serializedTensors); + } + else if (type_vect == torch::kFloat64) { + return copyTensorToVect(serializedTensors); + } + else if (type_vect == torch::kBool) { + return copyTensorToVect(serializedTensors); + } + else if (type_vect == torch::kInt8) { + return copyTensorToVect(serializedTensors); + } + else { + llvm::errs() << "Unsupported tensor dtype.\n"; + return nullptr; } - return static_cast(output); } - -//=== Implementation of the methods ===// void *PytorchSerDes::getSerializedData() { - std::vector serializedData = outputTensors; + std::vector serializedData = *(this->outputTensors); // Allocate memory for the output and copy the serialized data auto *output = new std::vector(serializedData); diff --git a/include/MLModelRunner/MLModelRunner.h b/include/MLModelRunner/MLModelRunner.h index 62fc698..a764193 100644 --- a/include/MLModelRunner/MLModelRunner.h +++ b/include/MLModelRunner/MLModelRunner.h @@ -82,7 +82,7 @@ class MLModelRunner { } /// Type of the MLModelRunner - enum class Kind : int { Unknown, Pipe, gRPC, ONNX, TFAOT }; + enum class Kind : int { Unknown, Pipe, gRPC, ONNX, TFAOT, PTAOT}; Kind getKind() const { return Type; } SerDesKind getSerDesKind() const { return SerDesType; } @@ -152,11 +152,14 @@ class MLModelRunner { SerDes = std::make_unique(); break; #ifndef C_LIBRARY - case SerDesKind::Protobuf: - SerDes = std::make_unique(); - break; - case SerDesKind::Tensorflow: - SerDes = std::make_unique(); + // case SerDesKind::Protobuf: + // SerDes = std::make_unique(); + // break; + // case SerDesKind::Tensorflow: + // SerDes = std::make_unique(); + // break; + case SerDesKind::Pytorch: + SerDes = std::make_unique(); break; #endif case SerDesKind::Unknown: diff --git a/include/MLModelRunner/ONNXModelRunner/ONNXModelRunner.h b/include/MLModelRunner/ONNXModelRunner/ONNXModelRunner.h old mode 100755 new mode 100644 diff --git a/include/MLModelRunner/ONNXModelRunner/utils.h b/include/MLModelRunner/ONNXModelRunner/utils.h old mode 100755 new mode 100644 diff --git a/include/MLModelRunner/PTModelRunner.h b/include/MLModelRunner/PTModelRunner.h index 2467217..3128b26 100644 --- a/include/MLModelRunner/PTModelRunner.h +++ b/include/MLModelRunner/PTModelRunner.h @@ -16,16 +16,13 @@ namespace MLBridge { /// PTModelRunner - PyTorch Compiled model implementation of the /// MLModelRunner. It uses an AOT-compiled model for efficient execution. -template class PTModelRunner final : public MLModelRunner { +class PTModelRunner final : public MLModelRunner { public: // New constructor that takes the model path as an input PTModelRunner(const std::string &modelPath, llvm::LLVMContext &Ctx) - : MLModelRunner(MLModelRunner::Kind::PTAOT, BaseSerDes::Kind::Pytorch, &Ctx), - CompiledModel(std::make_unique(modelPath)) { - - SerDes->setRequest(CompiledModel.get()); - - assert(CompiledModel && "The CompiledModel should be valid"); + : MLModelRunner(MLModelRunner::Kind::PTAOT, SerDesKind::Pytorch, &Ctx) + { + this->SerDes = new PytorchSerDes(modelPath); } virtual ~PTModelRunner() = default; @@ -38,39 +35,51 @@ template class PTModelRunner final : public MLModelRunner { return R->getKind() == MLModelRunner::Kind::PTAOT; } -private: - void *evaluateUntyped() override; + - std::unique_ptr CompiledModel; -}; + template + void populateFeatures(const std::pair &var1, + const std::pair &...var2); -/// A mock class satisfying the interface expected by PTModelRunner for -/// its `TGen` parameter. Useful to avoid conditional compilation complexity, as -/// a compile-time replacement for a real AOT-ed model. -class NoopSavedModelImpl final { -#define NOOP_MODEL_ERRMSG \ - "The mock AOT-ed saved model is a compile-time stub and should not be " \ - "called." + void populateFeatures() {} -public: - NoopSavedModelImpl() = default; - int LookupArgIndex(const std::string &) { - llvm_unreachable(NOOP_MODEL_ERRMSG); - } - int LookupResultIndex(const std::string &) { - llvm_unreachable(NOOP_MODEL_ERRMSG); - } - void Run() { llvm_unreachable(NOOP_MODEL_ERRMSG); } - void *result_data(int) { llvm_unreachable(NOOP_MODEL_ERRMSG); } - void *arg_data(int) { llvm_unreachable(NOOP_MODEL_ERRMSG); } -#undef NOOP_MODEL_ERRMSG + + void *evaluateUntyped() override; + PytorchSerDes *SerDes; }; -template bool isEmbeddedModelEvaluatorValid() { return true; } -template <> inline bool isEmbeddedModelEvaluatorValid() { - return false; +void* PTModelRunner::evaluateUntyped() { + + if ((*(this->SerDes->inputTensors)).empty()) { + llvm::errs() << "Input vector is empty.\n"; + return nullptr; + } + + try { + // Run the model with the input tensors + auto outputs = SerDes->CompiledModel->run((*(this->SerDes->inputTensors))); + + //Store the above output in the outputTensors, outputTensors is a pointer to the vector of tensors, already initialized in the constructor + for (auto i = outputs.begin(); i != outputs.end(); ++i) + (*(this->SerDes->outputTensors)).push_back(*i); + + // Convert to raw data format using deserializeUntyped + void* rawData = SerDes->deserializeUntyped(SerDes->outputTensors); + + return rawData; + } catch (const c10::Error& e) { + llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; + return nullptr; + } } + +template + void PTModelRunner::populateFeatures(const std::pair &var1, + const std::pair &...var2) { + SerDes->setFeature(var1.first, var1.second); + PTModelRunner::populateFeatures(var2...); + } } // namespace MLBridge #endif // TFMODELRUNNER_H diff --git a/include/MLModelRunner/PipeModelRunner.h b/include/MLModelRunner/PipeModelRunner.h old mode 100755 new mode 100644 diff --git a/include/SerDes/pytorchSerDes.h b/include/SerDes/pytorchSerDes.h index d74b1d0..88005f3 100644 --- a/include/SerDes/pytorchSerDes.h +++ b/include/SerDes/pytorchSerDes.h @@ -5,9 +5,9 @@ #include #include -#include -#include +// #include #include +#include namespace MLBridge { class PytorchSerDes : public BaseSerDes { @@ -16,7 +16,10 @@ class PytorchSerDes : public BaseSerDes { // Constructor that takes the model path and loads the model PytorchSerDes(const std::string &modelPath) : BaseSerDes(SerDesKind::Pytorch) { - CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); + c10::InferenceMode mode; + this->CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); + inputTensors = std::make_shared>(); + outputTensors = new std::vector(); } ~PytorchSerDes() { delete CompiledModel; } @@ -36,17 +39,23 @@ class PytorchSerDes : public BaseSerDes { reinterpret_cast(request); } + template + std::vector *copyTensorToVect(std::vector *serializedTensors) { + auto *ret = new std::vector(); + for (const auto &tensor : *serializedTensors) { + ret->insert(ret->end(), tensor.data_ptr(), tensor.data_ptr() + tensor.numel()); + } + return ret; + } + + void *getSerializedData() override; void cleanDataStructures() override; void *deserializeUntyped(void *data) override; - // void addTensorToInput(const torch::Tensor &newTensor); // Add a tensor to the - // input tensor vector - -private: - std::vector inputTensors; // Declaration of the input tensor - std::vector outputTensors; // Storage for the PyTorch output tensor - torch::inductor::AOTIModelContainerRunnerCpu *CompiledModel; + std::vector *outputTensors; // Storage for the PyTorch output tensor + std::shared_ptr> inputTensors; // Declaration of the input tensor + torch::inductor::AOTIModelContainerRunnerCpu* CompiledModel; }; } // namespace MLBridge diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a8a2680..c37107e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,12 +1,17 @@ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ") + +find_package(Torch REQUIRED) add_library(MLBridgeCPPTest OBJECT MLBridgeTest.cpp) -file(GLOB MODEL_OBJECTS ${CMAKE_CURRENT_SOURCE_DIR}/tf_models/*.o) -foreach(MODEL_OBJECT ${MODEL_OBJECTS}) - target_link_libraries(MLBridgeCPPTest PRIVATE ${MODEL_OBJECT}) -endforeach() +# file(GLOB MODEL_OBJECTS ${CMAKE_CURRENT_SOURCE_DIR}/tf_models/*.o) + +# foreach(MODEL_OBJECT ${MODEL_OBJECTS}) +# target_link_libraries(MLBridgeCPPTest PRIVATE ${MODEL_OBJECT}) +# endforeach() target_link_libraries(MLBridgeCPPTest PRIVATE MLCompilerBridge ) target_include_directories(MLBridgeCPPTest PRIVATE ${CMAKE_BINARY_DIR}/include ${TENSORFLOW_AOT_PATH}/include ${CMAKE_CURRENT_SOURCE_DIR}/include) -target_link_libraries(MLBridgeCPPTest PRIVATE tf_xla_runtime) +target_link_libraries(MLBridgeCPPTest PRIVATE tf_xla_runtime "${TORCH_LIBRARIES}") + +set_property(TARGET MLBridgeCPPTest PROPERTY CXX_STANDARD 17) diff --git a/test/MLBridgeTest.cpp b/test/MLBridgeTest.cpp index f879de9..3150eab 100644 --- a/test/MLBridgeTest.cpp +++ b/test/MLBridgeTest.cpp @@ -5,16 +5,10 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -#include "HelloMLBridge_Env.h" #include "MLModelRunner/MLModelRunner.h" -#include "MLModelRunner/ONNXModelRunner/ONNXModelRunner.h" -#include "MLModelRunner/PipeModelRunner.h" -#include "MLModelRunner/TFModelRunner.h" -#include "MLModelRunner/Utils/DataTypes.h" -#include "MLModelRunner/Utils/MLConfig.h" -#include "MLModelRunner/gRPCModelRunner.h" -#include "ProtosInclude.h" +#include "MLModelRunner/PTModelRunner.h" +// #include "MLModelRunner/TFModelRunner.h" +#include "SerDes/pytorchSerDes.h" #include "llvm/Support/CommandLine.h" #include #include @@ -23,266 +17,41 @@ #include #include #include +#include +#include -#define debug_out \ - if (!silent) \ - std::cout -using namespace grpc; - -static llvm::cl::opt - cl_server_address("test-server-address", llvm::cl::Hidden, - llvm::cl::desc("Server address, format :"), - llvm::cl::init("")); - -static llvm::cl::opt - cl_pipe_name("test-pipe-name", llvm::cl::Hidden, llvm::cl::init(""), - llvm::cl::desc("Name for pipe file")); - -static llvm::cl::opt - cl_onnx_path("onnx-model-path", llvm::cl::Hidden, llvm::cl::init(""), - llvm::cl::desc("Path to onnx model")); - -static llvm::cl::opt cl_test_config( - "test-config", llvm::cl::Hidden, - llvm::cl::desc("Method for communication with python model")); - -static llvm::cl::opt - silent("silent", llvm::cl::Hidden, llvm::cl::init(false), - llvm::cl::desc("Only print errors when if set to true")); - -namespace { -std::string basename; -SerDesKind SerDesType; - -std::string test_config; -std::string pipe_name; -std::string server_address; -std::string onnx_path; - -// send value of type T1. Test received value of type T2 against expected value -template -void testPrimitive(MLRunnerTy &MLRunner, std::string label, T1 value, - T2 expected) { - std::pair p("request_" + label, value); - MLRunner->populateFeatures(p); - T2 out = MLRunner->template evaluate(); - debug_out << " " << label << " reply: " << out << "\n"; - if (std::abs(out - expected) > 10e-6) { - std::cerr << "Error: Expected " << label << " reply: " << expected - << ", Received: " << out << "\n"; - exit(1); - } -} - -template -void testVector(MLRunnerTy &MLRunner, std::string label, std::vector value, - std::vector expected) { - std::pair> p("request_" + label, value); - MLRunner->populateFeatures(p); - T2 *out; - size_t size; - MLRunner->template evaluate(out, size); - std::vector reply(out, out + size); - debug_out << " " << label << " reply: "; - int i = 0; - for (auto x : reply) { - debug_out << x << " "; - if (std::abs(x - expected[i]) > 10e-6) { - std::cerr << "Error: Expected " << label << " reply: " << expected[i] - << ", Received: " << x << "\n"; - exit(1); - } - i++; - } - debug_out << "\n"; -} - -int testPipeBytes() { - if (pipe_name == "") { - std::cerr - << "Pipe name must be specified via --test-pipe-name=\n"; - exit(1); - } - basename = "./" + pipe_name; - SerDesType = SerDesKind::Bitstream; - auto MLRunner = std::make_unique( - basename + ".out", basename + ".in", SerDesType, nullptr); - testPrimitive(MLRunner, "int", 11, 12); - testPrimitive(MLRunner, "long", 1234567890l, 1234567891l); - testPrimitive(MLRunner, "float", 3.14f, 4.14f); - testPrimitive(MLRunner, "double", 0.123456789123456789, 1.123456789123456789); - testPrimitive(MLRunner, "char", 'a', 'b'); - testPrimitive(MLRunner, "bool", true, false); - testVector(MLRunner, "vec_int", std::vector{11, 22, 33}, - std::vector{12, 23, 34}); - testVector(MLRunner, "vec_long", std::vector{123456780, 222, 333}, - std::vector{123456780, 123456781, 123456782}); - testVector(MLRunner, "vec_float", std::vector{11.1, 22.2, 33.3}, - std::vector{1.11, 2.22, -3.33, 0}); - testVector(MLRunner, "vec_double", - std::vector{-1.1111111111, -2.2222222222, -3.3333333333}, - std::vector{1.12345678912345670, -1.12345678912345671}); - return 0; -} - -int testPipeJSON() { - if (pipe_name == "") { - std::cerr - << "Pipe name must be specified via --test-pipe-name=\n"; - exit(1); - } - basename = "./" + pipe_name; - SerDesType = SerDesKind::Json; - auto MLRunner = std::make_unique( - basename + ".out", basename + ".in", SerDesType, nullptr); - testPrimitive(MLRunner, "int", 11, 12); - testPrimitive(MLRunner, "long", 1234567890l, 12345l); - testPrimitive(MLRunner, "float", 3.14f, 4.14f); - - // FIXME: doesn't work if expected value is double - testPrimitive(MLRunner, "double", 0.123456789123456789, - 1.123456789123456789f); - testPrimitive(MLRunner, "char", 'a', 'b'); - testPrimitive(MLRunner, "bool", true, false); - testVector(MLRunner, "vec_int", std::vector{11, 22, 33}, - std::vector{12, 23, 34}); - - // FIXME: doesn't work if expected value is long - testVector(MLRunner, "vec_long", std::vector{123456780, 222, 333}, - std::vector{6780, 6781, 6782}); - testVector(MLRunner, "vec_float", std::vector{11.1, 22.2, 33.3}, - std::vector{1.11, 2.22, -3.33, 0}); - - // FIXME: doesn't work if expected value is double - testVector(MLRunner, "vec_double", - std::vector{-1.1111111111, -2.2222222222, -3.3333333333}, - std::vector{1.12345678912345670, -1.12345678912345671}); - return 0; -} - -void increment_port(int delta) { - int split = server_address.find(":"); - int port = stoi(server_address.substr(split + 1)); - server_address = - server_address.substr(0, split) + ":" + to_string(port + delta); -} - -int testGRPC() { -#define gRPCModelRunnerInit(datatype) \ - increment_port(1); \ - MLBridgeTestgRPC_##datatype::Reply response; \ - MLBridgeTestgRPC_##datatype::Request request; \ - auto MLRunner = std::make_unique< \ - gRPCModelRunner>( \ - server_address, &request, &response, nullptr); \ - MLRunner->setRequest(&request); \ - MLRunner->setResponse(&response) - - if (server_address == "") { - std::cerr << "Server Address must be specified via " - "--test-server-address=\":\"\n"; - exit(1); - } - { - gRPCModelRunnerInit(int); - testPrimitive(MLRunner, "int", 11, 12); - } - { - gRPCModelRunnerInit(long); - testPrimitive(MLRunner, "long", 1234567890l, 1234567891l); - } - { - gRPCModelRunnerInit(float); - testPrimitive(MLRunner, "float", 3.14f, 4.14f); - } - { - gRPCModelRunnerInit(double); - testPrimitive(MLRunner, "double", 0.123456789123456789, - 1.123456789123456789); - } - increment_port(1); - { - gRPCModelRunnerInit(bool); - testPrimitive(MLRunner, "bool", true, false); - } - { - gRPCModelRunnerInit(vec_int); - testVector(MLRunner, "vec_int", std::vector{11, 22, 33}, - std::vector{12, 23, 34}); - } - { - gRPCModelRunnerInit(vec_long); - testVector(MLRunner, "vec_long", std::vector{123456780, 222, 333}, - std::vector{123456780, 123456781, 123456782}); - } - { - gRPCModelRunnerInit(vec_float); - testVector(MLRunner, "vec_float", std::vector{11.1, 22.2, 33.3}, - std::vector{1.11, 2.22, -3.33, 0}); - } - { - gRPCModelRunnerInit(vec_double); - testVector(MLRunner, "vec_double", - std::vector{-1.1111111111, -2.2222222222, -3.3333333333}, - std::vector{1.12345678912345670, -1.12345678912345671}); - } -#undef gRPCModelRunnerInit - return 0; -} - -class ONNXTest : public MLBridgeTestEnv { -public: - int run(int expectedAction) { - onnx_path = cl_onnx_path.getValue(); - if (onnx_path == "") { - std::cerr << "ONNX model path must be specified via " - "--onnx-model-path=\n"; - exit(1); - } - FeatureVector.clear(); - int n = 100; - for (int i = 0; i < n; i++) { - float delta = (float)(i - expectedAction) / n; - FeatureVector.push_back(delta * delta); - } - - Agent *agent = new Agent(onnx_path); - std::map agents; - agents["agent"] = agent; - auto MLRunner = std::make_unique(this, agents, nullptr); - MLRunner->evaluate(); - if (lastAction != expectedAction) { - std::cerr << "Error: Expected action: " << expectedAction - << ", Computed action: " << lastAction << "\n"; - exit(1); - } - return 0; - } -}; - -} // namespace +using namespace MLBridge; +using namespace std; int main(int argc, char **argv) { - llvm::cl::ParseCommandLineOptions(argc, argv); - test_config = cl_test_config.getValue(); - - if (test_config == "pipe-bytes") { - pipe_name = cl_pipe_name.getValue(); - testPipeBytes(); - } else if (test_config == "pipe-json") { - pipe_name = cl_pipe_name.getValue(); - testPipeJSON(); - } else if (test_config == "grpc") { - server_address = cl_server_address.getValue(); - testGRPC(); - } else if (test_config == "onnx") { - ONNXTest t; - t.run(20); - } else - std::cerr << "--test-config must be provided from [pipe-bytes, pipe-json, " - "grpc, onnx]\n"; + + printf("Starting PTModelRunner test...\n"); + llvm::LLVMContext Ctx; + + string modelPath = "model.so"; + unique_ptr runner = make_unique(modelPath, Ctx); + + // Create some input data + vector inputData(10, 0.5f); // Example input with size 10 + + basic_string input_str = "input"; + pair< string, vector& > inputPair = make_pair(input_str, ref(inputData)); + + runner->populateFeatures(inputPair); + + void* result = runner->evaluateUntyped(); + + if (result) { + cout << "Model evaluation succeeded." << endl; + vector *output = reinterpret_cast *>(result); + cout << "Output: "; + for (auto &v : *output) { + cout << v << " "; + } + cout << endl; + + } else { + cerr << "Model evaluation failed." << endl; + } return 0; } From f5b2070fb87a896381eef70dbea96f1a2d3ef519 Mon Sep 17 00:00:00 2001 From: ashwinagrl Date: Fri, 10 Jan 2025 22:07:25 +0530 Subject: [PATCH 5/6] some changes --- MLModelRunner/gRPCModelRunner/CMakeLists.txt | 2 +- include/MLModelRunner/PTModelRunner.h | 22 ++--- test/MLBridgeTest.cpp | 95 +++++++++++++++----- 3 files changed, 77 insertions(+), 42 deletions(-) diff --git a/MLModelRunner/gRPCModelRunner/CMakeLists.txt b/MLModelRunner/gRPCModelRunner/CMakeLists.txt index 95b5b6e..3687585 100644 --- a/MLModelRunner/gRPCModelRunner/CMakeLists.txt +++ b/MLModelRunner/gRPCModelRunner/CMakeLists.txt @@ -13,7 +13,7 @@ endif() # Find gRPC installation # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. -find_package(gRPC 1.66.0 EXACT CONFIG REQUIRED) +find_package(gRPC 1.34.0 EXACT CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") set(_GRPC_GRPCPP gRPC::grpc++) diff --git a/include/MLModelRunner/PTModelRunner.h b/include/MLModelRunner/PTModelRunner.h index 3128b26..bd9cb93 100644 --- a/include/MLModelRunner/PTModelRunner.h +++ b/include/MLModelRunner/PTModelRunner.h @@ -50,24 +50,12 @@ class PTModelRunner final : public MLModelRunner { void* PTModelRunner::evaluateUntyped() { +auto outputs = SerDes->CompiledModel->run((*(this->SerDes->inputTensors))); +for (auto i = outputs.begin(); i != outputs.end(); ++i) + (*(this->SerDes->outputTensors)).push_back(*i); +void* rawData = SerDes->deserializeUntyped(SerDes->outputTensors); +return rawData; - if ((*(this->SerDes->inputTensors)).empty()) { - llvm::errs() << "Input vector is empty.\n"; - return nullptr; - } - - try { - // Run the model with the input tensors - auto outputs = SerDes->CompiledModel->run((*(this->SerDes->inputTensors))); - - //Store the above output in the outputTensors, outputTensors is a pointer to the vector of tensors, already initialized in the constructor - for (auto i = outputs.begin(); i != outputs.end(); ++i) - (*(this->SerDes->outputTensors)).push_back(*i); - - // Convert to raw data format using deserializeUntyped - void* rawData = SerDes->deserializeUntyped(SerDes->outputTensors); - - return rawData; } catch (const c10::Error& e) { llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; return nullptr; diff --git a/test/MLBridgeTest.cpp b/test/MLBridgeTest.cpp index 3150eab..f7a6715 100644 --- a/test/MLBridgeTest.cpp +++ b/test/MLBridgeTest.cpp @@ -5,53 +5,100 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// + #include "MLModelRunner/MLModelRunner.h" #include "MLModelRunner/PTModelRunner.h" -// #include "MLModelRunner/TFModelRunner.h" #include "SerDes/pytorchSerDes.h" #include "llvm/Support/CommandLine.h" #include +#include #include #include #include #include #include #include -#include -#include +#include using namespace MLBridge; using namespace std; -int main(int argc, char **argv) { - - printf("Starting PTModelRunner test...\n"); - llvm::LLVMContext Ctx; +int main(int argc, char **argv) +{ + if (argc != 2) + { + cerr << "Usage: " << argv[0] << " " << endl; + return 1; + } - string modelPath = "model.so"; - unique_ptr runner = make_unique(modelPath, Ctx); + int sz = stoi(argv[1]); + cout << "Running inference for size: " << sz << endl; + + // Create random input data of size sz + vector inputData(sz); + random_device rd; + mt19937 gen(rd()); + uniform_real_distribution dis(0.0, 1.0); + generate(inputData.begin(), inputData.end(), [&]() + { return dis(gen); }); + + llvm::LLVMContext Ctx; + string modelPath = "models_pt/model_" + to_string(sz) + ".so"; - // Create some input data - vector inputData(10, 0.5f); // Example input with size 10 + try + { + // Start measuring time just before loading the model + auto start = chrono::high_resolution_clock::now(); + unique_ptr runner = make_unique(modelPath, Ctx); + + // Prepare the input basic_string input_str = "input"; - pair< string, vector& > inputPair = make_pair(input_str, ref(inputData)); - + pair &> inputPair = make_pair(input_str, ref(inputData)); runner->populateFeatures(inputPair); - void* result = runner->evaluateUntyped(); + // Perform inference + void *result = runner->evaluateUntyped(); - if (result) { - cout << "Model evaluation succeeded." << endl; - vector *output = reinterpret_cast *>(result); - cout << "Output: "; - for (auto &v : *output) { - cout << v << " "; - } - cout << endl; + auto end = chrono::high_resolution_clock::now(); + chrono::duration inferenceTime = end - start; - } else { - cerr << "Model evaluation failed." << endl; + // Check result + if (result) + { + vector *output = reinterpret_cast *>(result); + cout << "Output: "; + for (auto &v : *output) + { + cout << v << " "; + } + cout << endl; + } + else + { + cerr << "Model evaluation failed." << endl; + return 1; } + + // // Log the results + // ofstream resultsFile("results.csv", ios::app); + // if (resultsFile.is_open()) + // { + // resultsFile << sz << "," << inferenceTime.count() << endl; + // resultsFile.close(); + // } + // else + // { + // cerr << "Failed to open results.csv" << endl; + // } + + cout << "Inference completed in: " << inferenceTime.count() << " ms" << endl; + } + catch (const exception &ex) + { + cerr << "Error: " << ex.what() << endl; + return 1; + } + return 0; } From c9578d31ebde40440464f0a2f4890c68fe733d73 Mon Sep 17 00:00:00 2001 From: ashwinagrl Date: Sun, 26 Jan 2025 02:21:07 +0530 Subject: [PATCH 6/6] Added compiled model object to PTModelRunner class --- include/MLModelRunner/PTModelRunner.h | 102 +++++++++++++++----------- include/SerDes/pytorchSerDes.h | 14 +--- test/MLBridgeTest.cpp | 2 +- 3 files changed, 64 insertions(+), 54 deletions(-) diff --git a/include/MLModelRunner/PTModelRunner.h b/include/MLModelRunner/PTModelRunner.h index bd9cb93..09eaf1a 100644 --- a/include/MLModelRunner/PTModelRunner.h +++ b/include/MLModelRunner/PTModelRunner.h @@ -11,63 +11,79 @@ #include #include +namespace MLBridge +{ + + /// PTModelRunner - PyTorch Compiled model implementation of the + /// MLModelRunner. It uses an AOT-compiled model for efficient execution. + class PTModelRunner final : public MLModelRunner + { + public: + // New constructor that takes the model path as an input + PTModelRunner(const std::string &modelPath, llvm::LLVMContext &Ctx) + : MLModelRunner(MLModelRunner::Kind::PTAOT, SerDesKind::Pytorch, &Ctx) + { + this->SerDes = new PytorchSerDes(); + + c10::InferenceMode mode; + this->CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); + } -namespace MLBridge { - -/// PTModelRunner - PyTorch Compiled model implementation of the -/// MLModelRunner. It uses an AOT-compiled model for efficient execution. -class PTModelRunner final : public MLModelRunner { -public: - // New constructor that takes the model path as an input - PTModelRunner(const std::string &modelPath, llvm::LLVMContext &Ctx) - : MLModelRunner(MLModelRunner::Kind::PTAOT, SerDesKind::Pytorch, &Ctx) - { - this->SerDes = new PytorchSerDes(modelPath); - } - - virtual ~PTModelRunner() = default; - - virtual void requestExit() override { - llvm_unreachable("requestExit() is not supported in PTModelRunner"); - } - - static bool classof(const MLModelRunner *R) { - return R->getKind() == MLModelRunner::Kind::PTAOT; - } + virtual ~PTModelRunner() = default; - + virtual void requestExit() override + { + llvm_unreachable("requestExit() is not supported in PTModelRunner"); + } - template - void populateFeatures(const std::pair &var1, - const std::pair &...var2); + static bool classof(const MLModelRunner *R) + { + return R->getKind() == MLModelRunner::Kind::PTAOT; + } - void populateFeatures() {} + template + void populateFeatures(const std::pair &var1, + const std::pair &...var2); + void populateFeatures() {} - void *evaluateUntyped() override; - PytorchSerDes *SerDes; -}; + void *evaluateUntyped() override; + PytorchSerDes *SerDes; + // Compiled model container added to the PTModelRunner + torch::inductor::AOTIModelContainerRunnerCpu *CompiledModel; + }; + void *PTModelRunner::evaluateUntyped() + { -void* PTModelRunner::evaluateUntyped() { -auto outputs = SerDes->CompiledModel->run((*(this->SerDes->inputTensors))); -for (auto i = outputs.begin(); i != outputs.end(); ++i) - (*(this->SerDes->outputTensors)).push_back(*i); -void* rawData = SerDes->deserializeUntyped(SerDes->outputTensors); -return rawData; + if ((*(this->SerDes->inputTensors)).empty()) + { + llvm::errs() << "Input vector is empty.\n"; + return nullptr; + } - } catch (const c10::Error& e) { - llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; - return nullptr; + try + { + auto outputs = this->CompiledModel->run((*(this->SerDes->inputTensors))); + for (auto i = outputs.begin(); i != outputs.end(); ++i) + (*(this->SerDes->outputTensors)).push_back(*i); + void *rawData = this->SerDes->deserializeUntyped(this->SerDes->outputTensors); + return rawData; } -} + catch (const c10::Error &e) + { + llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; + return nullptr; + } + } -template + template void PTModelRunner::populateFeatures(const std::pair &var1, - const std::pair &...var2) { + const std::pair &...var2) + { SerDes->setFeature(var1.first, var1.second); PTModelRunner::populateFeatures(var2...); - } + } } // namespace MLBridge #endif // TFMODELRUNNER_H diff --git a/include/SerDes/pytorchSerDes.h b/include/SerDes/pytorchSerDes.h index 88005f3..66845ec 100644 --- a/include/SerDes/pytorchSerDes.h +++ b/include/SerDes/pytorchSerDes.h @@ -12,17 +12,13 @@ namespace MLBridge { class PytorchSerDes : public BaseSerDes { public: - PytorchSerDes() : BaseSerDes(SerDesKind::Pytorch) {} - // Constructor that takes the model path and loads the model - PytorchSerDes(const std::string &modelPath) : BaseSerDes(SerDesKind::Pytorch) { - c10::InferenceMode mode; - this->CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); + PytorchSerDes() : BaseSerDes(SerDesKind::Pytorch) { inputTensors = std::make_shared>(); outputTensors = new std::vector(); } - ~PytorchSerDes() { delete CompiledModel; } + ~PytorchSerDes() {} static bool classof(const BaseSerDes *S) { return S->getKind() == SerDesKind::Pytorch; @@ -34,9 +30,7 @@ class PytorchSerDes : public BaseSerDes { SUPPORTED_TYPES(SET_FEATURE) #undef SET_FEATURE - void setRequest(void *request) override { - CompiledModel = - reinterpret_cast(request); + void setRequest(void *request){ } template @@ -55,7 +49,7 @@ class PytorchSerDes : public BaseSerDes { std::vector *outputTensors; // Storage for the PyTorch output tensor std::shared_ptr> inputTensors; // Declaration of the input tensor - torch::inductor::AOTIModelContainerRunnerCpu* CompiledModel; + }; } // namespace MLBridge diff --git a/test/MLBridgeTest.cpp b/test/MLBridgeTest.cpp index f7a6715..e8c5108 100644 --- a/test/MLBridgeTest.cpp +++ b/test/MLBridgeTest.cpp @@ -67,7 +67,7 @@ int main(int argc, char **argv) if (result) { vector *output = reinterpret_cast *>(result); - cout << "Output: "; + cout << "Output from the model: "; for (auto &v : *output) { cout << v << " ";