Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 12f9a52

Browse files
namchuainamchuai
committed
update
1 parent f9b7567 commit 12f9a52

File tree

7 files changed

+120
-83
lines changed

7 files changed

+120
-83
lines changed

‎engine/controllers/engines.cc‎

Lines changed: 55 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
#include "services/engine_service.h"
33
#include "utils/archive_utils.h"
44
#include "utils/cortex_utils.h"
5+
#include "utils/engine_constants.h"
56
#include "utils/logging_utils.h"
67
#include "utils/string_utils.h"
78

89
namespace {
910
// Need to change this after we rename repositories
10-
// TODO: namh try to remove this
1111
std::string NormalizeEngine(const std::string& engine) {
1212
if (engine == kLlamaEngine) {
1313
return kLlamaRepo;
@@ -55,55 +55,26 @@ void Engines::InstallEngine(
5555
void Engines::ListEngine(
5656
const HttpRequestPtr& req,
5757
std::function<void(const HttpResponsePtr&)>&& callback) const {
58-
// TODO: NamH need refactor this
59-
// auto status_list = engine_service_->GetEngineInfoList();
60-
58+
std::vector<std::string> supported_engines{kLlamaEngine, kOnnxEngine,
59+
kTrtLlmEngine};
6160
Json::Value ret;
62-
ret["object"] = "list";
63-
// Json::Value data(Json::arrayValue);
64-
// for (auto& status : status_list) {
65-
// Json::Value ret;
66-
// ret["name"] = status.name;
67-
// ret["description"] = status.description;
68-
// ret["version"] = status.version.value_or("");
69-
// ret["variant"] = status.variant.value_or("");
70-
// ret["productName"] = status.product_name;
71-
// ret["status"] = status.status;
72-
// ret["format"] = status.format;
73-
//
74-
// data.append(std::move(ret));
75-
// }
61+
for (const auto& engine : supported_engines) {
62+
std::cout << engine << std::endl;
7663

77-
// ret["data"] = data;
78-
ret["result"] = "OK";
79-
auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
80-
resp->setStatusCode(k200OK);
81-
callback(resp);
82-
}
64+
auto result = engine_service_->GetInstalledEngineVariants(engine);
65+
if (result.has_error()) {
66+
continue;
67+
}
68+
Json::Value variants(Json::arrayValue);
69+
for (const auto& variant : result.value()) {
70+
variants.append(variant.ToJson());
71+
}
72+
ret[engine] = variants;
73+
}
8374

84-
void Engines::GetEngine(const HttpRequestPtr& req,
85-
std::function<void(const HttpResponsePtr&)>&& callback,
86-
const std::string& engine) const {
87-
// auto status = engine_service_->GetEngineInfo(engine);
88-
Json::Value ret;
89-
// if (status.has_value()) {
90-
// ret["name"] = status->name;
91-
// ret["description"] = status->description;
92-
// ret["version"] = status->version.value_or("");
93-
// ret["variant"] = status->variant.value_or("");
94-
// ret["productName"] = status->product_name;
95-
// ret["status"] = status->status;
96-
// ret["format"] = status->format;
97-
//
98-
// auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
99-
// resp->setStatusCode(k200OK);
100-
// callback(resp);
101-
// } else {
102-
ret["message"] = "Engine not found";
10375
auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
104-
resp->setStatusCode(k400BadRequest);
76+
resp->setStatusCode(k200OK);
10577
callback(resp);
106-
// }
10778
}
10879

10980
void Engines::UninstallEngine(
@@ -309,3 +280,42 @@ void Engines::GetDefaultEngineVariant(
309280
callback(resp);
310281
}
311282
}
283+
284+
void Engines::LoadEngine(const HttpRequestPtr& req,
285+
std::function<void(const HttpResponsePtr&)>&& callback,
286+
const std::string& engine) {
287+
auto result = engine_service_->LoadEngine(engine);
288+
if (result.has_error()) {
289+
Json::Value res;
290+
res["message"] = result.error();
291+
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
292+
resp->setStatusCode(k400BadRequest);
293+
callback(resp);
294+
} else {
295+
Json::Value res;
296+
res["message"] = "Engine " + engine + " loaded successfully!";
297+
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
298+
resp->setStatusCode(k200OK);
299+
callback(resp);
300+
}
301+
}
302+
303+
void Engines::UnloadEngine(
304+
const HttpRequestPtr& req,
305+
std::function<void(const HttpResponsePtr&)>&& callback,
306+
const std::string& engine) {
307+
auto result = engine_service_->UnloadEngine(engine);
308+
if (result.has_error()) {
309+
Json::Value res;
310+
res["message"] = result.error();
311+
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
312+
resp->setStatusCode(k400BadRequest);
313+
callback(resp);
314+
} else {
315+
Json::Value res;
316+
res["message"] = "Engine " + engine + " unloaded successfully!";
317+
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
318+
resp->setStatusCode(k200OK);
319+
callback(resp);
320+
}
321+
}

‎engine/controllers/engines.h‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ class Engines : public drogon::HttpController<Engines, false> {
1717
METHOD_ADD(Engines::UninstallEngine, "/{1}", Delete);
1818
METHOD_ADD(Engines::ListEngine, "", Get);
1919

20-
// TODO: might better use query param
2120
METHOD_ADD(Engines::GetEngineVersions, "/{1}/versions", Get);
2221
METHOD_ADD(Engines::GetEngineVariants, "/{1}/versions/{2}", Get);
2322
METHOD_ADD(Engines::InstallEngineVariant, "/{1}/versions/{2}/{3}", Post);
@@ -28,10 +27,11 @@ class Engines : public drogon::HttpController<Engines, false> {
2827
METHOD_ADD(Engines::SetDefaultEngineVariant, "/{1}/default/{2}/{3}", Post);
2928
METHOD_ADD(Engines::GetDefaultEngineVariant, "/{1}/default", Get);
3029

30+
METHOD_ADD(Engines::LoadEngine, "/{1}/load", Post);
31+
METHOD_ADD(Engines::UnloadEngine, "/{1}/load", Delete);
32+
3133
ADD_METHOD_TO(Engines::InstallEngine, "/v1/engines/install/{1}", Post);
3234
ADD_METHOD_TO(Engines::UninstallEngine, "/v1/engines/{1}", Delete);
33-
// TODO: update response of this API
34-
ADD_METHOD_TO(Engines::ListEngine, "/v1/engines", Get);
3535

3636
METHOD_LIST_END
3737

@@ -45,10 +45,6 @@ class Engines : public drogon::HttpController<Engines, false> {
4545
void ListEngine(const HttpRequestPtr& req,
4646
std::function<void(const HttpResponsePtr&)>&& callback) const;
4747

48-
void GetEngine(const HttpRequestPtr& req,
49-
std::function<void(const HttpResponsePtr&)>&& callback,
50-
const std::string& engine) const;
51-
5248
void UninstallEngine(const HttpRequestPtr& req,
5349
std::function<void(const HttpResponsePtr&)>&& callback,
5450
const std::string& engine);
@@ -93,6 +89,14 @@ class Engines : public drogon::HttpController<Engines, false> {
9389
std::function<void(const HttpResponsePtr&)>&& callback,
9490
const std::string& engine) const;
9591

92+
void LoadEngine(const HttpRequestPtr& req,
93+
std::function<void(const HttpResponsePtr&)>&& callback,
94+
const std::string& engine);
95+
96+
void UnloadEngine(const HttpRequestPtr& req,
97+
std::function<void(const HttpResponsePtr&)>&& callback,
98+
const std::string& engine);
99+
96100
private:
97101
std::shared_ptr<EngineService> engine_service_;
98102
};

‎engine/main.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,11 @@ void RunServer(std::optional<int> port) {
8989

9090
auto event_queue_ptr = std::make_shared<EventQueue>();
9191
cortex::event::EventProcessor event_processor(event_queue_ptr);
92-
auto inference_svc = std::make_shared<services::InferenceService>();
9392

9493
auto download_service = std::make_shared<DownloadService>(event_queue_ptr);
9594
auto engine_service = std::make_shared<EngineService>(download_service);
95+
auto inference_svc =
96+
std::make_shared<services::InferenceService>(engine_service);
9697
auto model_service =
9798
std::make_shared<ModelService>(download_service, inference_svc);
9899

‎engine/services/engine_service.cc‎

Lines changed: 47 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#include "engine_service.h"
2+
#include <cstdlib>
3+
#include <filesystem>
24
#include <iostream>
35
#include <optional>
46
#include "algorithm"
@@ -644,7 +646,6 @@ EngineService::GetDefaultEngineVariant(const std::string& engine) {
644646
cpp::result<std::vector<EngineVariantResponse>, std::string>
645647
EngineService::GetInstalledEngineVariants(const std::string& engine) const {
646648
auto ne = NormalizeEngine(engine);
647-
648649
auto engines_variants_dir =
649650
file_manager_utils::GetEnginesContainerPath() / ne;
650651

@@ -708,18 +709,40 @@ cpp::result<void, std::string> EngineService::LoadEngine(
708709
return {};
709710
}
710711

711-
try {
712-
if (ne == kLlamaRepo) {
713-
cortex::cpuid::CpuInfo cpu_info;
714-
LOG_INFO << "CPU instruction set: " << cpu_info.to_string();
712+
CTL_INF("Loading engine: " << ne);
713+
714+
auto selected_engine_variant = GetDefaultEngineVariant(ne);
715+
716+
if (selected_engine_variant.has_error()) {
717+
// TODO: namh need to fallback
718+
return cpp::fail(selected_engine_variant.error());
719+
}
720+
721+
CTL_INF("Selected engine variant: "
722+
<< json_helper::DumpJsonString(selected_engine_variant->ToJson()));
723+
724+
auto user_defined_engine_path = getenv("ENGINE_PATH");
725+
const std::filesystem::path engine_dir_path = [&] {
726+
if (user_defined_engine_path != nullptr) {
727+
// for backward compatible
728+
return std::filesystem::path(user_defined_engine_path +
729+
GetEnginePath(ne));
730+
} else {
731+
return file_manager_utils::GetEnginesContainerPath() / ne /
732+
selected_engine_variant->variant /
733+
selected_engine_variant->version;
715734
}
735+
}();
736+
737+
if (!std::filesystem::exists(engine_dir_path)) {
738+
CTL_ERR("Directory " + engine_dir_path.string() + " is not exist!");
739+
return cpp::fail("Directory " + engine_dir_path.string() +
740+
" is not exist!");
741+
}
742+
743+
CTL_INF("Engine path: " << engine_dir_path.string());
716744

717-
std::string abs_path =
718-
(getenv("ENGINE_PATH")
719-
? getenv("ENGINE_PATH")
720-
: file_manager_utils::GetCortexDataPath().string()) +
721-
GetEnginePath(ne);
722-
LOG_INFO << "engine path: " << abs_path;
745+
try {
723746
#if defined(_WIN32)
724747
// TODO(?) If we only allow to load an engine at a time, the logic is simpler.
725748
// We would like to support running multiple engines at the same time. Therefore,
@@ -748,14 +771,15 @@ cpp::result<void, std::string> EngineService::LoadEngine(
748771
LOG_INFO << "Removed dll directory: " << kLlamaRepo;
749772
}
750773

751-
add_dll(ne, abs_path);
774+
add_dll(ne, engine_dir_path.string());
752775
} else if (IsEngineLoaded(kTrtLlmRepo) && ne == kLlamaRepo) {
753776
// Do nothing
754777
} else {
755-
add_dll(ne, abs_path);
778+
add_dll(ne, engine_dir_path.string());
756779
}
757780
#endif
758-
engines_[ne].dl = std::make_unique<cortex_cpp::dylib>(abs_path, "engine");
781+
engines_[ne].dl =
782+
std::make_unique<cortex_cpp::dylib>(engine_dir_path.string(), "engine");
759783

760784
} catch (const cortex_cpp::dylib::load_error& e) {
761785
LOG_ERROR << "Could not load engine: " << e.what();
@@ -769,14 +793,15 @@ cpp::result<void, std::string> EngineService::LoadEngine(
769793
auto& en = std::get<EngineI*>(engines_[ne].engine);
770794
if (ne == kLlamaRepo) { //fix for llamacpp engine first
771795
auto config = file_manager_utils::GetCortexConfig();
772-
if (en->IsSupported("SetFileLogger")) {
773-
en->SetFileLogger(config.maxLogLines,
774-
(std::filesystem::path(config.logFolderPath) /
775-
std::filesystem::path(config.logLlamaCppPath))
776-
.string());
777-
} else {
778-
LOG_WARN << "Method SetFileLogger is not supported yet";
779-
}
796+
// TODO: crash issue with trantor logging destructor.
797+
// if (en->IsSupported("SetFileLogger")) {
798+
// en->SetFileLogger(config.maxLogLines,
799+
// (std::filesystem::path(config.logFolderPath) /
800+
// std::filesystem::path(config.logLlamaCppPath))
801+
// .string());
802+
// } else {
803+
// LOG_WARN << "Method SetFileLogger is not supported yet";
804+
// }
780805
}
781806
LOG_INFO << "Loaded engine: " << ne;
782807
return {};
@@ -790,7 +815,6 @@ cpp::result<void, std::string> EngineService::UnloadEngine(
790815
}
791816
EngineI* e = std::get<EngineI*>(engines_[ne].engine);
792817
delete e;
793-
// TODO: namh need unload model?
794818
#if defined(_WIN32)
795819
if (!RemoveDllDirectory(engines_[ne].cookie)) {
796820
LOG_WARN << "Could not remove dll directory: " << ne;

‎engine/services/engine_service.h‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ class EngineService {
6363
#endif
6464
};
6565

66-
std::unordered_map<std::string, EngineInfo> engines_;
66+
std::unordered_map<std::string, EngineInfo> engines_{};
6767

6868
public:
6969
constexpr static auto kIncompatible = "Incompatible";
@@ -78,10 +78,7 @@ class EngineService {
7878
hw_inf_{.sys_inf = system_info_utils::GetSystemInfo(),
7979
.cuda_driver_version = system_info_utils::GetCudaVersion()} {}
8080

81-
// cpp::result<EngineInfo, std::string> GetEngineInfo(
82-
// const std::string& engine) const;
83-
84-
// std::vector<EngineInfo> GetEngineInfoList() const;
81+
std::vector<EngineInfo> GetEngineInfoList() const;
8582

8683
cpp::result<bool, std::string> InstallEngine(
8784
const std::string& engine, const std::string& version = "latest",

‎engine/services/inference_service.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ struct SyncQueue {
3232

3333
class InferenceService {
3434
public:
35+
explicit InferenceService(std::shared_ptr<EngineService> engine_service)
36+
: engine_service_{engine_service} {}
37+
3538
cpp::result<void, InferResult> HandleChatCompletion(
3639
std::shared_ptr<SyncQueue> q, std::shared_ptr<Json::Value> json_body);
3740

‎engine/utils/archive_utils.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ inline bool ExtractArchive(const std::string& input_path,
1818
const std::string& destination_path,
1919
bool ignore_parent_dir = false) {
2020
if (input_path.find(".zip") != std::string::npos) {
21-
// TODO: namh add support ignore_parent_dir
2221
return UnzipFile(input_path, destination_path);
2322
} else if (input_path.find(".tar") != std::string::npos ||
2423
input_path.find(".tar.gz") != std::string::npos) {
@@ -122,7 +121,6 @@ inline bool UntarFile(const std::string& input_tar_path,
122121

123122
if (archive_entry_filetype(entry) == AE_IFDIR) {
124123
if (!ignore_parent_dir) {
125-
// TODO: namh recheck if this ignore creating child folders
126124
std::filesystem::create_directories(full_path);
127125
}
128126
} else {

0 commit comments

Comments
 (0)