Merge pull request #44 from janhq/36-build-and-run-on-linux-amd64-with-cuda-enabled

CUDA support for Nitro on amd64 - linux

Author: tikikun

CMakeLists.txt

@@ -20,6 +20,16 @@ set(OPENSSL_USE_STATIC_LIBS TRUE)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/build_deps/_install) # This is the critical line for installing another package
 
+if (LLAMA_CUBLAS)
+    cmake_minimum_required(VERSION 3.17)
+
+    find_package(CUDAToolkit)
+    if (CUDAToolkit_FOUND)
+        message(STATUS "cuBLAS found")
+          add_compile_definitions(GGML_USE_CUBLAS)
+    endif()
+endif()
+
 add_subdirectory(llama.cpp)
 add_executable(${PROJECT_NAME} main.cc)
 

controllers/llamaCPP.cc

@@ -4,6 +4,8 @@
 #include <chrono>
 #include <cstring>
 #include <thread>
+#include <regex>
+
 std::string create_return_json(const std::string &id, const std::string &model,
                                const std::string &content,
                                Json::Value finish_reason = Json::Value()) {

controllers/llamaCPP.h

@@ -7,6 +7,7 @@
 #include "llama.h"
 #include "build-info.h"
 #include "grammar-parser.h"
+#include <regex>
 
 #ifndef NDEBUG
 // crash the server in debug mode, otherwise send an http 500 error
@@ -1385,6 +1386,10 @@ class llamaCPP : public drogon::HttpSimpleController<llamaCPP> {
     params.model = conf["llama_model_path"].asString();
     params.n_gpu_layers = conf["ngl"].asInt();
     params.n_ctx = conf["ctx_len"].asInt();
+    #ifdef GGML_USE_CUBLAS
+    LOG_INFO << "Setting up GGML CUBLAS PARAMS";
+    params.mul_mat_q = false;
+    #endif // GGML_USE_CUBLAS
     if (params.model_alias == "unknown") {
       params.model_alias = params.model;
     }

controllers/nitro_utils.h

@@ -50,7 +50,7 @@ inline void nitro_logo(){
             std::cout << resetColor << c;
             colorIndex = 0;
         } else {
-            std::cout << rainbowColors[colorIndex % 6] << c;
+            std::cout << rainbowColors[colorIndex % 2] << c;
             colorIndex++;
         }
     }