Some minor fix in cmakelists (#4185)

* fix: upgrade cmake_minimum_required to 3.11

* fix: upgrade xgrammar and only use gold linker in linux

* fix: remove unused properties

Author: windreamer

CMakeLists.txt

@@ -83,7 +83,7 @@ FetchContent_MakeAvailable(yaml-cpp)
 FetchContent_Declare(
   xgrammar
   GIT_REPOSITORY          https://github.com/mlc-ai/xgrammar.git
-  GIT_TAG                 v0.1.25
+  GIT_TAG                 v0.1.27
   GIT_SUBMODULES          "3rdparty/dlpack"
   GIT_PROGRESS            TRUE
   USES_TERMINAL_DOWNLOAD  TRUE
@@ -162,8 +162,15 @@ endif()
 
 set(CXX_STD "17" CACHE STRING "C++ standard")
 # enable gold linker for binary and .so
-set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
-set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
+if(NOT MSVC)
+  find_program(GOLD_PATH ld.gold REQUIRED)
+  if(NOT GOLD_PATH)
+    message(FATAL_ERROR "GNU gold linker is required but not found. "
+                         "Please install binutils-gold package.")
+  endif()
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
+  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
+endif()
 set(CUDA_PATH ${CUDA_TOOLKIT_ROOT_DIR})
 
 set(CUSPARSELT_PATH "" CACHE STRING "cuSPARSELt path")

lmdeploy/turbomind/tokenizer_info.py

@@ -285,59 +285,3 @@ def from_huggingface(
         else:
             # TODO(yixin): unsupported tokenizer
             raise ValueError(f'Unsupported tokenizer type: {type(tokenizer)}')
-
-    @property
-    def vocab_type(self) -> VocabType:
-        """The type of the vocabulary."""
-        return VocabType(self._handle.vocab_type)
-
-    @property
-    def vocab_size(self) -> int:
-        """The size of the vocabulary."""
-        return self._handle.vocab_size
-
-    @property
-    def add_prefix_space(self) -> bool:
-        """Whether the tokenizer will prepend a space before the text in the
-        tokenization process."""
-        return self._handle.add_prefix_space
-
-    @property
-    def prepend_space_in_tokenization(self) -> bool:
-        """Whether the tokenizer will prepend a space before the text in the
-        tokenization process.
-
-        This property is deprecated. Use add_prefix_space instead.
-        """
-        logger.warning('prepend_space_in_tokenization is deprecated. Use add_prefix_space instead.')
-        return self.add_prefix_space
-
-    @property
-    def decoded_vocab(self) -> List[bytes]:
-        """The decoded vocabulary of the tokenizer.
-
-        This converts the tokens in the LLM's vocabulary back to the original format of the input text. E.g. for type
-        ByteFallback, the token <0x1B> is converted back to "\u001b".
-        """
-        return self._handle.decoded_vocab
-
-    @property
-    def stop_token_ids(self) -> List[int]:
-        """The stop token ids."""
-        return self._handle.stop_token_ids
-
-    @property
-    def special_token_ids(self) -> List[int]:
-        """The special token ids.
-
-        Special tokens include control tokens, reserved tokens, padded tokens, etc. Now it is automatically detected
-        from the vocabulary.
-        """
-        return self._handle.special_token_ids
-
-    def dump_metadata(self) -> str:
-        """Dump the metadata of the tokenizer to a json string.
-
-        It can be used to construct the tokenizer info from the vocabulary and the metadata string.
-        """
-        return self._handle.dump_metadata()

src/turbomind/comm/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 find_package(Threads)
 

src/turbomind/comm/cuda_ipc/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 add_library(cuda_ipc_comm STATIC
         cuda_ipc_comm.cu

src/turbomind/comm/nccl/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 add_library(nccl_comm STATIC nccl.cu)
 target_link_libraries(nccl_comm PRIVATE rms_norm core ${NCCL_LIBRARIES} logger)

src/turbomind/core/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 add_library(core STATIC
         check.cc

src/turbomind/engine/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 add_library(engine STATIC gateway.cc request_queue.cc model_request.cc)
 target_link_libraries(engine PRIVATE core xgrammar)

src/turbomind/kernels/CMakeLists.txt

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 add_library(ban_bad_words STATIC ban_bad_words.cu)
 set_property(TARGET ban_bad_words PROPERTY POSITION_INDEPENDENT_CODE  ON)

src/turbomind/layers/CMakeLists.txt

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 add_subdirectory(sampling_layers)
 

src/turbomind/layers/sampling_layers/CMakeLists.txt

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.11)
 
 find_package(CUDAToolkit REQUIRED)