Some tips when you use trtexec to build and infer

[lix19937]

About llm, trtexec also can build and infer, but has some pitfalls.

[nvluxiaoz]

It has a lot of pitfalls. LLMs are very shape dependent and is stateful (KV Cache/Embedding cannot be random). It is also activation aware and autoregressive, so using trtexec does not give you any convenience. However, we are developing a tool to achieve trtexec-parity performance benchmark.

[lix19937]

@nvluxiaoz Yes, it's data-dependent and plugin need modify, I use trtexec infer just for comparing trt vs torch result.
Like follow llm_infer_prefill.sh

#
# lix19937@github 
# build shell of nvidia compiler 
#

#!/bin/bash

if [ $# -eq 0 ]; then
    echo "Usage: $0 <plan_filename>"
    exit 1
fi

# like ./ptq_v4.plan
plan_file=$1

dir_name=$(dirname   "$plan_file")
base_name=$(basename "$plan_file" .plan)

profile_file="$dir_name/${base_name}_infer.profile"
layer_info_file="$dir_name/${base_name}_infer.json"
log_file="$dir_name/${base_name}_infer.log"

cmd="~/tool/trtexec \
--fp16 \
--dumpProfile \
--separateProfileRun \
--loadEngine=$plan_file \
--exportProfile=$profile_file \
--exportLayerInfo=$layer_info_file \
--profilingVerbosity=detailed \
--plugins=../build/libNvInfer_plugin_for_compiler.so.1.0 \
--dumpLayerInfo \
--builderOptimizationLevel=5 \
--tilingOptimizationLevel=3 \
--useCudaGraph \
--warmUp=0   \
--duration=0  \
--iterations=1 \
--useSpinWait \
--noDataTransfers \
--stronglyTyped \
--shapes=\
rope_rotary_cos_sin:1x4096x128,\
context_lengths:1,\
last_token_ids:1x1,\
kvcache_start_index:0,\
input_ids:1x1790,\
image_embeds:900x2048,\
past_key_values.0:1x2x8x4096x128,\
past_key_values.1:1x2x8x4096x128,\
past_key_values.2:1x2x8x4096x128,\
past_key_values.3:1x2x8x4096x128,\
past_key_values.4:1x2x8x4096x128,\
past_key_values.5:1x2x8x4096x128,\
past_key_values.6:1x2x8x4096x128,\
past_key_values.7:1x2x8x4096x128,\
past_key_values.8:1x2x8x4096x128,\
past_key_values.9:1x2x8x4096x128,\
past_key_values.10:1x2x8x4096x128,\
past_key_values.11:1x2x8x4096x128,\
past_key_values.12:1x2x8x4096x128,\
past_key_values.13:1x2x8x4096x128,\
past_key_values.14:1x2x8x4096x128,\
past_key_values.15:1x2x8x4096x128,\
past_key_values.16:1x2x8x4096x128,\
past_key_values.17:1x2x8x4096x128,\
past_key_values.18:1x2x8x4096x128,\
past_key_values.19:1x2x8x4096x128,\
past_key_values.20:1x2x8x4096x128,\
past_key_values.21:1x2x8x4096x128,\
past_key_values.22:1x2x8x4096x128,\
past_key_values.23:1x2x8x4096x128,\
past_key_values.24:1x2x8x4096x128,\
past_key_values.25:1x2x8x4096x128,\
past_key_values.26:1x2x8x4096x128,\
past_key_values.27:1x2x8x4096x128 \
--loadInputs=\
"rope_rotary_cos_sin":vit_flex_out_mix/rope_rotary_cos_sin.bin,\
"context_lengths":vit_flex_out_mix/context_lengths.bin,\
"last_token_ids":vit_flex_out_mix/last_token_ids.bin,\
"input_ids":vit_flex_out_mix/input_ids.bin,\
"image_embeds":vit_flex_out_mix/image_embeds.bin,\
"past_key_values.27":vit_flex_out_mix/past_key_values.27.bin,\
"past_key_values.26":vit_flex_out_mix/past_key_values.26.bin,\
"past_key_values.25":vit_flex_out_mix/past_key_values.25.bin,\
"past_key_values.24":vit_flex_out_mix/past_key_values.24.bin,\
"past_key_values.23":vit_flex_out_mix/past_key_values.23.bin,\
"past_key_values.22":vit_flex_out_mix/past_key_values.22.bin,\
"past_key_values.21":vit_flex_out_mix/past_key_values.21.bin,\
"past_key_values.20":vit_flex_out_mix/past_key_values.20.bin,\
"past_key_values.19":vit_flex_out_mix/past_key_values.19.bin,\
"past_key_values.18":vit_flex_out_mix/past_key_values.18.bin,\
"past_key_values.17":vit_flex_out_mix/past_key_values.17.bin,\
"past_key_values.16":vit_flex_out_mix/past_key_values.16.bin,\
"past_key_values.15":vit_flex_out_mix/past_key_values.15.bin,\
"past_key_values.14":vit_flex_out_mix/past_key_values.14.bin,\
"past_key_values.13":vit_flex_out_mix/past_key_values.13.bin,\
"past_key_values.12":vit_flex_out_mix/past_key_values.12.bin,\
"past_key_values.11":vit_flex_out_mix/past_key_values.11.bin,\
"past_key_values.10":vit_flex_out_mix/past_key_values.10.bin,\
"past_key_values.9":vit_flex_out_mix/past_key_values.9.bin,\
"past_key_values.8":vit_flex_out_mix/past_key_values.8.bin,\
"past_key_values.7":vit_flex_out_mix/past_key_values.7.bin,\
"past_key_values.6":vit_flex_out_mix/past_key_values.6.bin,\
"past_key_values.5":vit_flex_out_mix/past_key_values.5.bin,\
"past_key_values.4":vit_flex_out_mix/past_key_values.4.bin,\
"past_key_values.3":vit_flex_out_mix/past_key_values.3.bin,\
"past_key_values.2":vit_flex_out_mix/past_key_values.2.bin,\
"past_key_values.1":vit_flex_out_mix/past_key_values.1.bin,\
"past_key_values.0":vit_flex_out_mix/past_key_values.0.bin \
--inputIOFormats=\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp32:chw,\
int32:chw,\
int64:chw,\
int32:chw,\
int32:chw,\
fp16:chw \
--exportOutput=prefill.json \
--verbose 2>&1 |tee  $log_file"

echo "Executing command:"
echo "$cmd"

eval $cmd

# bash  ./llm_infer_prefill.sh   f16_llm_without_ds_engine_1_1024/llm.engine

# bash  ./llm_infer_prefill.sh   f16_llm_without_ds_onnx/model_fp16.plan

and llm_infer_decode.sh

#
# lix19937@github 
# build shell of nvidia compiler 
#

#!/bin/bash

if [ $# -eq 0 ]; then
    echo "Usage: $0 <plan_filename>"
    exit 1
fi

# like ./ptq_v4.plan
plan_file=$1

dir_name=$(dirname   "$plan_file")
base_name=$(basename "$plan_file" .plan)

profile_file="$dir_name/${base_name}_infer.profile"
layer_info_file="$dir_name/${base_name}_infer.json"
log_file="$dir_name/${base_name}_infer.log"

# ~/tool/trtexec


cmd="/data/ljw/nfs/thoru_mytrtexec/bin/trtexec \
--fp16 \
--dumpProfile \
--separateProfileRun \
--loadEngine=$plan_file \
--exportProfile=$profile_file \
--exportLayerInfo=$layer_info_file \
--profilingVerbosity=detailed \
--plugins=../build/libNvInfer_plugin_for_compiler.so.1.0 \
--dumpLayerInfo \
--builderOptimizationLevel=5 \
--tilingOptimizationLevel=3 \
--useCudaGraph \
--warmUp=0   \
--duration=0  \
--iterations=1 \
--useSpinWait \
--noDataTransfers \
--stronglyTyped \
--useProfile=1 \
--shapes=\
rope_rotary_cos_sin:1x4096x128,\
context_lengths:1,\
last_token_ids:1x1,\
kvcache_start_index:1,\
input_ids:1x1,\
image_embeds:1x2048,\
past_key_values.0:1x2x8x4096x128,\
past_key_values.1:1x2x8x4096x128,\
past_key_values.2:1x2x8x4096x128,\
past_key_values.3:1x2x8x4096x128,\
past_key_values.4:1x2x8x4096x128,\
past_key_values.5:1x2x8x4096x128,\
past_key_values.6:1x2x8x4096x128,\
past_key_values.7:1x2x8x4096x128,\
past_key_values.8:1x2x8x4096x128,\
past_key_values.9:1x2x8x4096x128,\
past_key_values.10:1x2x8x4096x128,\
past_key_values.11:1x2x8x4096x128,\
past_key_values.12:1x2x8x4096x128,\
past_key_values.13:1x2x8x4096x128,\
past_key_values.14:1x2x8x4096x128,\
past_key_values.15:1x2x8x4096x128,\
past_key_values.16:1x2x8x4096x128,\
past_key_values.17:1x2x8x4096x128,\
past_key_values.18:1x2x8x4096x128,\
past_key_values.19:1x2x8x4096x128,\
past_key_values.20:1x2x8x4096x128,\
past_key_values.21:1x2x8x4096x128,\
past_key_values.22:1x2x8x4096x128,\
past_key_values.23:1x2x8x4096x128,\
past_key_values.24:1x2x8x4096x128,\
past_key_values.25:1x2x8x4096x128,\
past_key_values.26:1x2x8x4096x128,\
past_key_values.27:1x2x8x4096x128 \
--loadInputs=\
"rope_rotary_cos_sin":prefill_out_fp16/rope_rotary_cos_sin.bin,\
"context_lengths":prefill_out_fp16/context_lengths.bin,\
"last_token_ids":prefill_out_fp16/last_token_ids.bin,\
"kvcache_start_index":prefill_out_fp16/kvcache_start_index.bin,\
"input_ids":prefill_out_fp16/input_ids.bin,\
"image_embeds":prefill_out_fp16/image_embeds.bin,\
"past_key_values.27":prefill_out_fp16/past_key_values.27.bin,\
"past_key_values.26":prefill_out_fp16/past_key_values.26.bin,\
"past_key_values.25":prefill_out_fp16/past_key_values.25.bin,\
"past_key_values.24":prefill_out_fp16/past_key_values.24.bin,\
"past_key_values.23":prefill_out_fp16/past_key_values.23.bin,\
"past_key_values.22":prefill_out_fp16/past_key_values.22.bin,\
"past_key_values.21":prefill_out_fp16/past_key_values.21.bin,\
"past_key_values.20":prefill_out_fp16/past_key_values.20.bin,\
"past_key_values.19":prefill_out_fp16/past_key_values.19.bin,\
"past_key_values.18":prefill_out_fp16/past_key_values.18.bin,\
"past_key_values.17":prefill_out_fp16/past_key_values.17.bin,\
"past_key_values.16":prefill_out_fp16/past_key_values.16.bin,\
"past_key_values.15":prefill_out_fp16/past_key_values.15.bin,\
"past_key_values.14":prefill_out_fp16/past_key_values.14.bin,\
"past_key_values.13":prefill_out_fp16/past_key_values.13.bin,\
"past_key_values.12":prefill_out_fp16/past_key_values.12.bin,\
"past_key_values.11":prefill_out_fp16/past_key_values.11.bin,\
"past_key_values.10":prefill_out_fp16/past_key_values.10.bin,\
"past_key_values.9":prefill_out_fp16/past_key_values.9.bin,\
"past_key_values.8":prefill_out_fp16/past_key_values.8.bin,\
"past_key_values.7":prefill_out_fp16/past_key_values.7.bin,\
"past_key_values.6":prefill_out_fp16/past_key_values.6.bin,\
"past_key_values.5":prefill_out_fp16/past_key_values.5.bin,\
"past_key_values.4":prefill_out_fp16/past_key_values.4.bin,\
"past_key_values.3":prefill_out_fp16/past_key_values.3.bin,\
"past_key_values.2":prefill_out_fp16/past_key_values.2.bin,\
"past_key_values.1":prefill_out_fp16/past_key_values.1.bin,\
"past_key_values.0":prefill_out_fp16/past_key_values.0.bin \
--inputIOFormats=\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp16:chw,\
fp32:chw,\
int32:chw,\
int64:chw,\
int32:chw,\
int32:chw,\
fp16:chw \
--exportOutput=decode.json \
--verbose 2>&1 |tee  $log_file"

echo "Executing command:"
echo "$cmd"

eval $cmd

# bash  ./llm_infer_decode.sh   f16_llm_without_ds_engine_1_1024/llm.engine

# bash  ./llm_infer_decode.sh   f16_llm_without_ds_onnx/model_fp16.plan