Skip to content

Commit a4f6af1

Browse files
letonghanletonghan
authored
Refine dataprep test scripts (#1305)
* Refine dataprep Milvus CI Signed-off-by: letonghan <letong.han@intel.com>
1 parent 2102a8e commit a4f6af1

File tree

10 files changed

+35
-33
lines changed

10 files changed

+35
-33
lines changed

‎comps/dataprep/deployment/docker_compose/compose.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
include:
66
- ../../../third_parties/elasticsearch/deployment/docker_compose/compose.yaml
7+
- ../../../third_parties/milvus/deployment/docker_compose/compose.yaml
78
- ../../../third_parties/neo4j/deployment/docker_compose/compose.yaml
89
- ../../../third_parties/opensearch/deployment/docker_compose/compose.yaml
910
- ../../../third_parties/pgvector/deployment/docker_compose/compose.yaml
@@ -54,6 +55,12 @@ services:
5455
depends_on:
5556
tei-embedding-serving:
5657
condition: service_healthy
58+
standalone:
59+
condition: service_healthy
60+
etcd:
61+
condition: service_healthy
62+
minio:
63+
condition: service_healthy
5764

5865
dataprep-neo4j-llamaindex:
5966
image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}

‎tests/dataprep/test_dataprep_elasticsearch.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT=11100
11+
export TAG="comps"
1112

1213
function build_docker_images() {
1314
cd $WORKPATH
@@ -16,7 +17,7 @@ function build_docker_images() {
1617
docker pull docker.elastic.co/elasticsearch/elasticsearch:8.16.0
1718

1819
# build dataprep image for elasticsearch
19-
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
20+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
2021
if [ $? -ne 0 ]; then
2122
echo "opea/dataprep built fail"
2223
exit 1
@@ -30,7 +31,6 @@ function start_service() {
3031
export ELASTICSEARCH_PORT1=12300
3132
export ES_CONNECTION_STRING="http://${ip_address}:${ELASTICSEARCH_PORT1}"
3233
export INDEX_NAME="test-elasticsearch"
33-
export TAG=comps
3434
service_name="elasticsearch-vector-db dataprep-elasticsearch"
3535
cd $WORKPATH
3636
cd comps/dataprep/deployment/docker_compose/

‎tests/dataprep/test_dataprep_milvus.sh‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@ WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT=11101
11+
service_name="dataprep-milvus tei-embedding-serving etcd minio standalone"
12+
export TAG="comps"
1113

1214
function build_docker_images() {
1315
cd $WORKPATH
1416
echo $(pwd)
1517
# dataprep milvus image
16-
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
18+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
1719
if [ $? -ne 0 ]; then
1820
echo "opea/dataprep built fail"
1921
exit 1
@@ -23,23 +25,16 @@ function build_docker_images() {
2325
}
2426

2527
function start_service() {
26-
# start milvus vector db
27-
cd $WORKPATH/comps/third_parties/milvus/deployment/docker_compose/
28-
# wget https://raw.githubusercontent.com/milvus-io/milvus/v2.4.9/configs/milvus.yaml
29-
# wget https://github.com/milvus-io/milvus/releases/download/v2.4.9/milvus-standalone-docker-compose.yml -O docker-compose.yml
30-
# sed '/- \${DOCKER_VOLUME_DIRECTORY:-\.}\/volumes\/milvus:\/var\/lib\/milvus/a \ \ \ \ \ \ - \${DOCKER_VOLUME_DIRECTORY:-\.}\/milvus.yaml:\/milvus\/configs\/milvus.yaml' -i docker-compose.yml
31-
docker compose up -d
32-
sleep 30
33-
3428
export host_ip=${ip_address}
3529
export TEI_EMBEDDER_PORT=12005
3630
export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
3731
export MILVUS_HOST=${ip_address}
3832
export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
3933
export LOGFLAG=true
40-
service_name="dataprep-milvus tei-embedding-serving"
34+
4135
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
42-
docker compose up ${service_name} -d
36+
docker compose up ${service_name} -d > ${LOG_PATH}/start_services_with_compose.log
37+
4338
sleep 1m
4439
}
4540

@@ -124,12 +119,12 @@ function validate_microservice() {
124119
}
125120

126121
function stop_docker() {
127-
cd $WORKPATH
128-
rm -rf milvus/
129-
cid=$(docker ps -aq --filter "name=dataprep-milvus*")
130-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
131-
cid=$(docker ps -aq --filter "name=milvus-*")
132-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
122+
cd $WORKPATH/comps/third_parties/milvus/deployment/docker_compose/
123+
docker compose -f compose.yaml down --remove-orphans
124+
125+
cd $WORKPATH/comps/dataprep/deployment/docker_compose
126+
docker compose -f compose.yaml down ${service_name} --remove-orphans
127+
133128
}
134129

135130
function main() {

‎tests/dataprep/test_dataprep_neo4j_on_intel_hpu.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT=11103
1111
LLM_ENDPOINT_PORT=10510
12+
export TAG="comps"
1213

1314
function build_docker_images() {
1415
cd $WORKPATH
1516
echo $(pwd)
16-
docker build --no-cache -t opea/dataprep:comps --build-arg no_proxy=$no_proxy --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
17+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg no_proxy=$no_proxy --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
1718
if [ $? -ne 0 ]; then
1819
echo "opea/dataprep built fail"
1920
exit 1
@@ -27,7 +28,6 @@ function build_docker_images() {
2728
function start_service() {
2829
service_name="neo4j-apoc tei-embedding-serving tgi-gaudi-server dataprep-neo4j-llamaindex"
2930
export host_ip=${ip_address}
30-
export TAG="comps"
3131
export NEO4J_AUTH="neo4j/neo4jtest"
3232
export NEO4J_URL="bolt://${ip_address}:7687"
3333
export NEO4J_USERNAME="neo4j"

‎tests/dataprep/test_dataprep_opensearch.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT="11104"
1111
OPENSEARCH_INITIAL_ADMIN_PASSWORD="StRoNgOpEa0)"
12+
export TAG="comps"
1213

1314
function build_docker_images() {
1415
cd $WORKPATH
1516
echo $(pwd)
16-
docker build -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
17+
docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
1718
if [ $? -ne 0 ]; then
1819
echo "opea/dataprep built fail"
1920
exit 1
@@ -31,7 +32,6 @@ function start_service() {
3132
export INDEX_NAME="file-index"
3233
service_name="opensearch-vector-db dataprep-opensearch"
3334
export host_ip=${ip_address}
34-
export TAG="comps"
3535
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
3636
docker compose up ${service_name} -d
3737
sleep 1m

‎tests/dataprep/test_dataprep_pgvector.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT="11105"
11+
export TAG="comps"
1112

1213
function build_docker_images() {
1314
cd $WORKPATH
@@ -16,7 +17,7 @@ function build_docker_images() {
1617
docker pull pgvector/pgvector:0.7.0-pg16
1718

1819
# build dataprep image for pgvector
19-
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
20+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
2021
if [ $? -ne 0 ]; then
2122
echo "opea/dataprep built fail"
2223
exit 1
@@ -34,7 +35,6 @@ function start_service() {
3435

3536
service_name="pgvector-db dataprep-pgvector"
3637
export host_ip=${ip_address}
37-
export TAG="comps"
3838
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
3939
docker compose up ${service_name} -d
4040
sleep 1m

‎tests/dataprep/test_dataprep_pinecone.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@ set -x
77
WORKPATH=$(dirname "$PWD")
88
ip_address=$(hostname -I | awk '{print $1}')
99
DATAPREP_PORT="11106"
10+
export TAG="comps"
1011

1112
function build_docker_images() {
1213
cd $WORKPATH
1314

1415
# build dataprep image for pinecone
15-
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
16+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
1617
if [ $? -ne 0 ]; then
1718
echo "opea/dataprep built fail"
1819
exit 1
@@ -27,7 +28,6 @@ function start_service() {
2728
export HUGGINGFACEHUB_API_TOKEN=$HF_TOKEN
2829

2930
service_name="dataprep-pinecone"
30-
export TAG="comps"
3131
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
3232
docker compose up ${service_name} -d
3333
sleep 1m

‎tests/dataprep/test_dataprep_qdrant.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,13 @@ LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT="11107"
1111
TEI_EMBEDDER_PORT="10220"
12+
export TAG="comps"
1213

1314
function build_docker_images() {
1415
cd $WORKPATH
1516

1617
# dataprep qdrant image
17-
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
18+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
1819
if [ $? -ne 0 ]; then
1920
echo "opea/dataprep built fail"
2021
exit 1
@@ -32,7 +33,6 @@ function start_service() {
3233
export COLLECTION_NAME="rag-qdrant"
3334
export QDRANT_HOST=$ip_address
3435
export QDRANT_PORT=6360
35-
export TAG="comps"
3636
service_name="qdrant-vector-db tei-embedding-serving dataprep-qdrant"
3737
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
3838
docker compose up ${service_name} -d

‎tests/dataprep/test_dataprep_redis.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT="11108"
1111
TEI_EMBEDDER_PORT="10221"
12+
export TAG="comps"
1213

1314
function build_docker_images() {
1415
cd $WORKPATH
1516
echo $(pwd)
16-
docker build -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
17+
docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
1718
if [ $? -ne 0 ]; then
1819
echo "opea/dataprep built fail"
1920
exit 1
@@ -33,7 +34,6 @@ function start_service() {
3334
export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
3435
export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${TEI_EMBEDDER_PORT}"
3536
export INDEX_NAME="rag_redis"
36-
export TAG="comps"
3737
service_name="redis-vector-db tei-embedding-serving dataprep-redis"
3838
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
3939
docker compose up ${service_name} -d

‎tests/dataprep/test_dataprep_vdms.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@ WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
99
ip_address=$(hostname -I | awk '{print $1}')
1010
DATAPREP_PORT="11110"
11+
export TAG="comps"
1112

1213
function build_docker_images() {
1314
cd $WORKPATH
1415
echo $(pwd)
15-
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
16+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
1617

1718
if [ $? -ne 0 ]; then
1819
echo "opea/dataprep built fail"
@@ -30,7 +31,6 @@ function start_service() {
3031
export COLLECTION_NAME="test-comps"
3132
export QDRANT_HOST=$ip_address
3233
export QDRANT_PORT=$QDRANT_PORT
33-
export TAG="comps"
3434
service_name="vdms-vector-db dataprep-vdms"
3535
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
3636
docker compose up ${service_name} -d

0 commit comments

Comments
 (0)