Skip to content

Commit 7949045

Browse files
rui2zhangpre-commit-ci[bot]myqi
authored
EdgeCraftRAG: Add E2E test cases for EdgeCraftRAG - local LLM and vllm (#1137)
Signed-off-by: Zhang, Rui <rui2.zhang@intel.com> Signed-off-by: Mingyuan Qi <mingyuan.qi@intel.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Mingyuan Qi <mingyuan.qi@intel.com>
1 parent cbe952e commit 7949045

8 files changed

Lines changed: 524 additions & 3 deletions

File tree

‎EdgeCraftRAG/docker_compose/intel/gpu/arc/compose_vllm.yaml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
services:
5+
server:
6+
image: ${REGISTRY:-opea}/edgecraftrag-server:${TAG:-latest}
7+
container_name: edgecraftrag-server
8+
environment:
9+
no_proxy: ${no_proxy}
10+
http_proxy: ${http_proxy}
11+
https_proxy: ${https_proxy}
12+
HF_ENDPOINT: ${HF_ENDPOINT}
13+
vLLM_ENDPOINT: ${vLLM_ENDPOINT}
14+
volumes:
15+
- ${MODEL_PATH:-${PWD}}:/home/user/models
16+
- ${DOC_PATH:-${PWD}}:/home/user/docs
17+
- ${GRADIO_PATH:-${PWD}}:/home/user/gradio_cache
18+
- ${HF_CACHE:-${HOME}/.cache}:/home/user/.cache
19+
ports:
20+
- ${PIPELINE_SERVICE_PORT:-16010}:${PIPELINE_SERVICE_PORT:-16010}
21+
devices:
22+
- /dev/dri:/dev/dri
23+
group_add:
24+
- ${VIDEOGROUPID:-44}
25+
- ${RENDERGROUPID:-109}
26+
ecrag:
27+
image: ${REGISTRY:-opea}/edgecraftrag:${TAG:-latest}
28+
container_name: edgecraftrag
29+
environment:
30+
no_proxy: ${no_proxy}
31+
http_proxy: ${http_proxy}
32+
https_proxy: ${https_proxy}
33+
MEGA_SERVICE_PORT: ${MEGA_SERVICE_PORT:-16011}
34+
MEGA_SERVICE_HOST_IP: ${MEGA_SERVICE_HOST_IP:-${HOST_IP}}
35+
PIPELINE_SERVICE_PORT: ${PIPELINE_SERVICE_PORT:-16010}
36+
PIPELINE_SERVICE_HOST_IP: ${PIPELINE_SERVICE_HOST_IP:-${HOST_IP}}
37+
ports:
38+
- ${MEGA_SERVICE_PORT:-16011}:${MEGA_SERVICE_PORT:-16011}
39+
depends_on:
40+
- server
41+
ui:
42+
image: ${REGISTRY:-opea}/edgecraftrag-ui:${TAG:-latest}
43+
container_name: edgecraftrag-ui
44+
environment:
45+
no_proxy: ${no_proxy}
46+
http_proxy: ${http_proxy}
47+
https_proxy: ${https_proxy}
48+
MEGA_SERVICE_PORT: ${MEGA_SERVICE_PORT:-16011}
49+
MEGA_SERVICE_HOST_IP: ${MEGA_SERVICE_HOST_IP:-${HOST_IP}}
50+
PIPELINE_SERVICE_PORT: ${PIPELINE_SERVICE_PORT:-16010}
51+
PIPELINE_SERVICE_HOST_IP: ${PIPELINE_SERVICE_HOST_IP:-${HOST_IP}}
52+
UI_SERVICE_PORT: ${UI_SERVICE_PORT:-8082}
53+
UI_SERVICE_HOST_IP: ${UI_SERVICE_HOST_IP:-0.0.0.0}
54+
volumes:
55+
- ${GRADIO_PATH:-${PWD}}:/home/user/gradio_cache
56+
ports:
57+
- ${UI_SERVICE_PORT:-8082}:${UI_SERVICE_PORT:-8082}
58+
restart: always
59+
depends_on:
60+
- server
61+
- ecrag
62+
vllm-openvino-server:
63+
container_name: vllm-openvino-server
64+
image: opea/vllm-arc:latest
65+
ports:
66+
- ${VLLM_SERVICE_PORT:-8008}:80
67+
environment:
68+
HTTPS_PROXY: ${https_proxy}
69+
HTTP_PROXY: ${https_proxy}
70+
VLLM_OPENVINO_DEVICE: GPU
71+
HF_ENDPOINT: ${HF_ENDPOINT}
72+
HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
73+
volumes:
74+
- /dev/dri/by-path:/dev/dri/by-path
75+
- $HOME/.cache/huggingface:/root/.cache/huggingface
76+
devices:
77+
- /dev/dri
78+
group_add:
79+
- ${VIDEOGROUPID:-44}
80+
- ${RENDERGROUPID:-109}
81+
entrypoint: /bin/bash -c "\
82+
cd / && \
83+
export VLLM_CPU_KVCACHE_SPACE=50 && \
84+
export VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON && \
85+
python3 -m vllm.entrypoints.openai.api_server \
86+
--model '${LLM_MODEL}' \
87+
--max_model_len=4096 \
88+
--host 0.0.0.0 \
89+
--port 80"
90+
networks:
91+
default:
92+
driver: bridge

‎EdgeCraftRAG/tests/common.sh‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
function validate_services() {
6+
local URL="$1"
7+
local EXPECTED_RESULT="$2"
8+
local SERVICE_NAME="$3"
9+
local DOCKER_NAME="$4"
10+
local INPUT_DATA="$5"
11+
12+
echo "[ $SERVICE_NAME ] Validating $SERVICE_NAME service..."
13+
local RESPONSE=$(curl -s -w "%{http_code}" -o ${LOG_PATH}/${SERVICE_NAME}.log -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
14+
while [ ! -f ${LOG_PATH}/${SERVICE_NAME}.log ]; do
15+
sleep 1
16+
done
17+
local HTTP_STATUS="${RESPONSE: -3}"
18+
local CONTENT=$(cat ${LOG_PATH}/${SERVICE_NAME}.log)
19+
20+
if [ "$HTTP_STATUS" -eq 200 ]; then
21+
echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
22+
if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
23+
echo "[ $SERVICE_NAME ] Content is as expected."
24+
else
25+
echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
26+
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
27+
exit 1
28+
fi
29+
else
30+
echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
31+
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
32+
exit 1
33+
fi
34+
sleep 1s
35+
}
36+
37+
function check_gpu_usage() {
38+
echo $date > ${LOG_PATH}/gpu.log
39+
pci_address=$(lspci | grep -i '56a0' | awk '{print $1}')
40+
gpu_stats=$(sudo xpu-smi stats -d 0000:"$pci_address") #TODO need sudo
41+
gpu_utilization=$(echo "$gpu_stats" | grep -i "GPU Utilization" | awk -F'|' '{print $3}' | awk '{print $1}')
42+
memory_used=$(echo "$gpu_stats" | grep -i "GPU Memory Used" | awk -F'|' '{print $3}' | awk '{print $1}')
43+
memory_util=$(echo "$gpu_stats" | grep -i "GPU Memory Util" | awk -F'|' '{print $3}' | awk '{print $1}')
44+
45+
echo "GPU Utilization (%): $gpu_utilization" >> ${LOG_PATH}/gpu.log
46+
echo "GPU Memory Used (MiB): $memory_used" >> ${LOG_PATH}/gpu.log
47+
echo "GPU Memory Util (%): $memory_util" >> ${LOG_PATH}/gpu.log
48+
49+
if [ "$memory_used" -lt 1024 ]; then
50+
echo "GPU Memory Used is less than 1G. Please check."
51+
exit 1
52+
fi
53+
}

‎EdgeCraftRAG/tests/configs/test_data.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"text": "A test case for the rag pipeline. The test id is 1234567890. There are several tests in this test case. The first test is for node parser. There are 3 types of node parsers. Their names are Aa, Bb and Cc. The second test is for indexer. The indexer will do the indexing for the given nodes. The last test is for retriever. Retrieving text is based on similarity search."
3+
}

‎EdgeCraftRAG/tests/configs/test_pipeline_local_llm.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"name": "rag_test_local_llm",
3+
"node_parser": {
4+
"chunk_size": 400,
5+
"chunk_overlap": 48,
6+
"parser_type": "simple"
7+
},
8+
"indexer": {
9+
"indexer_type": "faiss_vector",
10+
"embedding_model": {
11+
"model_id": "BAAI/bge-small-en-v1.5",
12+
"model_path": "./models/BAAI/bge-small-en-v1.5",
13+
"device": "auto",
14+
"weight": "INT4"
15+
}
16+
},
17+
"retriever": {
18+
"retriever_type": "vectorsimilarity",
19+
"retrieve_topk": 30
20+
},
21+
"postprocessor": [
22+
{
23+
"processor_type": "reranker",
24+
"top_n": 2,
25+
"reranker_model": {
26+
"model_id": "BAAI/bge-reranker-large",
27+
"model_path": "./models/BAAI/bge-reranker-large",
28+
"device": "auto",
29+
"weight": "INT4"
30+
}
31+
}
32+
],
33+
"generator": {
34+
"model": {
35+
"model_id": "Qwen/Qwen2-7B-Instruct",
36+
"model_path": "./models/Qwen/Qwen2-7B-Instruct/INT4_compressed_weights",
37+
"device": "auto",
38+
"weight": "INT4"
39+
},
40+
"prompt_path": "./edgecraftrag/prompt_template/default_prompt.txt",
41+
"inference_type": "local"
42+
},
43+
"active": "True"
44+
}

‎EdgeCraftRAG/tests/configs/test_pipeline_vllm.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"name": "rag_test_local_llm",
3+
"node_parser": {
4+
"chunk_size": 400,
5+
"chunk_overlap": 48,
6+
"parser_type": "simple"
7+
},
8+
"indexer": {
9+
"indexer_type": "faiss_vector",
10+
"embedding_model": {
11+
"model_id": "BAAI/bge-small-en-v1.5",
12+
"model_path": "./models/BAAI/bge-small-en-v1.5",
13+
"device": "auto",
14+
"weight": "INT4"
15+
}
16+
},
17+
"retriever": {
18+
"retriever_type": "vectorsimilarity",
19+
"retrieve_topk": 30
20+
},
21+
"postprocessor": [
22+
{
23+
"processor_type": "reranker",
24+
"top_n": 2,
25+
"reranker_model": {
26+
"model_id": "BAAI/bge-reranker-large",
27+
"model_path": "./models/BAAI/bge-reranker-large",
28+
"device": "auto",
29+
"weight": "INT4"
30+
}
31+
}
32+
],
33+
"generator": {
34+
"model": {
35+
"model_id": "Qwen/Qwen2-7B-Instruct",
36+
"model_path": "./models/Qwen/Qwen2-7B-Instruct/INT4_compressed_weights",
37+
"device": "auto",
38+
"weight": "INT4"
39+
},
40+
"prompt_path": "./edgecraftrag/prompt_template/default_prompt.txt",
41+
"inference_type": "vllm"
42+
},
43+
"active": "True"
44+
}

‎EdgeCraftRAG/tests/test_compose_on_arc.sh‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -e
6+
source ./common.sh
7+
8+
IMAGE_REPO=${IMAGE_REPO:-"opea"}
9+
IMAGE_TAG=${IMAGE_TAG:-"latest"}
10+
echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
11+
echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
12+
export REGISTRY=${IMAGE_REPO}
13+
export TAG=${IMAGE_TAG}
14+
15+
WORKPATH=$(dirname "$PWD")
16+
LOG_PATH="$WORKPATH/tests"
17+
18+
ip_address=$(hostname -I | awk '{print $1}')
19+
HOST_IP=$ip_address
20+
21+
COMPOSE_FILE="compose.yaml"
22+
EC_RAG_SERVICE_PORT=16010
23+
#MODEL_PATH="$WORKPATH/models"
24+
MODEL_PATH="/home/media/models"
25+
HF_ENDPOINT=https://hf-mirror.com
26+
27+
28+
function build_docker_images() {
29+
cd $WORKPATH/docker_image_build
30+
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
31+
service_list="server ui ecrag"
32+
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
33+
34+
docker images && sleep 1s
35+
}
36+
37+
function start_services() {
38+
export MODEL_PATH=${MODEL_PATH}
39+
export HOST_IP=${HOST_IP}
40+
export LLM_MODEL=${LLM_MODEL}
41+
export HF_ENDPOINT=${HF_ENDPOINT}
42+
export vLLM_ENDPOINT=${vLLM_ENDPOINT}
43+
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
44+
export no_proxy="localhost, 127.0.0.1, 192.168.1.1"
45+
46+
cd $WORKPATH/docker_compose/intel/gpu/arc
47+
48+
# Start Docker Containers
49+
docker compose -f $COMPOSE_FILE up -d > ${LOG_PATH}/start_services_with_compose.log
50+
sleep 20
51+
}
52+
53+
function validate_rag() {
54+
cd $WORKPATH/tests
55+
56+
# setup pipeline
57+
validate_services \
58+
"${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/settings/pipelines" \
59+
"active" \
60+
"pipeline" \
61+
"edgecraftrag-server" \
62+
'@configs/test_pipeline_local_llm.json'
63+
64+
# add data
65+
validate_services \
66+
"${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/data" \
67+
"Done" \
68+
"data" \
69+
"edgecraftrag-server" \
70+
'@configs/test_data.json'
71+
72+
# query
73+
validate_services \
74+
"${HOST_IP}:${EC_RAG_SERVICE_PORT}/v1/chatqna" \
75+
"1234567890" \
76+
"query" \
77+
"vllm-openvino-server" \
78+
'{"messages":"What is the test id?"}'
79+
}
80+
81+
function validate_megaservice() {
82+
# Curl the Mega Service
83+
validate_services \
84+
"${HOST_IP}:16011/v1/chatqna" \
85+
"1234567890" \
86+
"query" \
87+
"vllm-openvino-server" \
88+
'{"messages":"What is the test id?"}'
89+
}
90+
91+
function stop_docker() {
92+
cd $WORKPATH/docker_compose/intel/gpu/arc
93+
docker compose -f $COMPOSE_FILE down
94+
}
95+
96+
97+
function main() {
98+
mkdir -p $LOG_PATH
99+
100+
stop_docker
101+
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
102+
start_services
103+
echo "EC_RAG service started" && sleep 1s
104+
105+
validate_rag
106+
validate_megaservice
107+
108+
stop_docker
109+
echo y | docker system prune
110+
111+
}
112+
113+
main

0 commit comments

Comments
 (0)