From fb8c2ed240967d15835819da3fdb863b0b87c4b0 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Mon, 12 Aug 2024 16:30:59 +0800
Subject: [PATCH 01/16] fix ut for vllm-on-ray

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/docker/gaudi/compose_vllm_ray.yaml       |  4 ++--
 ChatQnA/tests/_test_chatqna_vllm_ray_on_gaudi.sh | 13 +++++++------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/ChatQnA/docker/gaudi/compose_vllm_ray.yaml b/ChatQnA/docker/gaudi/compose_vllm_ray.yaml
index d7fa0ca6ef..bf08b722a5 100644
--- a/ChatQnA/docker/gaudi/compose_vllm_ray.yaml
+++ b/ChatQnA/docker/gaudi/compose_vllm_ray.yaml
@@ -122,12 +122,12 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HABANA_VISIBLE_DEVICES: all
       OMPI_MCA_btl_vader_single_copy_mechanism: none
-      LLM_MODEL: ${LLM_MODEL_ID}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
     runtime: habana
     cap_add:
       - SYS_NICE
     ipc: host
-    command: /bin/bash -c "ray start --head && python vllm_ray_openai.py --port_number 8000 --model_id_or_path $LLM_MODEL --tensor_parallel_size 2 --enforce_eager True"
+    command: /bin/bash -c "ray start --head && python vllm_ray_openai.py --port_number 8000 --model_id_or_path $LLM_MODEL_ID --tensor_parallel_size 2 --enforce_eager True"
   llm:
     image: opea/llm-vllm-ray:latest
     container_name: llm-vllm-ray-gaudi-server
diff --git a/ChatQnA/tests/_test_chatqna_vllm_ray_on_gaudi.sh b/ChatQnA/tests/_test_chatqna_vllm_ray_on_gaudi.sh
index ab41c3bb80..c0638d3e05 100644
--- a/ChatQnA/tests/_test_chatqna_vllm_ray_on_gaudi.sh
+++ b/ChatQnA/tests/_test_chatqna_vllm_ray_on_gaudi.sh
@@ -80,12 +80,13 @@ function start_services() {
     # Start Docker Containers
     docker compose -f compose_vllm_ray.yaml up -d
     n=0
-    until [[ "$n" -ge 400 ]]; do
-        docker logs vllm-ray-gaudi-server > vllm_ray_service_start.log
-        if grep -q Connected vllm_ray_service_start.log; then
-            break
-        fi
-        sleep 1s
+    until [[ "$n" -ge 25 ]]; do
+        echo "n=$n"
+        docker logs vllm-ray-gaudi-server
+        # if grep -q Connected vllm_ray_service_start.log; then
+        #     break
+        # fi
+        sleep 20s
         n=$((n+1))
     done
 }

From 3f81fc9b28d2651fab46aae3afb1c85b69c920c5 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Mon, 12 Aug 2024 16:32:35 +0800
Subject: [PATCH 02/16] fix ut for vllm

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/docker/gaudi/compose_vllm.yaml       |  4 ++--
 ChatQnA/tests/_test_chatqna_vllm_on_gaudi.sh | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ChatQnA/docker/gaudi/compose_vllm.yaml b/ChatQnA/docker/gaudi/compose_vllm.yaml
index 9082bff46e..27aacc3ea4 100644
--- a/ChatQnA/docker/gaudi/compose_vllm.yaml
+++ b/ChatQnA/docker/gaudi/compose_vllm.yaml
@@ -122,12 +122,12 @@ services:
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HABANA_VISIBLE_DEVICES: all
       OMPI_MCA_btl_vader_single_copy_mechanism: none
-      LLM_MODEL: ${LLM_MODEL_ID}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
     runtime: habana
     cap_add:
       - SYS_NICE
     ipc: host
-    command: /bin/bash -c "export VLLM_CPU_KVCACHE_SPACE=40 && python3 -m vllm.entrypoints.openai.api_server --enforce-eager --model $LLM_MODEL --tensor-parallel-size 1 --host 0.0.0.0 --port 80"
+    command: /bin/bash -c "export VLLM_CPU_KVCACHE_SPACE=40 && python3 -m vllm.entrypoints.openai.api_server --enforce-eager --model $LLM_MODEL_ID --tensor-parallel-size 1 --host 0.0.0.0 --port 80"
   llm:
     image: opea/llm-vllm:latest
     container_name: llm-vllm-gaudi-server
diff --git a/ChatQnA/tests/_test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/_test_chatqna_vllm_on_gaudi.sh
index 716a206785..02c1010498 100644
--- a/ChatQnA/tests/_test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/_test_chatqna_vllm_on_gaudi.sh
@@ -79,12 +79,12 @@ function start_services() {
     # Start Docker Containers
     docker compose -f compose_vllm.yaml up -d
     n=0
-    until [[ "$n" -ge 180 ]]; do
-        docker logs vllm-gaudi-server > vllm_service_start.log
-        if grep -q Connected vllm_service_start.log; then
-            break
-        fi
-        sleep 1s
+    until [[ "$n" -ge 18 ]]; do
+        docker logs vllm-gaudi-server 
+        # if grep -q Connected vllm_service_start.log; then
+        #     break
+        # fi
+        sleep 10s
         n=$((n+1))
     done
 }

From 825f617f7297ea3dd7346b04fd8ae5a9f0391539 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 09:22:16 +0800
Subject: [PATCH 03/16] refine for vllm on xeon

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/_test_chatqna_vllm_on_xeon.sh | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/ChatQnA/tests/_test_chatqna_vllm_on_xeon.sh b/ChatQnA/tests/_test_chatqna_vllm_on_xeon.sh
index e92954313b..174e27ee0b 100644
--- a/ChatQnA/tests/_test_chatqna_vllm_on_xeon.sh
+++ b/ChatQnA/tests/_test_chatqna_vllm_on_xeon.sh
@@ -26,20 +26,15 @@ function build_docker_images() {
     cd $WORKPATH/docker/ui
     docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile .
 
-#    cd $WORKPATH
-#    git clone https://github.com/vllm-project/vllm.git
-#    cd vllm
-#    docker build --no-cache -t opea/vllm:latest -f Dockerfile.cpu .
+    cd $WORKPATH
+    git clone https://github.com/vllm-project/vllm.git
+    cd vllm
+    docker build --no-cache -t opea/vllm:latest -f Dockerfile.cpu .
 
     docker images
 }
 
 function start_services() {
-    # build vllm for each test instead of pull from local registry
-    cd $WORKPATH
-    git clone https://github.com/vllm-project/vllm.git
-    cd vllm
-    docker build --no-cache -t opea/vllm:latest -f Dockerfile.cpu .
 
     cd $WORKPATH/docker/xeon
 
@@ -79,12 +74,12 @@ function start_services() {
     # Start Docker Containers
     docker compose -f compose_vllm.yaml up -d
     n=0
-    until [[ "$n" -ge 100 ]]; do
+    until [[ "$n" -ge 10 ]]; do
         docker logs vllm-service > ${LOG_PATH}/vllm_service_start.log
         if grep -q Connected ${LOG_PATH}/vllm_service_start.log; then
             break
         fi
-        sleep 1s
+        sleep 10s
         n=$((n+1))
     done
 }

From 04f381ac4dc294ede22491d1c70f31c6752c4d36 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 09:24:56 +0800
Subject: [PATCH 04/16] rename for ut

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ...est_chatqna_vllm_on_gaudi.sh => test_chatqna_vllm_on_gaudi.sh} | 0
 ..._test_chatqna_vllm_on_xeon.sh => test_chatqna_vllm_on_xeon.sh} | 0
 ...qna_vllm_ray_on_gaudi.sh => test_chatqna_vllm_ray_on_gaudi.sh} | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename ChatQnA/tests/{_test_chatqna_vllm_on_gaudi.sh => test_chatqna_vllm_on_gaudi.sh} (100%)
 rename ChatQnA/tests/{_test_chatqna_vllm_on_xeon.sh => test_chatqna_vllm_on_xeon.sh} (100%)
 rename ChatQnA/tests/{_test_chatqna_vllm_ray_on_gaudi.sh => test_chatqna_vllm_ray_on_gaudi.sh} (100%)

diff --git a/ChatQnA/tests/_test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
similarity index 100%
rename from ChatQnA/tests/_test_chatqna_vllm_on_gaudi.sh
rename to ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
diff --git a/ChatQnA/tests/_test_chatqna_vllm_on_xeon.sh b/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
similarity index 100%
rename from ChatQnA/tests/_test_chatqna_vllm_on_xeon.sh
rename to ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
diff --git a/ChatQnA/tests/_test_chatqna_vllm_ray_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh
similarity index 100%
rename from ChatQnA/tests/_test_chatqna_vllm_ray_on_gaudi.sh
rename to ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh

From 74d506a5e62ad9cb717b899ecb4d58e3d4251204 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 13 Aug 2024 02:05:51 +0000
Subject: [PATCH 05/16] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
index 02c1010498..dcdbb13d76 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
@@ -80,7 +80,7 @@ function start_services() {
     docker compose -f compose_vllm.yaml up -d
     n=0
     until [[ "$n" -ge 18 ]]; do
-        docker logs vllm-gaudi-server 
+        docker logs vllm-gaudi-server
         # if grep -q Connected vllm_service_start.log; then
         #     break
         # fi

From 6501abfbd7cbea286a8076dd45b62c8a61ff1f51 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 10:37:59 +0800
Subject: [PATCH 06/16] enlarge waiting time for vllm on gaudi

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
index 02c1010498..8a4c3bbc84 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
@@ -79,12 +79,12 @@ function start_services() {
     # Start Docker Containers
     docker compose -f compose_vllm.yaml up -d
     n=0
-    until [[ "$n" -ge 18 ]]; do
+    until [[ "$n" -ge 25 ]]; do
         docker logs vllm-gaudi-server 
         # if grep -q Connected vllm_service_start.log; then
         #     break
         # fi
-        sleep 10s
+        sleep 20s
         n=$((n+1))
     done
 }

From 4363dd0cf3c18095fbf960571e65c9f3fb77c1fd Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 13 Aug 2024 03:27:58 +0000
Subject: [PATCH 07/16] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
index 8a4c3bbc84..d74b12d01f 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
@@ -80,7 +80,7 @@ function start_services() {
     docker compose -f compose_vllm.yaml up -d
     n=0
     until [[ "$n" -ge 25 ]]; do
-        docker logs vllm-gaudi-server 
+        docker logs vllm-gaudi-server
         # if grep -q Connected vllm_service_start.log; then
         #     break
         # fi

From 965b655dc7768f312d67c41b3ff9f40d9e52ee5a Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 12:28:56 +0800
Subject: [PATCH 08/16] refine vllm on gaudi params

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/docker/gaudi/compose_vllm.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChatQnA/docker/gaudi/compose_vllm.yaml b/ChatQnA/docker/gaudi/compose_vllm.yaml
index 27aacc3ea4..0f225d9df7 100644
--- a/ChatQnA/docker/gaudi/compose_vllm.yaml
+++ b/ChatQnA/docker/gaudi/compose_vllm.yaml
@@ -127,7 +127,7 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
-    command: /bin/bash -c "export VLLM_CPU_KVCACHE_SPACE=40 && python3 -m vllm.entrypoints.openai.api_server --enforce-eager --model $LLM_MODEL_ID --tensor-parallel-size 1 --host 0.0.0.0 --port 80"
+    command: /bin/bash -c "export VLLM_CPU_KVCACHE_SPACE=40 && python3 -m vllm.entrypoints.openai.api_server --enforce-eager --model $LLM_MODEL_ID --tensor-parallel-size 1 --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048"
   llm:
     image: opea/llm-vllm:latest
     container_name: llm-vllm-gaudi-server

From be8eaf3b37b2dac2546319176cb3c681f5ccece4 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 13:53:45 +0800
Subject: [PATCH 09/16] fix port bug

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
index d74b12d01f..0f8222c09d 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
@@ -51,6 +51,7 @@ function start_services() {
     export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:8090"
     export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808"
     export vLLM_LLM_ENDPOINT="http://${ip_address}:8008"
+    export LLM_SERVICE_PORT=9000
     export REDIS_URL="redis://${ip_address}:6379"
     export INDEX_NAME="rag-redis"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}

From 5df278ac52d9c718984364254babe283a580514f Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 15:05:04 +0800
Subject: [PATCH 10/16] fix vllm xeon ut

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/test_chatqna_vllm_on_xeon.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh b/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
index 174e27ee0b..48751ed3c0 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
@@ -26,15 +26,19 @@ function build_docker_images() {
     cd $WORKPATH/docker/ui
     docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile .
 
-    cd $WORKPATH
-    git clone https://github.com/vllm-project/vllm.git
-    cd vllm
-    docker build --no-cache -t opea/vllm:latest -f Dockerfile.cpu .
+    # cd $WORKPATH
+    # git clone https://github.com/vllm-project/vllm.git
+    # cd vllm
+    # docker build --no-cache -t opea/vllm:latest -f Dockerfile.cpu .
 
     docker images
 }
 
 function start_services() {
+    cd $WORKPATH
+    git clone https://github.com/vllm-project/vllm.git
+    cd vllm
+    docker build --no-cache -t opea/vllm:latest -f Dockerfile.cpu .
 
     cd $WORKPATH/docker/xeon
 

From 50db808da14386f0fcb38bb9d93203de32c5fd92 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 16:19:44 +0800
Subject: [PATCH 11/16] fix ut bug

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/docker/gaudi/compose.yaml               | 2 +-
 ChatQnA/docker/gaudi/compose_vllm.yaml          | 2 +-
 ChatQnA/docker/gaudi/compose_vllm_ray.yaml      | 2 +-
 ChatQnA/tests/test_chatqna_on_gaudi.sh          | 4 ++--
 ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh     | 6 +++---
 ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh | 4 ++--
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ChatQnA/docker/gaudi/compose.yaml b/ChatQnA/docker/gaudi/compose.yaml
index 362ce18248..82b8d0b77d 100644
--- a/ChatQnA/docker/gaudi/compose.yaml
+++ b/ChatQnA/docker/gaudi/compose.yaml
@@ -114,7 +114,7 @@ services:
     image: ghcr.io/huggingface/tgi-gaudi:2.0.1
     container_name: tgi-gaudi-server
     ports:
-      - "8008:80"
+      - "8005:80"
     volumes:
       - "./data:/data"
     environment:
diff --git a/ChatQnA/docker/gaudi/compose_vllm.yaml b/ChatQnA/docker/gaudi/compose_vllm.yaml
index 0f225d9df7..ec82c9f22e 100644
--- a/ChatQnA/docker/gaudi/compose_vllm.yaml
+++ b/ChatQnA/docker/gaudi/compose_vllm.yaml
@@ -112,7 +112,7 @@ services:
     image: opea/llm-vllm-hpu:latest
     container_name: vllm-gaudi-server
     ports:
-      - "8008:80"
+      - "8007:80"
     volumes:
       - "./data:/data"
     environment:
diff --git a/ChatQnA/docker/gaudi/compose_vllm_ray.yaml b/ChatQnA/docker/gaudi/compose_vllm_ray.yaml
index bf08b722a5..5d98a16004 100644
--- a/ChatQnA/docker/gaudi/compose_vllm_ray.yaml
+++ b/ChatQnA/docker/gaudi/compose_vllm_ray.yaml
@@ -112,7 +112,7 @@ services:
     image: opea/llm-vllm-ray-hpu:latest
     container_name: vllm-ray-gaudi-server
     ports:
-      - "8008:8000"
+      - "8006:8000"
     volumes:
       - "./data:/data"
     environment:
diff --git a/ChatQnA/tests/test_chatqna_on_gaudi.sh b/ChatQnA/tests/test_chatqna_on_gaudi.sh
index 3a5e7537be..7c25cb0033 100644
--- a/ChatQnA/tests/test_chatqna_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_on_gaudi.sh
@@ -50,7 +50,7 @@ function start_services() {
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:8090"
     export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808"
-    export TGI_LLM_ENDPOINT="http://${ip_address}:8008"
+    export TGI_LLM_ENDPOINT="http://${ip_address}:8005"
     export REDIS_URL="redis://${ip_address}:6379"
     export REDIS_HOST=${ip_address}
     export INDEX_NAME="rag-redis"
@@ -215,7 +215,7 @@ function validate_microservices() {
 
     # tgi for llm service
     validate_service \
-        "${ip_address}:8008/generate" \
+        "${ip_address}:8005/generate" \
         "generated_text" \
         "tgi-llm" \
         "tgi-gaudi-server" \
diff --git a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
index 0f8222c09d..3823f542c1 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
@@ -50,7 +50,7 @@ function start_services() {
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:8090"
     export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808"
-    export vLLM_LLM_ENDPOINT="http://${ip_address}:8008"
+    export vLLM_LLM_ENDPOINT="http://${ip_address}:8007"
     export LLM_SERVICE_PORT=9000
     export REDIS_URL="redis://${ip_address}:6379"
     export INDEX_NAME="rag-redis"
@@ -166,7 +166,7 @@ function validate_microservices() {
 
     # vllm for llm service
     validate_services \
-        "${ip_address}:8008/v1/completions" \
+        "${ip_address}:8007/v1/completions" \
         "text" \
         "vllm-llm" \
         "vllm-gaudi-server" \
@@ -186,7 +186,7 @@ function validate_megaservice() {
     # Curl the Mega Service
     validate_services \
         "${ip_address}:8888/v1/chatqna" \
-        "billion" \
+        "data:" \
         "mega-chatqna" \
         "chatqna-gaudi-backend-server" \
         '{"messages": "What is the revenue of Nike in 2023?"}'
diff --git a/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh
index c0638d3e05..b974bbb449 100644
--- a/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh
@@ -50,7 +50,7 @@ function start_services() {
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:8090"
     export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808"
-    export vLLM_RAY_LLM_ENDPOINT="http://${ip_address}:8008"
+    export vLLM_RAY_LLM_ENDPOINT="http://${ip_address}:8006"
     export LLM_SERVICE_PORT=9000
     export REDIS_URL="redis://${ip_address}:6379"
     export INDEX_NAME="rag-redis"
@@ -167,7 +167,7 @@ function validate_microservices() {
 
     # vllm-on-ray for llm service
     validate_services \
-        "${ip_address}:8008/v1/chat/completions" \
+        "${ip_address}:8006/v1/chat/completions" \
         "content" \
         "vllm-ray-llm" \
         "vllm-ray-gaudi-server" \

From b3ffb615dc6ecc8e7c7f65d89bc828ce0be27b9f Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Tue, 13 Aug 2024 17:06:46 +0800
Subject: [PATCH 12/16] fix ut bug for vllm xeon

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/test_chatqna_vllm_on_xeon.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh b/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
index 48751ed3c0..cc2df2d492 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
@@ -184,7 +184,7 @@ function validate_megaservice() {
     # Curl the Mega Service
     validate_services \
         "${ip_address}:8888/v1/chatqna" \
-        "billion" \
+        "data" \
         "mega-chatqna" \
         "chatqna-xeon-backend-server" \
         '{"messages": "What is the revenue of Nike in 2023?"}'

From 72be34649a40660796f99df56e9c92c2018d4fe5 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Wed, 14 Aug 2024 08:44:05 +0800
Subject: [PATCH 13/16] fix bug for vllm ut

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/test_chatqna_vllm_on_xeon.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh b/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
index cc2df2d492..a424187cd8 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_xeon.sh
@@ -72,6 +72,7 @@ function start_services() {
             sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" compose_vllm.yaml
             sed -i "s#image: opea/chatqna-conversation-ui:latest#image: opea/chatqna-conversation-ui:${IMAGE_TAG}#g" compose_vllm.yaml
             sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" compose_vllm.yaml
+            sed -i "s#image: ${IMAGE_REPO}opea/vllm:latest#image: opea/vllm:latest#g" compose_vllm.yaml
         fi
     fi
 

From bc9086d1a515b8f2b4ae198b75aa61d245403c53 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Wed, 14 Aug 2024 13:55:41 +0800
Subject: [PATCH 14/16] update readme

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/docker/gaudi/README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ChatQnA/docker/gaudi/README.md b/ChatQnA/docker/gaudi/README.md
index f32d121308..a0cc06881a 100644
--- a/ChatQnA/docker/gaudi/README.md
+++ b/ChatQnA/docker/gaudi/README.md
@@ -175,9 +175,9 @@ export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
 export LLM_MODEL_ID_NAME="neural-chat-7b-v3-3"
 export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:8090"
 export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
-export TGI_LLM_ENDPOINT="http://${host_ip}:8008"
-export vLLM_LLM_ENDPOINT="http://${host_ip}:8008"
-export vLLM_RAY_LLM_ENDPOINT="http://${host_ip}:8008"
+export TGI_LLM_ENDPOINT="http://${host_ip}:8005"
+export vLLM_LLM_ENDPOINT="http://${host_ip}:8007"
+export vLLM_RAY_LLM_ENDPOINT="http://${host_ip}:8006"
 export LLM_SERVICE_PORT=9000
 export REDIS_URL="redis://${host_ip}:6379"
 export INDEX_NAME="rag-redis"
@@ -298,7 +298,7 @@ curl http://${host_ip}:8000/v1/reranking \
 
 ```bash
 #TGI Service
-curl http://${host_ip}:8008/generate \
+curl http://${host_ip}:8005/generate \
   -X POST \
   -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
   -H 'Content-Type: application/json'
@@ -306,7 +306,7 @@ curl http://${host_ip}:8008/generate \
 
 ```bash
 #vLLM Service
-curl http://${host_ip}:8008/v1/completions \
+curl http://${host_ip}:8007/v1/completions \
   -H "Content-Type: application/json" \
   -d '{
   "model": "${LLM_MODEL_ID}",
@@ -318,7 +318,7 @@ curl http://${host_ip}:8008/v1/completions \
 
 ```bash
 #vLLM-on-Ray Service
-curl http://${host_ip}:8008/v1/chat/completions \
+curl http://${host_ip}:8006/v1/chat/completions \
   -H "Content-Type: application/json" \
   -d '{"model": "${LLM_MODEL_ID}", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}'
 ```

From 2e1c11054cb40391e4712915110dd9cb25934404 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Wed, 14 Aug 2024 14:15:56 +0800
Subject: [PATCH 15/16] refine for vllm ut

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh     | 8 ++++----
 ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
index 3823f542c1..1898b38522 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
@@ -81,10 +81,10 @@ function start_services() {
     docker compose -f compose_vllm.yaml up -d
     n=0
     until [[ "$n" -ge 25 ]]; do
-        docker logs vllm-gaudi-server
-        # if grep -q Connected vllm_service_start.log; then
-        #     break
-        # fi
+        docker logs vllm-gaudi-server > vllm_service_start.log
+        if grep -q "Warmup finished" vllm_service_start.log; then
+            break
+        fi
         sleep 20s
         n=$((n+1))
     done
diff --git a/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh
index b974bbb449..9099598bb5 100644
--- a/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_ray_on_gaudi.sh
@@ -82,10 +82,10 @@ function start_services() {
     n=0
     until [[ "$n" -ge 25 ]]; do
         echo "n=$n"
-        docker logs vllm-ray-gaudi-server
-        # if grep -q Connected vllm_ray_service_start.log; then
-        #     break
-        # fi
+        docker logs vllm-ray-gaudi-server > vllm_ray_service_start.log
+        if grep -q "Warmup finished" vllm_ray_service_start.log; then
+            break
+        fi
         sleep 20s
         n=$((n+1))
     done

From 7de9566ba5c0c8400aad9f4784427877f3abb313 Mon Sep 17 00:00:00 2001
From: Xinyao Wang <xinyao.wang@intel.com>
Date: Wed, 14 Aug 2024 16:05:22 +0800
Subject: [PATCH 16/16] refine vllm ut

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
---
 ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
index 1898b38522..99e63e06ba 100644
--- a/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
+++ b/ChatQnA/tests/test_chatqna_vllm_on_gaudi.sh
@@ -81,6 +81,7 @@ function start_services() {
     docker compose -f compose_vllm.yaml up -d
     n=0
     until [[ "$n" -ge 25 ]]; do
+        echo "n=$n"
         docker logs vllm-gaudi-server > vllm_service_start.log
         if grep -q "Warmup finished" vllm_service_start.log; then
             break
