From cf60682c8290f1191c5d3e4609a8ad3b8d1b162a Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:02:03 +0700
Subject: [PATCH 01/17] DocSum - add files for deploy app with ROCm vLLM

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 ++
 .../amd/gpu/rocm-vllm/README.md               | 175 ++++++++++++
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 ++++++++
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 ++
 DocSum/docker_image_build/build.yaml          |   9 +
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ++++++++++++++++++
 6 files changed, 574 insertions(+)
 create mode 100644 DocSum/Dockerfile-vllm-rocm
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
new file mode 100644
index 0000000000..f0e8a8743a
--- /dev/null
+++ b/DocSum/Dockerfile-vllm-rocm
@@ -0,0 +1,18 @@
+FROM rocm/vllm-dev:main
+
+# Set the working directory
+WORKDIR /workspace
+
+# Copy the api_server.py into the image
+ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
+
+# Expose the port used by the API server
+EXPOSE 8011
+
+# Set environment variables
+ENV HUGGINGFACE_HUB_CACHE=/workspace
+ENV WILM_USE_TRITON_FLASH_ATTENTION=0
+ENV PYTORCH_JIT=0
+
+# Set the entrypoint to the api_server.py script
+ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
new file mode 100644
index 0000000000..4d41a5cd31
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
@@ -0,0 +1,175 @@
+# Build and deploy DocSum Application on AMD GPU (ROCm)
+
+## Build images
+
+## 🚀 Build Docker Images
+
+First of all, you need to build Docker Images locally and install the python package of it.
+
+### 1. Build LLM Image
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following four Docker Images:
+
+### 2. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples
+cd GenAIExamples/DocSum/
+docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+```
+
+### 3. Build UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+
+### 4. Build React UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
+
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+4. `opea/docsum-react-ui:latest`
+
+## 🚀 Start Microservices and MegaService
+
+### Required Models
+
+Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
+For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
+
+### Setup Environment Variables
+
+Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
+
+```bash
+export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${host_ip}
+export DOCSUM_TGI_SERVICE_PORT="18882"
+export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
+export DOCSUM_LLM_SERVER_PORT="8008"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
+```
+
+Note: Please replace with `host_ip` with your external IP address, do not use localhost.
+
+Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+Example for set isolation for 1 GPU
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+```
+
+Example for set isolation for 2 GPUs
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+      - /dev/dri/card1:/dev/dri/card1
+      - /dev/dri/renderD129:/dev/dri/renderD129
+```
+
+Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+### Start Microservice Docker Containers
+
+```bash
+cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
+docker compose up -d
+```
+
+### Validate Microservices
+
+1. TGI Service
+
+   ```bash
+   curl http://${host_ip}:8008/generate \
+     -X POST \
+     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
+     -H 'Content-Type: application/json'
+   ```
+
+2. LLM Microservice
+
+   ```bash
+   curl http://${host_ip}:9000/v1/docsum \
+     -X POST \
+     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
+     -H 'Content-Type: application/json'
+   ```
+
+3. MegaService
+
+   ```bash
+   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
+        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
+        }'
+   ```
+
+## 🚀 Launch the Svelte UI
+
+Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
+
+![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
+
+Here is an example for summarizing a article.
+
+![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
+
+## 🚀 Launch the React UI (Optional)
+
+To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
+
+```yaml
+docsum-rocm-react-ui-server:
+  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
+  container_name: docsum-rocm-react-ui-server
+  depends_on:
+    - docsum-rocm-backend-server
+  ports:
+    - "5174:80"
+  environment:
+    - no_proxy=${no_proxy}
+    - https_proxy=${https_proxy}
+    - http_proxy=${http_proxy}
+    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+```
+
+Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
+
+![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
new file mode 100644
index 0000000000..037aa06395
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
@@ -0,0 +1,107 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  docsum-vllm-service:
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
+    container_name: docsum-vllm-service
+    ports:
+      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+      WILM_USE_TRITON_FLASH_ATTENTION: 0
+      PYTORCH_JIT: 0
+    volumes:
+      - "./data:/data"
+    shm_size: 20G
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri/:/dev/dri/
+    cap_add:
+      - SYS_PTRACE
+    group_add:
+      - video
+    security_opt:
+      - seccomp:unconfined
+      - apparmor=unconfined
+    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
+    ipc: host
+
+  docsum-llm-server:
+    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+    container_name: docsum-llm-server
+    depends_on:
+      - docsum-vllm-service
+    ports:
+      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
+    ipc: host
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
+      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
+    restart: unless-stopped
+
+  whisper-service:
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
+    container_name: whisper-service
+    ports:
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    restart: unless-stopped
+
+  docsum-backend-server:
+    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
+    container_name: docsum-backend-server
+    depends_on:
+      - docsum-tgi-service
+      - docsum-llm-server
+    ports:
+      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
+    ipc: host
+    restart: always
+
+  docsum-gradio-ui:
+    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
+    container_name: docsum-ui-server
+    depends_on:
+      - docsum-backend-server
+    ports:
+      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
new file mode 100644
index 0000000000..43e71e0fbf
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+export HOST_IP=""
+export DOCSUM_MAX_INPUT_TOKENS=2048
+export DOCSUM_MAX_TOTAL_TOKENS=4096
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index 095fd28c93..dc0d546189 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,3 +47,12 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+  vllm_rocm:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      context: ../
+      dockerfile: ./Dockerfile-vllm-rocm
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
new file mode 100644
index 0000000000..d0919a019a
--- /dev/null
+++ b/DocSum/tests/test_compose_on_rocm_vllm.sh
@@ -0,0 +1,249 @@
+#!/bin/bash
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+
+WORKPATH=$(dirname "$PWD")
+LOG_PATH="$WORKPATH/tests"
+ip_address=$(hostname -I | awk '{print $1}')
+export MAX_INPUT_TOKENS=1024
+export MAX_TOTAL_TOKENS=2048
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${ip_address}
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export MEGA_SERVICE_HOST_IP=${HOST_IP}
+export LLM_SERVICE_HOST_IP=${HOST_IP}
+export ASR_SERVICE_HOST_IP=${HOST_IP}
+export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
+
+function build_docker_images() {
+    opea_branch=${opea_branch:-"main"}
+    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
+    if [[ "${opea_branch}" != "main" ]]; then
+        cd $WORKPATH
+        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
+        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
+        find . -type f -name "Dockerfile*" | while read -r file; do
+            echo "Processing file: $file"
+            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
+        done
+    fi
+
+    cd $WORKPATH/docker_image_build
+    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
+    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
+
+    docker images && sleep 1s
+}
+
+function start_services() {
+    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
+    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
+    # Start Docker Containers
+    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
+    sleep 1m
+}
+
+function validate_services() {
+    local URL="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+    local DOCKER_NAME="$4"
+    local INPUT_DATA="$5"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
+
+    echo "==========================================="
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
+            echo "CONTENT==> $CONTENT"
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+get_base64_str() {
+    local file_name=$1
+    base64 -w 0 "$file_name"
+}
+
+# Function to generate input data for testing based on the document type
+input_data_for_test() {
+    local document_type=$1
+    case $document_type in
+        ("text")
+            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
+            ;;
+        ("audio")
+            get_base64_str "$WORKPATH/tests/data/test.wav"
+            ;;
+        ("video")
+            get_base64_str "$WORKPATH/tests/data/test.mp4"
+            ;;
+        (*)
+            echo "Invalid document type" >&2
+            exit 1
+            ;;
+    esac
+}
+
+function validate_microservices() {
+    # Check if the microservices are running correctly.
+
+    # whisper microservice
+    ulimit -s 65536
+    validate_services \
+        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
+        '{"asr_result":"well"}' \
+        "whisper-service" \
+        "whisper-service" \
+        "{\"audio\": \"$(input_data_for_test "audio")\"}"
+
+    # vLLM service
+    validate_services \
+        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
+        "generated_text" \
+        "docsum-vllm-service" \
+        "docsum-vllm-service" \
+        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
+
+    # llm microservice
+    validate_services \
+        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
+        "text" \
+        "docsum-llm-server" \
+        "docsum-llm-server" \
+        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+}
+
+function validate_megaservice() {
+    local SERVICE_NAME="docsum-backend-server"
+    local DOCKER_NAME="docsum-backend-server"
+    local EXPECTED_RESULT="[DONE]"
+    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
+    local URL="${host_ip}:8888/v1/docsum"
+    local DATA_TYPE="type=text"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+function validate_megaservice_json() {
+    # Curl the Mega Service
+    echo ""
+    echo ">>> Checking text data with Content-Type: application/json"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+    echo ">>> Checking audio data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
+
+    echo ">>> Checking video data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
+
+}
+
+function stop_docker() {
+    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
+    docker compose stop && docker compose rm -f
+}
+
+function main() {
+    echo "==========================================="
+    echo ">>>> Stopping any running Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    if [[ "$IMAGE_REPO" == "opea" ]]; then
+        echo ">>>> Building Docker images..."
+        build_docker_images
+    fi
+
+    echo "==========================================="
+    echo ">>>> Starting Docker services..."
+    start_services
+
+    echo "==========================================="
+    echo ">>>> Validating microservices..."
+    validate_microservices
+
+    echo "==========================================="
+    echo ">>>> Validating megaservice..."
+    validate_megaservice
+    echo ">>>> Validating validate_megaservice_json..."
+    validate_megaservice_json
+
+    echo "==========================================="
+    echo ">>>> Stopping Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    echo ">>>> Pruning Docker system..."
+    echo y | docker system prune
+    echo ">>>> Docker system pruned successfully."
+    echo "==========================================="
+}
+
+main

From 1fd1de1530328321d28aa6d9db85fffeb876574c Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:07:05 +0700
Subject: [PATCH 02/17] DocSum - fix main

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 --
 .../amd/gpu/rocm-vllm/README.md               | 175 ------------
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 --------
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 --
 DocSum/docker_image_build/build.yaml          |   9 -
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ------------------
 6 files changed, 574 deletions(-)
 delete mode 100644 DocSum/Dockerfile-vllm-rocm
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
deleted file mode 100644
index f0e8a8743a..0000000000
--- a/DocSum/Dockerfile-vllm-rocm
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM rocm/vllm-dev:main
-
-# Set the working directory
-WORKDIR /workspace
-
-# Copy the api_server.py into the image
-ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
-
-# Expose the port used by the API server
-EXPOSE 8011
-
-# Set environment variables
-ENV HUGGINGFACE_HUB_CACHE=/workspace
-ENV WILM_USE_TRITON_FLASH_ATTENTION=0
-ENV PYTORCH_JIT=0
-
-# Set the entrypoint to the api_server.py script
-ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
deleted file mode 100644
index 4d41a5cd31..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
+++ /dev/null
@@ -1,175 +0,0 @@
-# Build and deploy DocSum Application on AMD GPU (ROCm)
-
-## Build images
-
-## 🚀 Build Docker Images
-
-First of all, you need to build Docker Images locally and install the python package of it.
-
-### 1. Build LLM Image
-
-```bash
-git clone https://github.com/opea-project/GenAIComps.git
-cd GenAIComps
-docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following four Docker Images:
-
-### 2. Build MegaService Docker Image
-
-To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
-
-```bash
-git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/DocSum/
-docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
-```
-
-### 3. Build UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-
-### 4. Build React UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
-
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-4. `opea/docsum-react-ui:latest`
-
-## 🚀 Start Microservices and MegaService
-
-### Required Models
-
-Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
-For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
-
-### Setup Environment Variables
-
-Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
-
-```bash
-export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${host_ip}
-export DOCSUM_TGI_SERVICE_PORT="18882"
-export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
-export DOCSUM_LLM_SERVER_PORT="8008"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
-```
-
-Note: Please replace with `host_ip` with your external IP address, do not use localhost.
-
-Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-Example for set isolation for 1 GPU
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-```
-
-Example for set isolation for 2 GPUs
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-      - /dev/dri/card1:/dev/dri/card1
-      - /dev/dri/renderD129:/dev/dri/renderD129
-```
-
-Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-### Start Microservice Docker Containers
-
-```bash
-cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
-docker compose up -d
-```
-
-### Validate Microservices
-
-1. TGI Service
-
-   ```bash
-   curl http://${host_ip}:8008/generate \
-     -X POST \
-     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
-     -H 'Content-Type: application/json'
-   ```
-
-2. LLM Microservice
-
-   ```bash
-   curl http://${host_ip}:9000/v1/docsum \
-     -X POST \
-     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
-     -H 'Content-Type: application/json'
-   ```
-
-3. MegaService
-
-   ```bash
-   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
-        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
-        }'
-   ```
-
-## 🚀 Launch the Svelte UI
-
-Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
-
-![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
-
-Here is an example for summarizing a article.
-
-![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
-
-## 🚀 Launch the React UI (Optional)
-
-To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
-
-```yaml
-docsum-rocm-react-ui-server:
-  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
-  container_name: docsum-rocm-react-ui-server
-  depends_on:
-    - docsum-rocm-backend-server
-  ports:
-    - "5174:80"
-  environment:
-    - no_proxy=${no_proxy}
-    - https_proxy=${https_proxy}
-    - http_proxy=${http_proxy}
-    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
-```
-
-Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
-
-![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
deleted file mode 100644
index 037aa06395..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-services:
-  docsum-vllm-service:
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
-    container_name: docsum-vllm-service
-    ports:
-      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_HUB_DISABLE_PROGRESS_BARS: 1
-      HF_HUB_ENABLE_HF_TRANSFER: 0
-      WILM_USE_TRITON_FLASH_ATTENTION: 0
-      PYTORCH_JIT: 0
-    volumes:
-      - "./data:/data"
-    shm_size: 20G
-    devices:
-      - /dev/kfd:/dev/kfd
-      - /dev/dri/:/dev/dri/
-    cap_add:
-      - SYS_PTRACE
-    group_add:
-      - video
-    security_opt:
-      - seccomp:unconfined
-      - apparmor=unconfined
-    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
-    ipc: host
-
-  docsum-llm-server:
-    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-    container_name: docsum-llm-server
-    depends_on:
-      - docsum-vllm-service
-    ports:
-      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
-    ipc: host
-    cap_add:
-      - SYS_PTRACE
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
-      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
-      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
-      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
-    restart: unless-stopped
-
-  whisper-service:
-    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
-    container_name: whisper-service
-    ports:
-      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
-    ipc: host
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-    restart: unless-stopped
-
-  docsum-backend-server:
-    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
-    container_name: docsum-backend-server
-    depends_on:
-      - docsum-tgi-service
-      - docsum-llm-server
-    ports:
-      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      MEGA_SERVICE_HOST_IP: ${HOST_IP}
-      LLM_SERVICE_HOST_IP: ${HOST_IP}
-      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
-    ipc: host
-    restart: always
-
-  docsum-gradio-ui:
-    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
-    container_name: docsum-ui-server
-    depends_on:
-      - docsum-backend-server
-    ports:
-      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-    ipc: host
-    restart: always
-
-networks:
-  default:
-    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
deleted file mode 100644
index 43e71e0fbf..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-export HOST_IP=""
-export DOCSUM_MAX_INPUT_TOKENS=2048
-export DOCSUM_MAX_TOTAL_TOKENS=4096
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index dc0d546189..095fd28c93 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,12 +47,3 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-  vllm_rocm:
-    build:
-      args:
-        http_proxy: ${http_proxy}
-        https_proxy: ${https_proxy}
-        no_proxy: ${no_proxy}
-      context: ../
-      dockerfile: ./Dockerfile-vllm-rocm
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
deleted file mode 100644
index d0919a019a..0000000000
--- a/DocSum/tests/test_compose_on_rocm_vllm.sh
+++ /dev/null
@@ -1,249 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-set -xe
-IMAGE_REPO=${IMAGE_REPO:-"opea"}
-IMAGE_TAG=${IMAGE_TAG:-"latest"}
-echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
-echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
-
-WORKPATH=$(dirname "$PWD")
-LOG_PATH="$WORKPATH/tests"
-ip_address=$(hostname -I | awk '{print $1}')
-export MAX_INPUT_TOKENS=1024
-export MAX_TOTAL_TOKENS=2048
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${ip_address}
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export MEGA_SERVICE_HOST_IP=${HOST_IP}
-export LLM_SERVICE_HOST_IP=${HOST_IP}
-export ASR_SERVICE_HOST_IP=${HOST_IP}
-export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
-
-function build_docker_images() {
-    opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
-
-    cd $WORKPATH/docker_image_build
-    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
-
-    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
-    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
-
-    docker images && sleep 1s
-}
-
-function start_services() {
-    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
-    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
-    # Start Docker Containers
-    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
-    sleep 1m
-}
-
-function validate_services() {
-    local URL="$1"
-    local EXPECTED_RESULT="$2"
-    local SERVICE_NAME="$3"
-    local DOCKER_NAME="$4"
-    local INPUT_DATA="$5"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
-
-    echo "==========================================="
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
-            echo "CONTENT==> $CONTENT"
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-get_base64_str() {
-    local file_name=$1
-    base64 -w 0 "$file_name"
-}
-
-# Function to generate input data for testing based on the document type
-input_data_for_test() {
-    local document_type=$1
-    case $document_type in
-        ("text")
-            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
-            ;;
-        ("audio")
-            get_base64_str "$WORKPATH/tests/data/test.wav"
-            ;;
-        ("video")
-            get_base64_str "$WORKPATH/tests/data/test.mp4"
-            ;;
-        (*)
-            echo "Invalid document type" >&2
-            exit 1
-            ;;
-    esac
-}
-
-function validate_microservices() {
-    # Check if the microservices are running correctly.
-
-    # whisper microservice
-    ulimit -s 65536
-    validate_services \
-        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
-        '{"asr_result":"well"}' \
-        "whisper-service" \
-        "whisper-service" \
-        "{\"audio\": \"$(input_data_for_test "audio")\"}"
-
-    # vLLM service
-    validate_services \
-        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
-        "generated_text" \
-        "docsum-vllm-service" \
-        "docsum-vllm-service" \
-        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
-
-    # llm microservice
-    validate_services \
-        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
-        "text" \
-        "docsum-llm-server" \
-        "docsum-llm-server" \
-        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-}
-
-function validate_megaservice() {
-    local SERVICE_NAME="docsum-backend-server"
-    local DOCKER_NAME="docsum-backend-server"
-    local EXPECTED_RESULT="[DONE]"
-    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-    local URL="${host_ip}:8888/v1/docsum"
-    local DATA_TYPE="type=text"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-function validate_megaservice_json() {
-    # Curl the Mega Service
-    echo ""
-    echo ">>> Checking text data with Content-Type: application/json"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-    echo ">>> Checking audio data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
-
-    echo ">>> Checking video data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
-
-}
-
-function stop_docker() {
-    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
-    docker compose stop && docker compose rm -f
-}
-
-function main() {
-    echo "==========================================="
-    echo ">>>> Stopping any running Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    if [[ "$IMAGE_REPO" == "opea" ]]; then
-        echo ">>>> Building Docker images..."
-        build_docker_images
-    fi
-
-    echo "==========================================="
-    echo ">>>> Starting Docker services..."
-    start_services
-
-    echo "==========================================="
-    echo ">>>> Validating microservices..."
-    validate_microservices
-
-    echo "==========================================="
-    echo ">>>> Validating megaservice..."
-    validate_megaservice
-    echo ">>>> Validating validate_megaservice_json..."
-    validate_megaservice_json
-
-    echo "==========================================="
-    echo ">>>> Stopping Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    echo ">>>> Pruning Docker system..."
-    echo y | docker system prune
-    echo ">>>> Docker system pruned successfully."
-    echo "==========================================="
-}
-
-main

From bd2d47e7e53e1241c27aed0f823fa680d8ecf4e2 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:02:03 +0700
Subject: [PATCH 03/17] DocSum - add files for deploy app with ROCm vLLM

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 ++
 .../amd/gpu/rocm-vllm/README.md               | 175 ++++++++++++
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 ++++++++
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 ++
 DocSum/docker_image_build/build.yaml          |   9 +
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ++++++++++++++++++
 6 files changed, 574 insertions(+)
 create mode 100644 DocSum/Dockerfile-vllm-rocm
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
new file mode 100644
index 0000000000..f0e8a8743a
--- /dev/null
+++ b/DocSum/Dockerfile-vllm-rocm
@@ -0,0 +1,18 @@
+FROM rocm/vllm-dev:main
+
+# Set the working directory
+WORKDIR /workspace
+
+# Copy the api_server.py into the image
+ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
+
+# Expose the port used by the API server
+EXPOSE 8011
+
+# Set environment variables
+ENV HUGGINGFACE_HUB_CACHE=/workspace
+ENV WILM_USE_TRITON_FLASH_ATTENTION=0
+ENV PYTORCH_JIT=0
+
+# Set the entrypoint to the api_server.py script
+ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
new file mode 100644
index 0000000000..4d41a5cd31
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
@@ -0,0 +1,175 @@
+# Build and deploy DocSum Application on AMD GPU (ROCm)
+
+## Build images
+
+## 🚀 Build Docker Images
+
+First of all, you need to build Docker Images locally and install the python package of it.
+
+### 1. Build LLM Image
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following four Docker Images:
+
+### 2. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples
+cd GenAIExamples/DocSum/
+docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+```
+
+### 3. Build UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+
+### 4. Build React UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
+
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+4. `opea/docsum-react-ui:latest`
+
+## 🚀 Start Microservices and MegaService
+
+### Required Models
+
+Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
+For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
+
+### Setup Environment Variables
+
+Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
+
+```bash
+export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${host_ip}
+export DOCSUM_TGI_SERVICE_PORT="18882"
+export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
+export DOCSUM_LLM_SERVER_PORT="8008"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
+```
+
+Note: Please replace with `host_ip` with your external IP address, do not use localhost.
+
+Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+Example for set isolation for 1 GPU
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+```
+
+Example for set isolation for 2 GPUs
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+      - /dev/dri/card1:/dev/dri/card1
+      - /dev/dri/renderD129:/dev/dri/renderD129
+```
+
+Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+### Start Microservice Docker Containers
+
+```bash
+cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
+docker compose up -d
+```
+
+### Validate Microservices
+
+1. TGI Service
+
+   ```bash
+   curl http://${host_ip}:8008/generate \
+     -X POST \
+     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
+     -H 'Content-Type: application/json'
+   ```
+
+2. LLM Microservice
+
+   ```bash
+   curl http://${host_ip}:9000/v1/docsum \
+     -X POST \
+     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
+     -H 'Content-Type: application/json'
+   ```
+
+3. MegaService
+
+   ```bash
+   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
+        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
+        }'
+   ```
+
+## 🚀 Launch the Svelte UI
+
+Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
+
+![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
+
+Here is an example for summarizing a article.
+
+![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
+
+## 🚀 Launch the React UI (Optional)
+
+To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
+
+```yaml
+docsum-rocm-react-ui-server:
+  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
+  container_name: docsum-rocm-react-ui-server
+  depends_on:
+    - docsum-rocm-backend-server
+  ports:
+    - "5174:80"
+  environment:
+    - no_proxy=${no_proxy}
+    - https_proxy=${https_proxy}
+    - http_proxy=${http_proxy}
+    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+```
+
+Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
+
+![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
new file mode 100644
index 0000000000..037aa06395
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
@@ -0,0 +1,107 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  docsum-vllm-service:
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
+    container_name: docsum-vllm-service
+    ports:
+      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+      WILM_USE_TRITON_FLASH_ATTENTION: 0
+      PYTORCH_JIT: 0
+    volumes:
+      - "./data:/data"
+    shm_size: 20G
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri/:/dev/dri/
+    cap_add:
+      - SYS_PTRACE
+    group_add:
+      - video
+    security_opt:
+      - seccomp:unconfined
+      - apparmor=unconfined
+    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
+    ipc: host
+
+  docsum-llm-server:
+    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+    container_name: docsum-llm-server
+    depends_on:
+      - docsum-vllm-service
+    ports:
+      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
+    ipc: host
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
+      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
+    restart: unless-stopped
+
+  whisper-service:
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
+    container_name: whisper-service
+    ports:
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    restart: unless-stopped
+
+  docsum-backend-server:
+    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
+    container_name: docsum-backend-server
+    depends_on:
+      - docsum-tgi-service
+      - docsum-llm-server
+    ports:
+      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
+    ipc: host
+    restart: always
+
+  docsum-gradio-ui:
+    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
+    container_name: docsum-ui-server
+    depends_on:
+      - docsum-backend-server
+    ports:
+      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
new file mode 100644
index 0000000000..43e71e0fbf
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+export HOST_IP=""
+export DOCSUM_MAX_INPUT_TOKENS=2048
+export DOCSUM_MAX_TOTAL_TOKENS=4096
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index 095fd28c93..dc0d546189 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,3 +47,12 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+  vllm_rocm:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      context: ../
+      dockerfile: ./Dockerfile-vllm-rocm
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
new file mode 100644
index 0000000000..d0919a019a
--- /dev/null
+++ b/DocSum/tests/test_compose_on_rocm_vllm.sh
@@ -0,0 +1,249 @@
+#!/bin/bash
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+
+WORKPATH=$(dirname "$PWD")
+LOG_PATH="$WORKPATH/tests"
+ip_address=$(hostname -I | awk '{print $1}')
+export MAX_INPUT_TOKENS=1024
+export MAX_TOTAL_TOKENS=2048
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${ip_address}
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export MEGA_SERVICE_HOST_IP=${HOST_IP}
+export LLM_SERVICE_HOST_IP=${HOST_IP}
+export ASR_SERVICE_HOST_IP=${HOST_IP}
+export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
+
+function build_docker_images() {
+    opea_branch=${opea_branch:-"main"}
+    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
+    if [[ "${opea_branch}" != "main" ]]; then
+        cd $WORKPATH
+        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
+        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
+        find . -type f -name "Dockerfile*" | while read -r file; do
+            echo "Processing file: $file"
+            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
+        done
+    fi
+
+    cd $WORKPATH/docker_image_build
+    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
+    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
+
+    docker images && sleep 1s
+}
+
+function start_services() {
+    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
+    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
+    # Start Docker Containers
+    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
+    sleep 1m
+}
+
+function validate_services() {
+    local URL="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+    local DOCKER_NAME="$4"
+    local INPUT_DATA="$5"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
+
+    echo "==========================================="
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
+            echo "CONTENT==> $CONTENT"
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+get_base64_str() {
+    local file_name=$1
+    base64 -w 0 "$file_name"
+}
+
+# Function to generate input data for testing based on the document type
+input_data_for_test() {
+    local document_type=$1
+    case $document_type in
+        ("text")
+            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
+            ;;
+        ("audio")
+            get_base64_str "$WORKPATH/tests/data/test.wav"
+            ;;
+        ("video")
+            get_base64_str "$WORKPATH/tests/data/test.mp4"
+            ;;
+        (*)
+            echo "Invalid document type" >&2
+            exit 1
+            ;;
+    esac
+}
+
+function validate_microservices() {
+    # Check if the microservices are running correctly.
+
+    # whisper microservice
+    ulimit -s 65536
+    validate_services \
+        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
+        '{"asr_result":"well"}' \
+        "whisper-service" \
+        "whisper-service" \
+        "{\"audio\": \"$(input_data_for_test "audio")\"}"
+
+    # vLLM service
+    validate_services \
+        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
+        "generated_text" \
+        "docsum-vllm-service" \
+        "docsum-vllm-service" \
+        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
+
+    # llm microservice
+    validate_services \
+        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
+        "text" \
+        "docsum-llm-server" \
+        "docsum-llm-server" \
+        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+}
+
+function validate_megaservice() {
+    local SERVICE_NAME="docsum-backend-server"
+    local DOCKER_NAME="docsum-backend-server"
+    local EXPECTED_RESULT="[DONE]"
+    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
+    local URL="${host_ip}:8888/v1/docsum"
+    local DATA_TYPE="type=text"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+function validate_megaservice_json() {
+    # Curl the Mega Service
+    echo ""
+    echo ">>> Checking text data with Content-Type: application/json"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+    echo ">>> Checking audio data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
+
+    echo ">>> Checking video data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
+
+}
+
+function stop_docker() {
+    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
+    docker compose stop && docker compose rm -f
+}
+
+function main() {
+    echo "==========================================="
+    echo ">>>> Stopping any running Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    if [[ "$IMAGE_REPO" == "opea" ]]; then
+        echo ">>>> Building Docker images..."
+        build_docker_images
+    fi
+
+    echo "==========================================="
+    echo ">>>> Starting Docker services..."
+    start_services
+
+    echo "==========================================="
+    echo ">>>> Validating microservices..."
+    validate_microservices
+
+    echo "==========================================="
+    echo ">>>> Validating megaservice..."
+    validate_megaservice
+    echo ">>>> Validating validate_megaservice_json..."
+    validate_megaservice_json
+
+    echo "==========================================="
+    echo ">>>> Stopping Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    echo ">>>> Pruning Docker system..."
+    echo y | docker system prune
+    echo ">>>> Docker system pruned successfully."
+    echo "==========================================="
+}
+
+main

From 2459ecbc53fdb7c9c449930700cff290de15c152 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:07:05 +0700
Subject: [PATCH 04/17] DocSum - fix main

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 --
 .../amd/gpu/rocm-vllm/README.md               | 175 ------------
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 --------
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 --
 DocSum/docker_image_build/build.yaml          |   9 -
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ------------------
 6 files changed, 574 deletions(-)
 delete mode 100644 DocSum/Dockerfile-vllm-rocm
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
deleted file mode 100644
index f0e8a8743a..0000000000
--- a/DocSum/Dockerfile-vllm-rocm
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM rocm/vllm-dev:main
-
-# Set the working directory
-WORKDIR /workspace
-
-# Copy the api_server.py into the image
-ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
-
-# Expose the port used by the API server
-EXPOSE 8011
-
-# Set environment variables
-ENV HUGGINGFACE_HUB_CACHE=/workspace
-ENV WILM_USE_TRITON_FLASH_ATTENTION=0
-ENV PYTORCH_JIT=0
-
-# Set the entrypoint to the api_server.py script
-ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
deleted file mode 100644
index 4d41a5cd31..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
+++ /dev/null
@@ -1,175 +0,0 @@
-# Build and deploy DocSum Application on AMD GPU (ROCm)
-
-## Build images
-
-## 🚀 Build Docker Images
-
-First of all, you need to build Docker Images locally and install the python package of it.
-
-### 1. Build LLM Image
-
-```bash
-git clone https://github.com/opea-project/GenAIComps.git
-cd GenAIComps
-docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following four Docker Images:
-
-### 2. Build MegaService Docker Image
-
-To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
-
-```bash
-git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/DocSum/
-docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
-```
-
-### 3. Build UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-
-### 4. Build React UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
-
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-4. `opea/docsum-react-ui:latest`
-
-## 🚀 Start Microservices and MegaService
-
-### Required Models
-
-Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
-For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
-
-### Setup Environment Variables
-
-Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
-
-```bash
-export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${host_ip}
-export DOCSUM_TGI_SERVICE_PORT="18882"
-export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
-export DOCSUM_LLM_SERVER_PORT="8008"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
-```
-
-Note: Please replace with `host_ip` with your external IP address, do not use localhost.
-
-Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-Example for set isolation for 1 GPU
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-```
-
-Example for set isolation for 2 GPUs
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-      - /dev/dri/card1:/dev/dri/card1
-      - /dev/dri/renderD129:/dev/dri/renderD129
-```
-
-Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-### Start Microservice Docker Containers
-
-```bash
-cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
-docker compose up -d
-```
-
-### Validate Microservices
-
-1. TGI Service
-
-   ```bash
-   curl http://${host_ip}:8008/generate \
-     -X POST \
-     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
-     -H 'Content-Type: application/json'
-   ```
-
-2. LLM Microservice
-
-   ```bash
-   curl http://${host_ip}:9000/v1/docsum \
-     -X POST \
-     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
-     -H 'Content-Type: application/json'
-   ```
-
-3. MegaService
-
-   ```bash
-   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
-        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
-        }'
-   ```
-
-## 🚀 Launch the Svelte UI
-
-Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
-
-![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
-
-Here is an example for summarizing a article.
-
-![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
-
-## 🚀 Launch the React UI (Optional)
-
-To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
-
-```yaml
-docsum-rocm-react-ui-server:
-  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
-  container_name: docsum-rocm-react-ui-server
-  depends_on:
-    - docsum-rocm-backend-server
-  ports:
-    - "5174:80"
-  environment:
-    - no_proxy=${no_proxy}
-    - https_proxy=${https_proxy}
-    - http_proxy=${http_proxy}
-    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
-```
-
-Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
-
-![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
deleted file mode 100644
index 037aa06395..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-services:
-  docsum-vllm-service:
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
-    container_name: docsum-vllm-service
-    ports:
-      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_HUB_DISABLE_PROGRESS_BARS: 1
-      HF_HUB_ENABLE_HF_TRANSFER: 0
-      WILM_USE_TRITON_FLASH_ATTENTION: 0
-      PYTORCH_JIT: 0
-    volumes:
-      - "./data:/data"
-    shm_size: 20G
-    devices:
-      - /dev/kfd:/dev/kfd
-      - /dev/dri/:/dev/dri/
-    cap_add:
-      - SYS_PTRACE
-    group_add:
-      - video
-    security_opt:
-      - seccomp:unconfined
-      - apparmor=unconfined
-    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
-    ipc: host
-
-  docsum-llm-server:
-    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-    container_name: docsum-llm-server
-    depends_on:
-      - docsum-vllm-service
-    ports:
-      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
-    ipc: host
-    cap_add:
-      - SYS_PTRACE
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
-      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
-      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
-      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
-    restart: unless-stopped
-
-  whisper-service:
-    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
-    container_name: whisper-service
-    ports:
-      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
-    ipc: host
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-    restart: unless-stopped
-
-  docsum-backend-server:
-    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
-    container_name: docsum-backend-server
-    depends_on:
-      - docsum-tgi-service
-      - docsum-llm-server
-    ports:
-      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      MEGA_SERVICE_HOST_IP: ${HOST_IP}
-      LLM_SERVICE_HOST_IP: ${HOST_IP}
-      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
-    ipc: host
-    restart: always
-
-  docsum-gradio-ui:
-    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
-    container_name: docsum-ui-server
-    depends_on:
-      - docsum-backend-server
-    ports:
-      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-    ipc: host
-    restart: always
-
-networks:
-  default:
-    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
deleted file mode 100644
index 43e71e0fbf..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-export HOST_IP=""
-export DOCSUM_MAX_INPUT_TOKENS=2048
-export DOCSUM_MAX_TOTAL_TOKENS=4096
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index dc0d546189..095fd28c93 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,12 +47,3 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-  vllm_rocm:
-    build:
-      args:
-        http_proxy: ${http_proxy}
-        https_proxy: ${https_proxy}
-        no_proxy: ${no_proxy}
-      context: ../
-      dockerfile: ./Dockerfile-vllm-rocm
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
deleted file mode 100644
index d0919a019a..0000000000
--- a/DocSum/tests/test_compose_on_rocm_vllm.sh
+++ /dev/null
@@ -1,249 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-set -xe
-IMAGE_REPO=${IMAGE_REPO:-"opea"}
-IMAGE_TAG=${IMAGE_TAG:-"latest"}
-echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
-echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
-
-WORKPATH=$(dirname "$PWD")
-LOG_PATH="$WORKPATH/tests"
-ip_address=$(hostname -I | awk '{print $1}')
-export MAX_INPUT_TOKENS=1024
-export MAX_TOTAL_TOKENS=2048
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${ip_address}
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export MEGA_SERVICE_HOST_IP=${HOST_IP}
-export LLM_SERVICE_HOST_IP=${HOST_IP}
-export ASR_SERVICE_HOST_IP=${HOST_IP}
-export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
-
-function build_docker_images() {
-    opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
-
-    cd $WORKPATH/docker_image_build
-    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
-
-    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
-    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
-
-    docker images && sleep 1s
-}
-
-function start_services() {
-    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
-    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
-    # Start Docker Containers
-    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
-    sleep 1m
-}
-
-function validate_services() {
-    local URL="$1"
-    local EXPECTED_RESULT="$2"
-    local SERVICE_NAME="$3"
-    local DOCKER_NAME="$4"
-    local INPUT_DATA="$5"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
-
-    echo "==========================================="
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
-            echo "CONTENT==> $CONTENT"
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-get_base64_str() {
-    local file_name=$1
-    base64 -w 0 "$file_name"
-}
-
-# Function to generate input data for testing based on the document type
-input_data_for_test() {
-    local document_type=$1
-    case $document_type in
-        ("text")
-            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
-            ;;
-        ("audio")
-            get_base64_str "$WORKPATH/tests/data/test.wav"
-            ;;
-        ("video")
-            get_base64_str "$WORKPATH/tests/data/test.mp4"
-            ;;
-        (*)
-            echo "Invalid document type" >&2
-            exit 1
-            ;;
-    esac
-}
-
-function validate_microservices() {
-    # Check if the microservices are running correctly.
-
-    # whisper microservice
-    ulimit -s 65536
-    validate_services \
-        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
-        '{"asr_result":"well"}' \
-        "whisper-service" \
-        "whisper-service" \
-        "{\"audio\": \"$(input_data_for_test "audio")\"}"
-
-    # vLLM service
-    validate_services \
-        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
-        "generated_text" \
-        "docsum-vllm-service" \
-        "docsum-vllm-service" \
-        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
-
-    # llm microservice
-    validate_services \
-        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
-        "text" \
-        "docsum-llm-server" \
-        "docsum-llm-server" \
-        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-}
-
-function validate_megaservice() {
-    local SERVICE_NAME="docsum-backend-server"
-    local DOCKER_NAME="docsum-backend-server"
-    local EXPECTED_RESULT="[DONE]"
-    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-    local URL="${host_ip}:8888/v1/docsum"
-    local DATA_TYPE="type=text"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-function validate_megaservice_json() {
-    # Curl the Mega Service
-    echo ""
-    echo ">>> Checking text data with Content-Type: application/json"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-    echo ">>> Checking audio data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
-
-    echo ">>> Checking video data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
-
-}
-
-function stop_docker() {
-    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
-    docker compose stop && docker compose rm -f
-}
-
-function main() {
-    echo "==========================================="
-    echo ">>>> Stopping any running Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    if [[ "$IMAGE_REPO" == "opea" ]]; then
-        echo ">>>> Building Docker images..."
-        build_docker_images
-    fi
-
-    echo "==========================================="
-    echo ">>>> Starting Docker services..."
-    start_services
-
-    echo "==========================================="
-    echo ">>>> Validating microservices..."
-    validate_microservices
-
-    echo "==========================================="
-    echo ">>>> Validating megaservice..."
-    validate_megaservice
-    echo ">>>> Validating validate_megaservice_json..."
-    validate_megaservice_json
-
-    echo "==========================================="
-    echo ">>>> Stopping Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    echo ">>>> Pruning Docker system..."
-    echo y | docker system prune
-    echo ">>>> Docker system pruned successfully."
-    echo "==========================================="
-}
-
-main

From 6d5049dd1c6bb3e201c4ca807da6950e0ab4b9d2 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:02:03 +0700
Subject: [PATCH 05/17] DocSum - add files for deploy app with ROCm vLLM

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 ++
 .../amd/gpu/rocm-vllm/README.md               | 175 ++++++++++++
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 ++++++++
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 ++
 DocSum/docker_image_build/build.yaml          |   9 +
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ++++++++++++++++++
 6 files changed, 574 insertions(+)
 create mode 100644 DocSum/Dockerfile-vllm-rocm
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
new file mode 100644
index 0000000000..f0e8a8743a
--- /dev/null
+++ b/DocSum/Dockerfile-vllm-rocm
@@ -0,0 +1,18 @@
+FROM rocm/vllm-dev:main
+
+# Set the working directory
+WORKDIR /workspace
+
+# Copy the api_server.py into the image
+ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
+
+# Expose the port used by the API server
+EXPOSE 8011
+
+# Set environment variables
+ENV HUGGINGFACE_HUB_CACHE=/workspace
+ENV WILM_USE_TRITON_FLASH_ATTENTION=0
+ENV PYTORCH_JIT=0
+
+# Set the entrypoint to the api_server.py script
+ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
new file mode 100644
index 0000000000..4d41a5cd31
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
@@ -0,0 +1,175 @@
+# Build and deploy DocSum Application on AMD GPU (ROCm)
+
+## Build images
+
+## 🚀 Build Docker Images
+
+First of all, you need to build Docker Images locally and install the python package of it.
+
+### 1. Build LLM Image
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following four Docker Images:
+
+### 2. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples
+cd GenAIExamples/DocSum/
+docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+```
+
+### 3. Build UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+
+### 4. Build React UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
+
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+4. `opea/docsum-react-ui:latest`
+
+## 🚀 Start Microservices and MegaService
+
+### Required Models
+
+Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
+For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
+
+### Setup Environment Variables
+
+Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
+
+```bash
+export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${host_ip}
+export DOCSUM_TGI_SERVICE_PORT="18882"
+export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
+export DOCSUM_LLM_SERVER_PORT="8008"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
+```
+
+Note: Please replace with `host_ip` with your external IP address, do not use localhost.
+
+Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+Example for set isolation for 1 GPU
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+```
+
+Example for set isolation for 2 GPUs
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+      - /dev/dri/card1:/dev/dri/card1
+      - /dev/dri/renderD129:/dev/dri/renderD129
+```
+
+Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+### Start Microservice Docker Containers
+
+```bash
+cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
+docker compose up -d
+```
+
+### Validate Microservices
+
+1. TGI Service
+
+   ```bash
+   curl http://${host_ip}:8008/generate \
+     -X POST \
+     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
+     -H 'Content-Type: application/json'
+   ```
+
+2. LLM Microservice
+
+   ```bash
+   curl http://${host_ip}:9000/v1/docsum \
+     -X POST \
+     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
+     -H 'Content-Type: application/json'
+   ```
+
+3. MegaService
+
+   ```bash
+   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
+        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
+        }'
+   ```
+
+## 🚀 Launch the Svelte UI
+
+Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
+
+![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
+
+Here is an example for summarizing a article.
+
+![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
+
+## 🚀 Launch the React UI (Optional)
+
+To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
+
+```yaml
+docsum-rocm-react-ui-server:
+  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
+  container_name: docsum-rocm-react-ui-server
+  depends_on:
+    - docsum-rocm-backend-server
+  ports:
+    - "5174:80"
+  environment:
+    - no_proxy=${no_proxy}
+    - https_proxy=${https_proxy}
+    - http_proxy=${http_proxy}
+    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+```
+
+Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
+
+![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
new file mode 100644
index 0000000000..037aa06395
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
@@ -0,0 +1,107 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  docsum-vllm-service:
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
+    container_name: docsum-vllm-service
+    ports:
+      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+      WILM_USE_TRITON_FLASH_ATTENTION: 0
+      PYTORCH_JIT: 0
+    volumes:
+      - "./data:/data"
+    shm_size: 20G
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri/:/dev/dri/
+    cap_add:
+      - SYS_PTRACE
+    group_add:
+      - video
+    security_opt:
+      - seccomp:unconfined
+      - apparmor=unconfined
+    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
+    ipc: host
+
+  docsum-llm-server:
+    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+    container_name: docsum-llm-server
+    depends_on:
+      - docsum-vllm-service
+    ports:
+      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
+    ipc: host
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
+      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
+    restart: unless-stopped
+
+  whisper-service:
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
+    container_name: whisper-service
+    ports:
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    restart: unless-stopped
+
+  docsum-backend-server:
+    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
+    container_name: docsum-backend-server
+    depends_on:
+      - docsum-tgi-service
+      - docsum-llm-server
+    ports:
+      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
+    ipc: host
+    restart: always
+
+  docsum-gradio-ui:
+    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
+    container_name: docsum-ui-server
+    depends_on:
+      - docsum-backend-server
+    ports:
+      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
new file mode 100644
index 0000000000..43e71e0fbf
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+export HOST_IP=""
+export DOCSUM_MAX_INPUT_TOKENS=2048
+export DOCSUM_MAX_TOTAL_TOKENS=4096
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index 095fd28c93..dc0d546189 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,3 +47,12 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+  vllm_rocm:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      context: ../
+      dockerfile: ./Dockerfile-vllm-rocm
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
new file mode 100644
index 0000000000..d0919a019a
--- /dev/null
+++ b/DocSum/tests/test_compose_on_rocm_vllm.sh
@@ -0,0 +1,249 @@
+#!/bin/bash
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+
+WORKPATH=$(dirname "$PWD")
+LOG_PATH="$WORKPATH/tests"
+ip_address=$(hostname -I | awk '{print $1}')
+export MAX_INPUT_TOKENS=1024
+export MAX_TOTAL_TOKENS=2048
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${ip_address}
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export MEGA_SERVICE_HOST_IP=${HOST_IP}
+export LLM_SERVICE_HOST_IP=${HOST_IP}
+export ASR_SERVICE_HOST_IP=${HOST_IP}
+export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
+
+function build_docker_images() {
+    opea_branch=${opea_branch:-"main"}
+    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
+    if [[ "${opea_branch}" != "main" ]]; then
+        cd $WORKPATH
+        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
+        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
+        find . -type f -name "Dockerfile*" | while read -r file; do
+            echo "Processing file: $file"
+            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
+        done
+    fi
+
+    cd $WORKPATH/docker_image_build
+    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
+    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
+
+    docker images && sleep 1s
+}
+
+function start_services() {
+    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
+    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
+    # Start Docker Containers
+    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
+    sleep 1m
+}
+
+function validate_services() {
+    local URL="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+    local DOCKER_NAME="$4"
+    local INPUT_DATA="$5"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
+
+    echo "==========================================="
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
+            echo "CONTENT==> $CONTENT"
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+get_base64_str() {
+    local file_name=$1
+    base64 -w 0 "$file_name"
+}
+
+# Function to generate input data for testing based on the document type
+input_data_for_test() {
+    local document_type=$1
+    case $document_type in
+        ("text")
+            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
+            ;;
+        ("audio")
+            get_base64_str "$WORKPATH/tests/data/test.wav"
+            ;;
+        ("video")
+            get_base64_str "$WORKPATH/tests/data/test.mp4"
+            ;;
+        (*)
+            echo "Invalid document type" >&2
+            exit 1
+            ;;
+    esac
+}
+
+function validate_microservices() {
+    # Check if the microservices are running correctly.
+
+    # whisper microservice
+    ulimit -s 65536
+    validate_services \
+        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
+        '{"asr_result":"well"}' \
+        "whisper-service" \
+        "whisper-service" \
+        "{\"audio\": \"$(input_data_for_test "audio")\"}"
+
+    # vLLM service
+    validate_services \
+        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
+        "generated_text" \
+        "docsum-vllm-service" \
+        "docsum-vllm-service" \
+        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
+
+    # llm microservice
+    validate_services \
+        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
+        "text" \
+        "docsum-llm-server" \
+        "docsum-llm-server" \
+        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+}
+
+function validate_megaservice() {
+    local SERVICE_NAME="docsum-backend-server"
+    local DOCKER_NAME="docsum-backend-server"
+    local EXPECTED_RESULT="[DONE]"
+    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
+    local URL="${host_ip}:8888/v1/docsum"
+    local DATA_TYPE="type=text"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+function validate_megaservice_json() {
+    # Curl the Mega Service
+    echo ""
+    echo ">>> Checking text data with Content-Type: application/json"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+    echo ">>> Checking audio data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
+
+    echo ">>> Checking video data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
+
+}
+
+function stop_docker() {
+    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
+    docker compose stop && docker compose rm -f
+}
+
+function main() {
+    echo "==========================================="
+    echo ">>>> Stopping any running Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    if [[ "$IMAGE_REPO" == "opea" ]]; then
+        echo ">>>> Building Docker images..."
+        build_docker_images
+    fi
+
+    echo "==========================================="
+    echo ">>>> Starting Docker services..."
+    start_services
+
+    echo "==========================================="
+    echo ">>>> Validating microservices..."
+    validate_microservices
+
+    echo "==========================================="
+    echo ">>>> Validating megaservice..."
+    validate_megaservice
+    echo ">>>> Validating validate_megaservice_json..."
+    validate_megaservice_json
+
+    echo "==========================================="
+    echo ">>>> Stopping Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    echo ">>>> Pruning Docker system..."
+    echo y | docker system prune
+    echo ">>>> Docker system pruned successfully."
+    echo "==========================================="
+}
+
+main

From 9dfbdc5cffe708b084e7367d6df2910908f5e76a Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:07:05 +0700
Subject: [PATCH 06/17] DocSum - fix main

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 --
 .../amd/gpu/rocm-vllm/README.md               | 175 ------------
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 --------
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 --
 DocSum/docker_image_build/build.yaml          |   9 -
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ------------------
 6 files changed, 574 deletions(-)
 delete mode 100644 DocSum/Dockerfile-vllm-rocm
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
deleted file mode 100644
index f0e8a8743a..0000000000
--- a/DocSum/Dockerfile-vllm-rocm
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM rocm/vllm-dev:main
-
-# Set the working directory
-WORKDIR /workspace
-
-# Copy the api_server.py into the image
-ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
-
-# Expose the port used by the API server
-EXPOSE 8011
-
-# Set environment variables
-ENV HUGGINGFACE_HUB_CACHE=/workspace
-ENV WILM_USE_TRITON_FLASH_ATTENTION=0
-ENV PYTORCH_JIT=0
-
-# Set the entrypoint to the api_server.py script
-ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
deleted file mode 100644
index 4d41a5cd31..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
+++ /dev/null
@@ -1,175 +0,0 @@
-# Build and deploy DocSum Application on AMD GPU (ROCm)
-
-## Build images
-
-## 🚀 Build Docker Images
-
-First of all, you need to build Docker Images locally and install the python package of it.
-
-### 1. Build LLM Image
-
-```bash
-git clone https://github.com/opea-project/GenAIComps.git
-cd GenAIComps
-docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following four Docker Images:
-
-### 2. Build MegaService Docker Image
-
-To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
-
-```bash
-git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/DocSum/
-docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
-```
-
-### 3. Build UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-
-### 4. Build React UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
-
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-4. `opea/docsum-react-ui:latest`
-
-## 🚀 Start Microservices and MegaService
-
-### Required Models
-
-Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
-For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
-
-### Setup Environment Variables
-
-Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
-
-```bash
-export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${host_ip}
-export DOCSUM_TGI_SERVICE_PORT="18882"
-export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
-export DOCSUM_LLM_SERVER_PORT="8008"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
-```
-
-Note: Please replace with `host_ip` with your external IP address, do not use localhost.
-
-Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-Example for set isolation for 1 GPU
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-```
-
-Example for set isolation for 2 GPUs
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-      - /dev/dri/card1:/dev/dri/card1
-      - /dev/dri/renderD129:/dev/dri/renderD129
-```
-
-Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-### Start Microservice Docker Containers
-
-```bash
-cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
-docker compose up -d
-```
-
-### Validate Microservices
-
-1. TGI Service
-
-   ```bash
-   curl http://${host_ip}:8008/generate \
-     -X POST \
-     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
-     -H 'Content-Type: application/json'
-   ```
-
-2. LLM Microservice
-
-   ```bash
-   curl http://${host_ip}:9000/v1/docsum \
-     -X POST \
-     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
-     -H 'Content-Type: application/json'
-   ```
-
-3. MegaService
-
-   ```bash
-   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
-        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
-        }'
-   ```
-
-## 🚀 Launch the Svelte UI
-
-Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
-
-![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
-
-Here is an example for summarizing a article.
-
-![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
-
-## 🚀 Launch the React UI (Optional)
-
-To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
-
-```yaml
-docsum-rocm-react-ui-server:
-  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
-  container_name: docsum-rocm-react-ui-server
-  depends_on:
-    - docsum-rocm-backend-server
-  ports:
-    - "5174:80"
-  environment:
-    - no_proxy=${no_proxy}
-    - https_proxy=${https_proxy}
-    - http_proxy=${http_proxy}
-    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
-```
-
-Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
-
-![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
deleted file mode 100644
index 037aa06395..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-services:
-  docsum-vllm-service:
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
-    container_name: docsum-vllm-service
-    ports:
-      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_HUB_DISABLE_PROGRESS_BARS: 1
-      HF_HUB_ENABLE_HF_TRANSFER: 0
-      WILM_USE_TRITON_FLASH_ATTENTION: 0
-      PYTORCH_JIT: 0
-    volumes:
-      - "./data:/data"
-    shm_size: 20G
-    devices:
-      - /dev/kfd:/dev/kfd
-      - /dev/dri/:/dev/dri/
-    cap_add:
-      - SYS_PTRACE
-    group_add:
-      - video
-    security_opt:
-      - seccomp:unconfined
-      - apparmor=unconfined
-    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
-    ipc: host
-
-  docsum-llm-server:
-    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-    container_name: docsum-llm-server
-    depends_on:
-      - docsum-vllm-service
-    ports:
-      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
-    ipc: host
-    cap_add:
-      - SYS_PTRACE
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
-      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
-      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
-      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
-    restart: unless-stopped
-
-  whisper-service:
-    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
-    container_name: whisper-service
-    ports:
-      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
-    ipc: host
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-    restart: unless-stopped
-
-  docsum-backend-server:
-    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
-    container_name: docsum-backend-server
-    depends_on:
-      - docsum-tgi-service
-      - docsum-llm-server
-    ports:
-      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      MEGA_SERVICE_HOST_IP: ${HOST_IP}
-      LLM_SERVICE_HOST_IP: ${HOST_IP}
-      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
-    ipc: host
-    restart: always
-
-  docsum-gradio-ui:
-    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
-    container_name: docsum-ui-server
-    depends_on:
-      - docsum-backend-server
-    ports:
-      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-    ipc: host
-    restart: always
-
-networks:
-  default:
-    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
deleted file mode 100644
index 43e71e0fbf..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-export HOST_IP=""
-export DOCSUM_MAX_INPUT_TOKENS=2048
-export DOCSUM_MAX_TOTAL_TOKENS=4096
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index dc0d546189..095fd28c93 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,12 +47,3 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-  vllm_rocm:
-    build:
-      args:
-        http_proxy: ${http_proxy}
-        https_proxy: ${https_proxy}
-        no_proxy: ${no_proxy}
-      context: ../
-      dockerfile: ./Dockerfile-vllm-rocm
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
deleted file mode 100644
index d0919a019a..0000000000
--- a/DocSum/tests/test_compose_on_rocm_vllm.sh
+++ /dev/null
@@ -1,249 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-set -xe
-IMAGE_REPO=${IMAGE_REPO:-"opea"}
-IMAGE_TAG=${IMAGE_TAG:-"latest"}
-echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
-echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
-
-WORKPATH=$(dirname "$PWD")
-LOG_PATH="$WORKPATH/tests"
-ip_address=$(hostname -I | awk '{print $1}')
-export MAX_INPUT_TOKENS=1024
-export MAX_TOTAL_TOKENS=2048
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${ip_address}
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export MEGA_SERVICE_HOST_IP=${HOST_IP}
-export LLM_SERVICE_HOST_IP=${HOST_IP}
-export ASR_SERVICE_HOST_IP=${HOST_IP}
-export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
-
-function build_docker_images() {
-    opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
-
-    cd $WORKPATH/docker_image_build
-    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
-
-    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
-    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
-
-    docker images && sleep 1s
-}
-
-function start_services() {
-    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
-    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
-    # Start Docker Containers
-    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
-    sleep 1m
-}
-
-function validate_services() {
-    local URL="$1"
-    local EXPECTED_RESULT="$2"
-    local SERVICE_NAME="$3"
-    local DOCKER_NAME="$4"
-    local INPUT_DATA="$5"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
-
-    echo "==========================================="
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
-            echo "CONTENT==> $CONTENT"
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-get_base64_str() {
-    local file_name=$1
-    base64 -w 0 "$file_name"
-}
-
-# Function to generate input data for testing based on the document type
-input_data_for_test() {
-    local document_type=$1
-    case $document_type in
-        ("text")
-            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
-            ;;
-        ("audio")
-            get_base64_str "$WORKPATH/tests/data/test.wav"
-            ;;
-        ("video")
-            get_base64_str "$WORKPATH/tests/data/test.mp4"
-            ;;
-        (*)
-            echo "Invalid document type" >&2
-            exit 1
-            ;;
-    esac
-}
-
-function validate_microservices() {
-    # Check if the microservices are running correctly.
-
-    # whisper microservice
-    ulimit -s 65536
-    validate_services \
-        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
-        '{"asr_result":"well"}' \
-        "whisper-service" \
-        "whisper-service" \
-        "{\"audio\": \"$(input_data_for_test "audio")\"}"
-
-    # vLLM service
-    validate_services \
-        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
-        "generated_text" \
-        "docsum-vllm-service" \
-        "docsum-vllm-service" \
-        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
-
-    # llm microservice
-    validate_services \
-        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
-        "text" \
-        "docsum-llm-server" \
-        "docsum-llm-server" \
-        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-}
-
-function validate_megaservice() {
-    local SERVICE_NAME="docsum-backend-server"
-    local DOCKER_NAME="docsum-backend-server"
-    local EXPECTED_RESULT="[DONE]"
-    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-    local URL="${host_ip}:8888/v1/docsum"
-    local DATA_TYPE="type=text"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-function validate_megaservice_json() {
-    # Curl the Mega Service
-    echo ""
-    echo ">>> Checking text data with Content-Type: application/json"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-    echo ">>> Checking audio data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
-
-    echo ">>> Checking video data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
-
-}
-
-function stop_docker() {
-    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
-    docker compose stop && docker compose rm -f
-}
-
-function main() {
-    echo "==========================================="
-    echo ">>>> Stopping any running Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    if [[ "$IMAGE_REPO" == "opea" ]]; then
-        echo ">>>> Building Docker images..."
-        build_docker_images
-    fi
-
-    echo "==========================================="
-    echo ">>>> Starting Docker services..."
-    start_services
-
-    echo "==========================================="
-    echo ">>>> Validating microservices..."
-    validate_microservices
-
-    echo "==========================================="
-    echo ">>>> Validating megaservice..."
-    validate_megaservice
-    echo ">>>> Validating validate_megaservice_json..."
-    validate_megaservice_json
-
-    echo "==========================================="
-    echo ">>>> Stopping Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    echo ">>>> Pruning Docker system..."
-    echo y | docker system prune
-    echo ">>>> Docker system pruned successfully."
-    echo "==========================================="
-}
-
-main

From a8857ae326b2d71ca66bc6f86715ac9ab467ac85 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:02:03 +0700
Subject: [PATCH 07/17] DocSum - add files for deploy app with ROCm vLLM

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 ++
 .../amd/gpu/rocm-vllm/README.md               | 175 ++++++++++++
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 ++++++++
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 ++
 DocSum/docker_image_build/build.yaml          |   9 +
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ++++++++++++++++++
 6 files changed, 574 insertions(+)
 create mode 100644 DocSum/Dockerfile-vllm-rocm
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
new file mode 100644
index 0000000000..f0e8a8743a
--- /dev/null
+++ b/DocSum/Dockerfile-vllm-rocm
@@ -0,0 +1,18 @@
+FROM rocm/vllm-dev:main
+
+# Set the working directory
+WORKDIR /workspace
+
+# Copy the api_server.py into the image
+ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
+
+# Expose the port used by the API server
+EXPOSE 8011
+
+# Set environment variables
+ENV HUGGINGFACE_HUB_CACHE=/workspace
+ENV WILM_USE_TRITON_FLASH_ATTENTION=0
+ENV PYTORCH_JIT=0
+
+# Set the entrypoint to the api_server.py script
+ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
new file mode 100644
index 0000000000..4d41a5cd31
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
@@ -0,0 +1,175 @@
+# Build and deploy DocSum Application on AMD GPU (ROCm)
+
+## Build images
+
+## 🚀 Build Docker Images
+
+First of all, you need to build Docker Images locally and install the python package of it.
+
+### 1. Build LLM Image
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following four Docker Images:
+
+### 2. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples
+cd GenAIExamples/DocSum/
+docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+```
+
+### 3. Build UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+
+### 4. Build React UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
+
+docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum-tgi:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-ui:latest`
+4. `opea/docsum-react-ui:latest`
+
+## 🚀 Start Microservices and MegaService
+
+### Required Models
+
+Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
+For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
+
+### Setup Environment Variables
+
+Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
+
+```bash
+export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${host_ip}
+export DOCSUM_TGI_SERVICE_PORT="18882"
+export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
+export DOCSUM_LLM_SERVER_PORT="8008"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
+```
+
+Note: Please replace with `host_ip` with your external IP address, do not use localhost.
+
+Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+Example for set isolation for 1 GPU
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+```
+
+Example for set isolation for 2 GPUs
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+      - /dev/dri/card1:/dev/dri/card1
+      - /dev/dri/renderD129:/dev/dri/renderD129
+```
+
+Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+### Start Microservice Docker Containers
+
+```bash
+cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
+docker compose up -d
+```
+
+### Validate Microservices
+
+1. TGI Service
+
+   ```bash
+   curl http://${host_ip}:8008/generate \
+     -X POST \
+     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
+     -H 'Content-Type: application/json'
+   ```
+
+2. LLM Microservice
+
+   ```bash
+   curl http://${host_ip}:9000/v1/docsum \
+     -X POST \
+     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
+     -H 'Content-Type: application/json'
+   ```
+
+3. MegaService
+
+   ```bash
+   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
+        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
+        }'
+   ```
+
+## 🚀 Launch the Svelte UI
+
+Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
+
+![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
+
+Here is an example for summarizing a article.
+
+![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
+
+## 🚀 Launch the React UI (Optional)
+
+To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
+
+```yaml
+docsum-rocm-react-ui-server:
+  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
+  container_name: docsum-rocm-react-ui-server
+  depends_on:
+    - docsum-rocm-backend-server
+  ports:
+    - "5174:80"
+  environment:
+    - no_proxy=${no_proxy}
+    - https_proxy=${https_proxy}
+    - http_proxy=${http_proxy}
+    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+```
+
+Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
+
+![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
new file mode 100644
index 0000000000..037aa06395
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
@@ -0,0 +1,107 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  docsum-vllm-service:
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
+    container_name: docsum-vllm-service
+    ports:
+      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+      WILM_USE_TRITON_FLASH_ATTENTION: 0
+      PYTORCH_JIT: 0
+    volumes:
+      - "./data:/data"
+    shm_size: 20G
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri/:/dev/dri/
+    cap_add:
+      - SYS_PTRACE
+    group_add:
+      - video
+    security_opt:
+      - seccomp:unconfined
+      - apparmor=unconfined
+    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
+    ipc: host
+
+  docsum-llm-server:
+    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+    container_name: docsum-llm-server
+    depends_on:
+      - docsum-vllm-service
+    ports:
+      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
+    ipc: host
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
+      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
+    restart: unless-stopped
+
+  whisper-service:
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
+    container_name: whisper-service
+    ports:
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    restart: unless-stopped
+
+  docsum-backend-server:
+    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
+    container_name: docsum-backend-server
+    depends_on:
+      - docsum-tgi-service
+      - docsum-llm-server
+    ports:
+      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
+    ipc: host
+    restart: always
+
+  docsum-gradio-ui:
+    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
+    container_name: docsum-ui-server
+    depends_on:
+      - docsum-backend-server
+    ports:
+      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
new file mode 100644
index 0000000000..43e71e0fbf
--- /dev/null
+++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+export HOST_IP=""
+export DOCSUM_MAX_INPUT_TOKENS=2048
+export DOCSUM_MAX_TOTAL_TOKENS=4096
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index 095fd28c93..dc0d546189 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,3 +47,12 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+  vllm_rocm:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      context: ../
+      dockerfile: ./Dockerfile-vllm-rocm
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
new file mode 100644
index 0000000000..d0919a019a
--- /dev/null
+++ b/DocSum/tests/test_compose_on_rocm_vllm.sh
@@ -0,0 +1,249 @@
+#!/bin/bash
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+
+WORKPATH=$(dirname "$PWD")
+LOG_PATH="$WORKPATH/tests"
+ip_address=$(hostname -I | awk '{print $1}')
+export MAX_INPUT_TOKENS=1024
+export MAX_TOTAL_TOKENS=2048
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HOST_IP=${ip_address}
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_WHISPER_PORT="7066"
+export DOCSUM_BACKEND_SERVER_PORT="8888"
+export DOCSUM_FRONTEND_PORT="5173"
+export MEGA_SERVICE_HOST_IP=${HOST_IP}
+export LLM_SERVICE_HOST_IP=${HOST_IP}
+export ASR_SERVICE_HOST_IP=${HOST_IP}
+export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
+
+function build_docker_images() {
+    opea_branch=${opea_branch:-"main"}
+    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
+    if [[ "${opea_branch}" != "main" ]]; then
+        cd $WORKPATH
+        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
+        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
+        find . -type f -name "Dockerfile*" | while read -r file; do
+            echo "Processing file: $file"
+            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
+        done
+    fi
+
+    cd $WORKPATH/docker_image_build
+    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
+    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
+
+    docker images && sleep 1s
+}
+
+function start_services() {
+    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
+    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
+    # Start Docker Containers
+    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
+    sleep 1m
+}
+
+function validate_services() {
+    local URL="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+    local DOCKER_NAME="$4"
+    local INPUT_DATA="$5"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
+
+    echo "==========================================="
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
+            echo "CONTENT==> $CONTENT"
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+get_base64_str() {
+    local file_name=$1
+    base64 -w 0 "$file_name"
+}
+
+# Function to generate input data for testing based on the document type
+input_data_for_test() {
+    local document_type=$1
+    case $document_type in
+        ("text")
+            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
+            ;;
+        ("audio")
+            get_base64_str "$WORKPATH/tests/data/test.wav"
+            ;;
+        ("video")
+            get_base64_str "$WORKPATH/tests/data/test.mp4"
+            ;;
+        (*)
+            echo "Invalid document type" >&2
+            exit 1
+            ;;
+    esac
+}
+
+function validate_microservices() {
+    # Check if the microservices are running correctly.
+
+    # whisper microservice
+    ulimit -s 65536
+    validate_services \
+        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
+        '{"asr_result":"well"}' \
+        "whisper-service" \
+        "whisper-service" \
+        "{\"audio\": \"$(input_data_for_test "audio")\"}"
+
+    # vLLM service
+    validate_services \
+        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
+        "generated_text" \
+        "docsum-vllm-service" \
+        "docsum-vllm-service" \
+        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
+
+    # llm microservice
+    validate_services \
+        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
+        "text" \
+        "docsum-llm-server" \
+        "docsum-llm-server" \
+        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+}
+
+function validate_megaservice() {
+    local SERVICE_NAME="docsum-backend-server"
+    local DOCKER_NAME="docsum-backend-server"
+    local EXPECTED_RESULT="[DONE]"
+    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
+    local URL="${host_ip}:8888/v1/docsum"
+    local DATA_TYPE="type=text"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
+
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+function validate_megaservice_json() {
+    # Curl the Mega Service
+    echo ""
+    echo ">>> Checking text data with Content-Type: application/json"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+
+    echo ">>> Checking audio data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
+
+    echo ">>> Checking video data"
+    validate_services \
+        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
+        "[DONE]" \
+        "docsum-backend-server" \
+        "docsum-backend-server" \
+        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
+
+}
+
+function stop_docker() {
+    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
+    docker compose stop && docker compose rm -f
+}
+
+function main() {
+    echo "==========================================="
+    echo ">>>> Stopping any running Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    if [[ "$IMAGE_REPO" == "opea" ]]; then
+        echo ">>>> Building Docker images..."
+        build_docker_images
+    fi
+
+    echo "==========================================="
+    echo ">>>> Starting Docker services..."
+    start_services
+
+    echo "==========================================="
+    echo ">>>> Validating microservices..."
+    validate_microservices
+
+    echo "==========================================="
+    echo ">>>> Validating megaservice..."
+    validate_megaservice
+    echo ">>>> Validating validate_megaservice_json..."
+    validate_megaservice_json
+
+    echo "==========================================="
+    echo ">>>> Stopping Docker containers..."
+    stop_docker
+
+    echo "==========================================="
+    echo ">>>> Pruning Docker system..."
+    echo y | docker system prune
+    echo ">>>> Docker system pruned successfully."
+    echo "==========================================="
+}
+
+main

From 5a38b266ac77a2bf0766cefab14ec62f28633a8d Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <YundunovCN@sibedge.com>
Date: Thu, 13 Feb 2025 10:07:05 +0700
Subject: [PATCH 08/17] DocSum - fix main

Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
---
 DocSum/Dockerfile-vllm-rocm                   |  18 --
 .../amd/gpu/rocm-vllm/README.md               | 175 ------------
 .../amd/gpu/rocm-vllm/compose.yaml            | 107 --------
 .../amd/gpu/rocm-vllm/set_env.sh              |  16 --
 DocSum/docker_image_build/build.yaml          |   9 -
 DocSum/tests/test_compose_on_rocm_vllm.sh     | 249 ------------------
 6 files changed, 574 deletions(-)
 delete mode 100644 DocSum/Dockerfile-vllm-rocm
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
 delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
 delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh

diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm
deleted file mode 100644
index f0e8a8743a..0000000000
--- a/DocSum/Dockerfile-vllm-rocm
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM rocm/vllm-dev:main
-
-# Set the working directory
-WORKDIR /workspace
-
-# Copy the api_server.py into the image
-ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
-
-# Expose the port used by the API server
-EXPOSE 8011
-
-# Set environment variables
-ENV HUGGINGFACE_HUB_CACHE=/workspace
-ENV WILM_USE_TRITON_FLASH_ATTENTION=0
-ENV PYTORCH_JIT=0
-
-# Set the entrypoint to the api_server.py script
-ENTRYPOINT ["python3", "/workspace/api_server.py"]
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
deleted file mode 100644
index 4d41a5cd31..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md
+++ /dev/null
@@ -1,175 +0,0 @@
-# Build and deploy DocSum Application on AMD GPU (ROCm)
-
-## Build images
-
-## 🚀 Build Docker Images
-
-First of all, you need to build Docker Images locally and install the python package of it.
-
-### 1. Build LLM Image
-
-```bash
-git clone https://github.com/opea-project/GenAIComps.git
-cd GenAIComps
-docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following four Docker Images:
-
-### 2. Build MegaService Docker Image
-
-To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
-
-```bash
-git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/DocSum/
-docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
-```
-
-### 3. Build UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-
-### 4. Build React UI Docker Image
-
-Build the frontend Docker image via below command:
-
-```bash
-cd GenAIExamples/DocSum/ui
-export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
-
-docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f ./docker/Dockerfile.react .
-```
-
-Then run the command `docker images`, you will have the following Docker Images:
-
-1. `opea/llm-docsum-tgi:latest`
-2. `opea/docsum:latest`
-3. `opea/docsum-ui:latest`
-4. `opea/docsum-react-ui:latest`
-
-## 🚀 Start Microservices and MegaService
-
-### Required Models
-
-Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model.
-For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
-
-### Setup Environment Variables
-
-Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
-
-```bash
-export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${host_ip}
-export DOCSUM_TGI_SERVICE_PORT="18882"
-export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
-export DOCSUM_LLM_SERVER_PORT="8008"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
-```
-
-Note: Please replace with `host_ip` with your external IP address, do not use localhost.
-
-Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-Example for set isolation for 1 GPU
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-```
-
-Example for set isolation for 2 GPUs
-
-```
-      - /dev/dri/card0:/dev/dri/card0
-      - /dev/dri/renderD128:/dev/dri/renderD128
-      - /dev/dri/card1:/dev/dri/card1
-      - /dev/dri/renderD129:/dev/dri/renderD129
-```
-
-Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
-
-### Start Microservice Docker Containers
-
-```bash
-cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
-docker compose up -d
-```
-
-### Validate Microservices
-
-1. TGI Service
-
-   ```bash
-   curl http://${host_ip}:8008/generate \
-     -X POST \
-     -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
-     -H 'Content-Type: application/json'
-   ```
-
-2. LLM Microservice
-
-   ```bash
-   curl http://${host_ip}:9000/v1/docsum \
-     -X POST \
-     -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
-     -H 'Content-Type: application/json'
-   ```
-
-3. MegaService
-
-   ```bash
-   curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{
-        "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false
-        }'
-   ```
-
-## 🚀 Launch the Svelte UI
-
-Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
-
-![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b)
-
-Here is an example for summarizing a article.
-
-![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55)
-
-## 🚀 Launch the React UI (Optional)
-
-To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below:
-
-```yaml
-docsum-rocm-react-ui-server:
-  image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}
-  container_name: docsum-rocm-react-ui-server
-  depends_on:
-    - docsum-rocm-backend-server
-  ports:
-    - "5174:80"
-  environment:
-    - no_proxy=${no_proxy}
-    - https_proxy=${https_proxy}
-    - http_proxy=${http_proxy}
-    - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
-```
-
-Open this URL `http://{host_ip}:5175` in your browser to access the frontend.
-
-![project-screenshot](../../../../assets/img/docsum-ui-react.png)
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
deleted file mode 100644
index 037aa06395..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-services:
-  docsum-vllm-service:
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
-    container_name: docsum-vllm-service
-    ports:
-      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_HUB_DISABLE_PROGRESS_BARS: 1
-      HF_HUB_ENABLE_HF_TRANSFER: 0
-      WILM_USE_TRITON_FLASH_ATTENTION: 0
-      PYTORCH_JIT: 0
-    volumes:
-      - "./data:/data"
-    shm_size: 20G
-    devices:
-      - /dev/kfd:/dev/kfd
-      - /dev/dri/:/dev/dri/
-    cap_add:
-      - SYS_PTRACE
-    group_add:
-      - video
-    security_opt:
-      - seccomp:unconfined
-      - apparmor=unconfined
-    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
-    ipc: host
-
-  docsum-llm-server:
-    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-    container_name: docsum-llm-server
-    depends_on:
-      - docsum-vllm-service
-    ports:
-      - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000"
-    ipc: host
-    cap_add:
-      - SYS_PTRACE
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
-      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
-      LOGFLAG: ${DOCSUM_LOGFLAG:-False}
-      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
-      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
-    restart: unless-stopped
-
-  whisper-service:
-    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
-    container_name: whisper-service
-    ports:
-      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
-    ipc: host
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-    restart: unless-stopped
-
-  docsum-backend-server:
-    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
-    container_name: docsum-backend-server
-    depends_on:
-      - docsum-tgi-service
-      - docsum-llm-server
-    ports:
-      - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      MEGA_SERVICE_HOST_IP: ${HOST_IP}
-      LLM_SERVICE_HOST_IP: ${HOST_IP}
-      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
-    ipc: host
-    restart: always
-
-  docsum-gradio-ui:
-    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
-    container_name: docsum-ui-server
-    depends_on:
-      - docsum-backend-server
-    ports:
-      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
-    environment:
-      no_proxy: ${no_proxy}
-      https_proxy: ${https_proxy}
-      http_proxy: ${http_proxy}
-      BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-      DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT}
-    ipc: host
-    restart: always
-
-networks:
-  default:
-    driver: bridge
diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
deleted file mode 100644
index 43e71e0fbf..0000000000
--- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-export HOST_IP=""
-export DOCSUM_MAX_INPUT_TOKENS=2048
-export DOCSUM_MAX_TOTAL_TOKENS=4096
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=""
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml
index dc0d546189..095fd28c93 100644
--- a/DocSum/docker_image_build/build.yaml
+++ b/DocSum/docker_image_build/build.yaml
@@ -47,12 +47,3 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
-  vllm_rocm:
-    build:
-      args:
-        http_proxy: ${http_proxy}
-        https_proxy: ${https_proxy}
-        no_proxy: ${no_proxy}
-      context: ../
-      dockerfile: ./Dockerfile-vllm-rocm
-    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh
deleted file mode 100644
index d0919a019a..0000000000
--- a/DocSum/tests/test_compose_on_rocm_vllm.sh
+++ /dev/null
@@ -1,249 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2024 Advanced Micro Devices, Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-set -xe
-IMAGE_REPO=${IMAGE_REPO:-"opea"}
-IMAGE_TAG=${IMAGE_TAG:-"latest"}
-echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
-echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
-
-WORKPATH=$(dirname "$PWD")
-LOG_PATH="$WORKPATH/tests"
-ip_address=$(hostname -I | awk '{print $1}')
-export MAX_INPUT_TOKENS=1024
-export MAX_TOTAL_TOKENS=2048
-export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${ip_address}
-export DOCSUM_VLLM_SERVICE_PORT="8008"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_WHISPER_PORT="7066"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
-export MEGA_SERVICE_HOST_IP=${HOST_IP}
-export LLM_SERVICE_HOST_IP=${HOST_IP}
-export ASR_SERVICE_HOST_IP=${HOST_IP}
-export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"
-
-function build_docker_images() {
-    opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
-
-    cd $WORKPATH/docker_image_build
-    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
-
-    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper"
-    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
-
-    docker images && sleep 1s
-}
-
-function start_services() {
-    cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm
-    sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
-    # Start Docker Containers
-    docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
-    sleep 1m
-}
-
-function validate_services() {
-    local URL="$1"
-    local EXPECTED_RESULT="$2"
-    local SERVICE_NAME="$3"
-    local DOCKER_NAME="$4"
-    local INPUT_DATA="$5"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
-
-    echo "==========================================="
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "EXPECTED_RESULT==> $EXPECTED_RESULT"
-            echo "CONTENT==> $CONTENT"
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-get_base64_str() {
-    local file_name=$1
-    base64 -w 0 "$file_name"
-}
-
-# Function to generate input data for testing based on the document type
-input_data_for_test() {
-    local document_type=$1
-    case $document_type in
-        ("text")
-            echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are."
-            ;;
-        ("audio")
-            get_base64_str "$WORKPATH/tests/data/test.wav"
-            ;;
-        ("video")
-            get_base64_str "$WORKPATH/tests/data/test.mp4"
-            ;;
-        (*)
-            echo "Invalid document type" >&2
-            exit 1
-            ;;
-    esac
-}
-
-function validate_microservices() {
-    # Check if the microservices are running correctly.
-
-    # whisper microservice
-    ulimit -s 65536
-    validate_services \
-        "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \
-        '{"asr_result":"well"}' \
-        "whisper-service" \
-        "whisper-service" \
-        "{\"audio\": \"$(input_data_for_test "audio")\"}"
-
-    # vLLM service
-    validate_services \
-        "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \
-        "generated_text" \
-        "docsum-vllm-service" \
-        "docsum-vllm-service" \
-        '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
-
-    # llm microservice
-    validate_services \
-        "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \
-        "text" \
-        "docsum-llm-server" \
-        "docsum-llm-server" \
-        '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-}
-
-function validate_megaservice() {
-    local SERVICE_NAME="docsum-backend-server"
-    local DOCKER_NAME="docsum-backend-server"
-    local EXPECTED_RESULT="[DONE]"
-    local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-    local URL="${host_ip}:8888/v1/docsum"
-    local DATA_TYPE="type=text"
-
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL")
-
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
-
-        local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
-
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-        fi
-    else
-        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-        exit 1
-    fi
-    sleep 1s
-}
-
-function validate_megaservice_json() {
-    # Curl the Mega Service
-    echo ""
-    echo ">>> Checking text data with Content-Type: application/json"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
-
-    echo ">>> Checking audio data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"audio\",  \"messages\": \"$(input_data_for_test "audio")\"}"
-
-    echo ">>> Checking video data"
-    validate_services \
-        "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \
-        "[DONE]" \
-        "docsum-backend-server" \
-        "docsum-backend-server" \
-        "{\"type\": \"video\",  \"messages\": \"$(input_data_for_test "video")\"}"
-
-}
-
-function stop_docker() {
-    cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/
-    docker compose stop && docker compose rm -f
-}
-
-function main() {
-    echo "==========================================="
-    echo ">>>> Stopping any running Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    if [[ "$IMAGE_REPO" == "opea" ]]; then
-        echo ">>>> Building Docker images..."
-        build_docker_images
-    fi
-
-    echo "==========================================="
-    echo ">>>> Starting Docker services..."
-    start_services
-
-    echo "==========================================="
-    echo ">>>> Validating microservices..."
-    validate_microservices
-
-    echo "==========================================="
-    echo ">>>> Validating megaservice..."
-    validate_megaservice
-    echo ">>>> Validating validate_megaservice_json..."
-    validate_megaservice_json
-
-    echo "==========================================="
-    echo ">>>> Stopping Docker containers..."
-    stop_docker
-
-    echo "==========================================="
-    echo ">>>> Pruning Docker system..."
-    echo y | docker system prune
-    echo ">>>> Docker system pruned successfully."
-    echo "==========================================="
-}
-
-main

From 895f8015292a87fb9b0f1ab561a4d723cd042675 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Tue, 22 Apr 2025 00:16:15 +0700
Subject: [PATCH 09/17] ChatQnA - refactoring README.md

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md | 675 ++++--------------
 .../docker_compose/amd/gpu/rocm/README_old.md | 621 ++++++++++++++++
 2 files changed, 751 insertions(+), 545 deletions(-)
 create mode 100644 ChatQnA/docker_compose/amd/gpu/rocm/README_old.md

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index 0edcf44141..9c4887aaed 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -1,621 +1,206 @@
-# Build and Deploy ChatQnA Application on AMD GPU (ROCm)
+# Deploying ChatQnA on AMD ROCm GPU
 
-## Build Docker Images
+This document outlines the single node deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservices on Intel Xeon server and AMD GPU. The steps include pulling Docker images, container deployment via Docker Compose, and service execution using microservices `llm`.
 
-### 1. Build Docker Image
+Note: The default LLM is `meta-llama/Meta-Llama-3-8B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models).
 
-- #### Create application install directory and go to it:
+## Table of Contents
 
-  ```bash
-  mkdir ~/chatqna-install && cd chatqna-install
-  ```
+1. [ChatQnA Quick Start Deployment](#chatqna-quick-start-deployment)
+2. [ChatQnA Docker Compose Files](#chatqna-docker-compose-files)
+3. [Validate Microservices](#validate-microservices)
+4. [Conclusion](#conclusion)
 
-- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
+## ChatQnA Quick Start Deployment
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git
-  ```
+This section describes how to quickly deploy and test the ChatQnA service manually on an AMD ROCm GPU. The basic steps are:
 
-  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
+1. [Access the Code](#access-the-code)
+2. [Configure the Deployment Environment](#configure-the-deployment-environment)
+3. [Deploy the Services Using Docker Compose](#deploy-the-services-using-docker-compose)
+4. [Check the Deployment Status](#check-the-deployment-status)
+5. [Validate the Pipeline](#validate-the-pipeline)
+6. [Cleanup the Deployment](#cleanup-the-deployment)
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
-  ```
+### Access the Code
 
-  We remind you that when using a specific version of the code, you need to use the README from this version:
-
-- #### Go to build directory:
-
-  ```bash
-  cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_image_build
-  ```
-
-- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
-  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
-
-  ```bash
-  echo Y | rm -R GenAIComps
-  ```
-
-- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git
-  ```
-
-  If you use a specific tag of the GenAIExamples repository,
-  then you should also use the corresponding tag for GenAIComps. (v1.3 replace with its own value):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout v1.3
-  ```
-
-  We remind you that when using a specific version of the code, you need to use the README from this version.
-
-- #### Setting the list of images for the build (from the build file.yaml)
-
-  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
-
-  #### vLLM-based application
-
-  ```bash
-  service_list="dataprep retriever vllm-rocm chatqna chatqna-ui nginx"
-  ```
-
-  #### vLLM-based application with FaqGen
-
-  ```bash
-  service_list="dataprep retriever vllm-rocm llm-faqgen chatqna chatqna-ui nginx"
-  ```
-
-  #### TGI-based application
-
-  ```bash
-  service_list="dataprep retriever chatqna chatqna-ui nginx"
-  ```
-
-  #### TGI-based application with FaqGen
-
-  ```bash
-  service_list="dataprep retriever llm-faqgen chatqna chatqna-ui nginx"
-  ```
-
-- #### Pull Docker Images
-
-  ```bash
-  docker pull redis/redis-stack:7.2.0-v9
-  docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  ```
-
-- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
-
-  ```bash
-  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  ```
-
-- #### Build Docker Images
-
-  ```bash
-  docker compose -f build.yaml build ${service_list} --no-cache
-  ```
-
-  After the build, we check the list of images with the command:
-
-  ```bash
-  docker image ls
-  ```
-
-  The list of images should include:
-
-  ##### vLLM-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### vLLM-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
----
-
-## Deploy the ChatQnA Application
-
-### Docker Compose Configuration for AMD GPUs
-
-To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
-
-- compose_vllm.yaml - for vLLM-based application
-- compose_faqgen_vllm.yaml - for vLLM-based application with FaqGen
-- compose.yaml - for TGI-based
-- compose_faqgen.yaml - for TGI-based application with FaqGen
-
-```yaml
-shm_size: 1g
-devices:
-  - /dev/kfd:/dev/kfd
-  - /dev/dri:/dev/dri
-cap_add:
-  - SYS_PTRACE
-group_add:
-  - video
-security_opt:
-  - seccomp:unconfined
-```
-
-This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
-
-```yaml
-shm_size: 1g
-devices:
-  - /dev/kfd:/dev/kfd
-  - /dev/dri/card0:/dev/dri/card0
-  - /dev/dri/render128:/dev/dri/render128
-cap_add:
-  - SYS_PTRACE
-group_add:
-  - video
-security_opt:
-  - seccomp:unconfined
-```
-
-**How to Identify GPU Device IDs:**
-Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
-
-### Set deploy environment variables
-
-#### Setting variables in the operating system environment:
-
-##### Set variable HUGGINGFACEHUB_API_TOKEN:
-
-```bash
-### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
-export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
-```
-
-#### Set variables value in set_env\*\*\*\*.sh file:
-
-Go to Docker Compose directory:
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-```
-
-The example uses the Nano text editor. You can use any convenient text editor:
-
-#### If you use vLLM based application
-
-```bash
-nano set_env_vllm.sh
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-nano set_env_vllm_faqgen.sh
-```
-
-#### If you use TGI based application
-
-```bash
-nano set_env.sh
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-nano set_env_faqgen.sh
-```
-
-If you are in a proxy environment, also set the proxy-related environment variables:
+Clone the GenAIExample repository and access the ChatQnA AMD ROCm GPU platform Docker Compose files and supporting scripts:
 
 ```bash
-export http_proxy="Your_HTTP_Proxy"
-export https_proxy="Your_HTTPs_Proxy"
+git clone https://github.com/opea-project/GenAIExamples.git
+cd GenAIExamples/ChatQnA
 ```
 
-Set the values of the variables:
-
-- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
-
-  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
-
-  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
-
-  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
-
-  We set these values in the file set_env\*\*\*\*.sh
-
-- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
-  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
-
-#### Set variables with script set_env\*\*\*\*.sh
-
-#### If you use vLLM based application
+Then checkout a released version, such as v1.2:
 
 ```bash
-. set_env_vllm.sh
+git checkout v1.2
 ```
 
-#### If you use vLLM based application with FaqGen
+### Configure the Deployment Environment
 
-```bash
-. set_env_faqgen_vllm.sh
-```
+To set up environment variables for deploying ChatQnA services, set up some parameters specific to the deployment environment and source the `set_env_*.sh` script in this directory:
 
-#### If you use TGI based application
+if used vLLM - set_env_vllm.sh
+if used vLLM with FaqGen - set_env_faqgen_vllm.sh
+if used TGI - set_env.sh
+if used TGI with FaqGen - set_env_faqgen.sh
 
 ```bash
-. set_env.sh
+export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
+source ./set_env_*.sh # replace the script name with the appropriate one
 ```
 
-#### If you use TGI based application with FaqGen
+Consult the section on [ChatQnA Service configuration](#chatqna-configuration) for information on how service specific configuration parameters affect deployments.
 
-```bash
-. set_env_faqgen.sh
-```
+### Deploy the Services Using Docker Compose
 
-### Start the services:
-
-#### If you use vLLM based application
-
-```bash
-docker compose -f compose_vllm.yaml up -d
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-docker compose -f compose_faqgen_vllm.yaml up -d
-```
-
-#### If you use TGI based application
+To deploy the ChatQnA services, execute the `docker compose up` command with the appropriate arguments. For a default deployment with TGI, execute the command below. It uses the 'compose.yaml' file.
 
 ```bash
+cd docker_compose/amd/gpu/rocm
+# if used TGI
 docker compose -f compose.yaml up -d
-```
+# if used TGI with FaqGen
+# docker compose -f compose_faqgen.yaml up -d
+# if used vLLM
+# docker compose -f compose_vllm.yaml up -d
+# if used vLLM with FaqGen
+# docker compose -f compose_faqgen_vllm.yaml up -d
 
-#### If you use TGI based application with FaqGen
 
-```bash
-docker compose -f compose_faqgen.yaml up -d
 ```
 
-All containers should be running and should not restart:
-
-##### If you use vLLM based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use vLLM based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use TGI based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
-##### If you use TGI based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
----
-
-## Validate the Services
-
-### 1. Validate TEI Embedding Service
+> **Note**: developers should build docker image from source when:
+>
+> - Developing off the git main branch (as the container's ports in the repo may be different > from the published docker image).
+> - Unable to download the docker image.
+> - Use a specific version of Docker image.
 
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
-    -X POST \
-    -d '{"inputs":"What is Deep Learning?"}' \
-    -H 'Content-Type: application/json'
-```
+Please refer to the table below to build different microservices from source:
 
-Checking the response from the service. The response should be similar to text:
-
-```textmate
-[[0.00037115702,-0.06356819,0.0024758505,..................,0.022725677,0.016026087,-0.02125421,-0.02984927,-0.0049473033]]
-```
+| Microservice | Deployment Guide                                                                                                                  |
+| ------------ | --------------------------------------------------------------------------------------------------------------------------------- |
+| vLLM         | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker)                    |
+| LLM          | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                                                |
+| WHISPER      | [Whisper build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/asr/src#211-whisper-server-image)                |
+| SPEECHT5     | [SpeechT5 build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/tts/src#211-speecht5-server-image)              |
+| GPT-SOVITS   | [GPT-SOVITS build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/gpt-sovits/src#build-the-image) |
+| MegaService  | [MegaService build guide](../../../../README_miscellaneous.md#build-megaservice-docker-image)                                     |
+| UI           | [Basic UI build guide](../../../../README_miscellaneous.md#build-ui-docker-image)                                                 |
 
-If the service response has a meaningful response in the value,
-then we consider the TEI Embedding Service to be successfully launched
+### Check the Deployment Status
 
-### 2. Validate Retriever Microservice
+After running docker compose, check if all the containers launched via docker compose have started:
 
 ```bash
-export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
-curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
-  -X POST \
-  -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
-  -H 'Content-Type: application/json'
+docker ps -a
 ```
 
-Checking the response from the service. The response should be similar to JSON:
+For the default deployment, the following 5 containers should have started:
 
-```json
-{ "id": "e191846168aed1f80b2ea12df80844d2", "retrieved_docs": [], "initial_query": "test", "top_n": 1, "metadata": [] }
 ```
-
-If the response corresponds to the form of the provided JSON, then we consider the
-Retriever Microservice verification successful.
-
-### 3. Validate TEI Reranking Service
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
-    -X POST \
-    -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
-    -H 'Content-Type: application/json'
+1c67e44c39d2   opea/chatqna-ui:latest   "docker-entrypoint.s…"   About a minute ago   Up About a minute             0.0.0.0:5173->5173/tcp, :::5173->5173/tcp   audioqna-xeon-ui-server
+833a42677247   opea/chatqna:latest      "python audioqna.py"     About a minute ago   Up About a minute             0.0.0.0:3008->8888/tcp, :::3008->8888/tcp   audioqna-xeon-backend-server
+5dc4eb9bf499   opea/speecht5:latest      "python speecht5_ser…"   About a minute ago   Up About a minute             0.0.0.0:7055->7055/tcp, :::7055->7055/tcp   speecht5-service
+814e6efb1166   opea/vllm:latest          "python3 -m vllm.ent…"   About a minute ago   Up About a minute (healthy)   0.0.0.0:3006->80/tcp, :::3006->80/tcp       vllm-service
+46f7a00f4612   opea/whisper:latest       "python whisper_serv…"   About a minute ago   Up About a minute             0.0.0.0:7066->7066/tcp, :::7066->7066/tcp   whisper-service
 ```
 
-Checking the response from the service. The response should be similar to JSON:
+If any issues are encountered during deployment, refer to the [Troubleshooting](../../../../README_miscellaneous.md#troubleshooting) section.
 
-```json
-[
-  { "index": 1, "score": 0.94238955 },
-  { "index": 0, "score": 0.120219156 }
-]
-```
-
-If the response corresponds to the form of the provided JSON, then we consider the TEI Reranking Service
-verification successful.
+### Validate the Pipeline
 
-### 4. Validate the vLLM/TGI Service
-
-#### If you use vLLM:
+Once the AudioQnA services are running, test the pipeline using the following command:
 
 ```bash
-DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
-'"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
+# Test the AudioQnA megaservice by recording a .wav file, encoding the file into the base64 format, and then sending the base64 string to the megaservice endpoint.
+# The megaservice will return a spoken response as a base64 string. To listen to the response, decode the base64 string and save it as a .wav file.
+wget https://github.com/intel/intel-extension-for-transformers/raw/refs/heads/main/intel_extension_for_transformers/neural_chat/assets/audio/sample_2.wav
+base64_audio=$(base64 -w 0 sample_2.wav)
 
-curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "id": "chatcmpl-91003647d1c7469a89e399958f390f67",
-  "object": "chat.completion",
-  "created": 1742877228,
-  "model": "meta-llama/Meta-Llama-3-8B-Instruct",
-  "choices": [
-    {
-      "index": 0,
-      "message": {
-        "role": "assistant",
-        "content": "Deep Learning ( DL) is a subfield of Machine Learning (ML) that focuses on the design of algorithms and architectures inspired by the structure and function of the human brain. These algorithms are designed to analyze and interpret data that is presented in the form of patterns or signals, and they often mimic the way the human brain",
-        "tool_calls": []
-      },
-      "logprobs": null,
-      "finish_reason": "length",
-      "stop_reason": null
-    }
-  ],
-  "usage": { "prompt_tokens": 16, "total_tokens": 80, "completion_tokens": 64, "prompt_tokens_details": null },
-  "prompt_logprobs": null
-}
-```
-
-If the service response has a meaningful response in the value of the "choices.message.content" key,
-then we consider the vLLM service to be successfully launched
-
-#### If you use TGI:
-
-```bash
-DATA='{"inputs":"What is a Deep Learning?",'\
-'"parameters":{"max_new_tokens":64,"do_sample": true}}'
-
-curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "generated_text": " What is its application in Computer Vision?\nWhat is a Deep Learning?\nDeep learning is a subfield of machine learning that involves the use of artificial neural networks to model high-level abstractions in data. It involves the use of deep neural networks, which are composed of multiple layers, to learn complex patterns in data. The"
-}
-```
-
-If the service response has a meaningful response in the value of the "generated_text" key,
-then we consider the TGI service to be successfully launched
+# if you are using speecht5 as the tts service, voice can be "default" or "male"
+# if you are using gpt-sovits for the tts service, you can set the reference audio following https://github.com/opea-project/GenAIComps/blob/main/comps/third_parties/gpt-sovits/src/README.md
 
-### 5. Validate the LLM Service (if your used application with FaqGen)
-
-```bash
-DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
-'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
-'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
-
-curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
+curl http://${host_ip}:3008/v1/audioqna \
   -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
+  -H "Content-Type: application/json" \
+  -d "{\"audio\": \"${base64_audio}\", \"max_tokens\": 64, \"voice\": \"default\"}" \
+  | sed 's/^"//;s/"$//' | base64 -d > output.wav
 ```
 
-Checking the response from the service. The response should be similar to JSON:
+**Note** : Access the AudioQnA UI by web browser through this URL: `http://${host_ip}:5173`. Please confirm the `5173` port is opened in the firewall. To validate each microservice used in the pipeline refer to the [Validate Microservices](#validate-microservices) section.
 
-```json
-{
-  "id": "58f0632f5f03af31471b895b0d0d397b",
-  "text": " Q: What is Text Embeddings Inference (TEI)?\n         A: TEI is a toolkit for deploying and serving open source text embeddings and sequence classification models.\n\n         Q: What models does TEI support?\n         A: TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.\n\n         Q: What is the purpose of TEI?\n         A: The purpose of TEI is to enable high-performance extraction for text embeddings and sequence classification models.\n\n         Q: What are the benefits of using TEI?\n         A: The benefits of using TEI include high",
-  "prompt": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-}
-```
-
-If the service response has a meaningful response in the value of the "text" key,
-then we consider the LLM service to be successfully launched
+### Cleanup the Deployment
 
-### 6. Validate the MegaService
+To stop the containers associated with the deployment, execute the following command:
 
 ```bash
-curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
-  -H "Content-Type: application/json" \
-  -d '{"messages": "What is the revenue of Nike in 2023?"}'
-```
-
-Checking the response from the service. The response should be similar to text:
-
-```textmate
-data: b' What'
-data: b' is'
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b' Nike'
-data: b' in'
-data: b' '
-data: b'202'
-data: b'3'
-data: b'?\n'
-data: b'        '
-data: b' Answer'
-data: b':'
-data: b' According'
-data: b' to'
-data: b' the'
-data: b' search'
-data: b' results'
-data: b','
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b''
-
-data: [DONE]
-
+docker compose -f compose.yaml down
 ```
 
-If the output lines in the "data" keys contain words (tokens) containing meaning, then the service
-is considered launched successfully.
-
-### 7. Validate the Frontend (UI)
-
-To access the UI, use the URL - http://${EXTERNAL_HOST_IP}:${CHATQNA_NGINX_PORT}
-A page should open when you click through to this address:
+## AudioQnA Docker Compose Files
 
-![UI start page](../../../../assets/img/ui-starting-page.png)
+In the context of deploying an AudioQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
 
-If a page of this type has opened, then we believe that the service is running and responding,
-and we can proceed to functional UI testing.
+| File                                               | Description                                                                               |
+| -------------------------------------------------- | ----------------------------------------------------------------------------------------- |
+| [compose.yaml](./compose.yaml)                     | Default compose file using vllm as serving framework and redis as vector database         |
+| [compose_tgi.yaml](./compose_tgi.yaml)             | The LLM serving framework is TGI. All other configurations remain the same as the default |
+| [compose_multilang.yaml](./compose_multilang.yaml) | The TTS component is GPT-SoVITS. All other configurations remain the same as the default  |
 
-Let's enter the task for the service in the "Enter prompt here" field.
-For example, "What is a Deep Learning?" and press Enter.
-After that, a page with the result of the task should open:
+## Validate MicroServices
 
-#### If used application without FaqGen
+1. Whisper Service
 
-![UI result page](../../../../assets/img/ui-result-page.png)
+   ```bash
+   wget https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample.wav
+   curl http://${host_ip}:${WHISPER_SERVER_PORT}/v1/audio/transcriptions \
+     -H "Content-Type: multipart/form-data" \
+     -F file="@./sample.wav" \
+     -F model="openai/whisper-small"
+   ```
 
-#### If used application with FaqGen
+2. LLM backend Service
 
-![UI result page](../../../../assets/img/ui-result-page-faqgen.png)
+   In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready and the container (`vllm-service` or `tgi-service`) status shown via `docker ps` will be `healthy`. Before that, the status will be `health: starting`.
 
-If the result shown on the page is correct, then we consider the verification of the UI service to be successful.
+   Or try the command below to check whether the LLM serving is ready.
 
-### 5. Stop application
+   ```bash
+   # vLLM service
+   docker logs vllm-service 2>&1 | grep complete
+   # If the service is ready, you will get the response like below.
+   INFO:     Application startup complete.
+   ```
 
-#### If you use vLLM
+   ```bash
+   # TGI service
+   docker logs tgi-service | grep Connected
+   # If the service is ready, you will get the response like below.
+   2024-09-03T02:47:53.402023Z  INFO text_generation_router::server: router/src/server.rs:2311: Connected
+   ```
 
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_vllm.yaml down
-```
+   Then try the `cURL` command below to validate services.
 
-#### If you use vLLM with FaqGen
+   ```bash
+   # either vLLM or TGI service
+   curl http://${host_ip}:${LLM_SERVER_PORT}/v1/chat/completions \
+     -X POST \
+     -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \
+     -H 'Content-Type: application/json'
+   ```
 
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen_vllm.yaml down
-```
+3. TTS Service
 
-#### If you use TGI
+   ```bash
+   # speecht5 service
+   curl http://${host_ip}:${SPEECHT5_SERVER_PORT}/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3
 
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose.yaml down
-```
+   # gpt-sovits service (optional)
+   curl http://${host_ip}:${GPT_SOVITS_SERVER_PORT}/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3
+   ```
 
-#### If you use TGI with FaqGen
+## Conclusion
 
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen.yaml down
-```
+This guide should enable developers to deploy the default configuration or any of the other compose yaml files for different configurations. It also highlights the configurable parameters that can be set before deployment.
\ No newline at end of file
diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md b/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
new file mode 100644
index 0000000000..0edcf44141
--- /dev/null
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
@@ -0,0 +1,621 @@
+# Build and Deploy ChatQnA Application on AMD GPU (ROCm)
+
+## Build Docker Images
+
+### 1. Build Docker Image
+
+- #### Create application install directory and go to it:
+
+  ```bash
+  mkdir ~/chatqna-install && cd chatqna-install
+  ```
+
+- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIExamples.git
+  ```
+
+  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
+  ```
+
+  We remind you that when using a specific version of the code, you need to use the README from this version:
+
+- #### Go to build directory:
+
+  ```bash
+  cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_image_build
+  ```
+
+- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
+  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
+
+  ```bash
+  echo Y | rm -R GenAIComps
+  ```
+
+- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIComps.git
+  ```
+
+  If you use a specific tag of the GenAIExamples repository,
+  then you should also use the corresponding tag for GenAIComps. (v1.3 replace with its own value):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout v1.3
+  ```
+
+  We remind you that when using a specific version of the code, you need to use the README from this version.
+
+- #### Setting the list of images for the build (from the build file.yaml)
+
+  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
+
+  #### vLLM-based application
+
+  ```bash
+  service_list="dataprep retriever vllm-rocm chatqna chatqna-ui nginx"
+  ```
+
+  #### vLLM-based application with FaqGen
+
+  ```bash
+  service_list="dataprep retriever vllm-rocm llm-faqgen chatqna chatqna-ui nginx"
+  ```
+
+  #### TGI-based application
+
+  ```bash
+  service_list="dataprep retriever chatqna chatqna-ui nginx"
+  ```
+
+  #### TGI-based application with FaqGen
+
+  ```bash
+  service_list="dataprep retriever llm-faqgen chatqna chatqna-ui nginx"
+  ```
+
+- #### Pull Docker Images
+
+  ```bash
+  docker pull redis/redis-stack:7.2.0-v9
+  docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  ```
+
+- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
+
+  ```bash
+  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  ```
+
+- #### Build Docker Images
+
+  ```bash
+  docker compose -f build.yaml build ${service_list} --no-cache
+  ```
+
+  After the build, we check the list of images with the command:
+
+  ```bash
+  docker image ls
+  ```
+
+  The list of images should include:
+
+  ##### vLLM-based application:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - opea/vllm-rocm:latest
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+  ##### vLLM-based application with FaqGen:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - opea/vllm-rocm:latest
+  - opea/llm-faqgen:latest
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+  ##### TGI-based application:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+  ##### TGI-based application with FaqGen:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  - opea/llm-faqgen:latest
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+---
+
+## Deploy the ChatQnA Application
+
+### Docker Compose Configuration for AMD GPUs
+
+To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
+
+- compose_vllm.yaml - for vLLM-based application
+- compose_faqgen_vllm.yaml - for vLLM-based application with FaqGen
+- compose.yaml - for TGI-based
+- compose_faqgen.yaml - for TGI-based application with FaqGen
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri:/dev/dri
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri/card0:/dev/dri/card0
+  - /dev/dri/render128:/dev/dri/render128
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+**How to Identify GPU Device IDs:**
+Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
+
+### Set deploy environment variables
+
+#### Setting variables in the operating system environment:
+
+##### Set variable HUGGINGFACEHUB_API_TOKEN:
+
+```bash
+### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
+export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
+```
+
+#### Set variables value in set_env\*\*\*\*.sh file:
+
+Go to Docker Compose directory:
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+```
+
+The example uses the Nano text editor. You can use any convenient text editor:
+
+#### If you use vLLM based application
+
+```bash
+nano set_env_vllm.sh
+```
+
+#### If you use vLLM based application with FaqGen
+
+```bash
+nano set_env_vllm_faqgen.sh
+```
+
+#### If you use TGI based application
+
+```bash
+nano set_env.sh
+```
+
+#### If you use TGI based application with FaqGen
+
+```bash
+nano set_env_faqgen.sh
+```
+
+If you are in a proxy environment, also set the proxy-related environment variables:
+
+```bash
+export http_proxy="Your_HTTP_Proxy"
+export https_proxy="Your_HTTPs_Proxy"
+```
+
+Set the values of the variables:
+
+- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
+
+  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
+
+  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
+
+  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
+
+  We set these values in the file set_env\*\*\*\*.sh
+
+- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
+  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
+
+#### Set variables with script set_env\*\*\*\*.sh
+
+#### If you use vLLM based application
+
+```bash
+. set_env_vllm.sh
+```
+
+#### If you use vLLM based application with FaqGen
+
+```bash
+. set_env_faqgen_vllm.sh
+```
+
+#### If you use TGI based application
+
+```bash
+. set_env.sh
+```
+
+#### If you use TGI based application with FaqGen
+
+```bash
+. set_env_faqgen.sh
+```
+
+### Start the services:
+
+#### If you use vLLM based application
+
+```bash
+docker compose -f compose_vllm.yaml up -d
+```
+
+#### If you use vLLM based application with FaqGen
+
+```bash
+docker compose -f compose_faqgen_vllm.yaml up -d
+```
+
+#### If you use TGI based application
+
+```bash
+docker compose -f compose.yaml up -d
+```
+
+#### If you use TGI based application with FaqGen
+
+```bash
+docker compose -f compose_faqgen.yaml up -d
+```
+
+All containers should be running and should not restart:
+
+##### If you use vLLM based application:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-vllm-service
+- chatqna-backend-server
+- chatqna-ui-server
+- chatqna-nginx-server
+
+##### If you use vLLM based application with FaqGen:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-vllm-service
+- chatqna-llm-faqgen
+- chatqna-backend-server
+- chatqna-ui-server
+- chatqna-nginx-server
+
+##### If you use TGI based application:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-tgi-service
+- chatqna-backend-server
+- chatqna-ui-server
+- chaqna-nginx-server
+
+##### If you use TGI based application with FaqGen:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-tgi-service
+- chatqna-llm-faqgen
+- chatqna-backend-server
+- chatqna-ui-server
+- chaqna-nginx-server
+
+---
+
+## Validate the Services
+
+### 1. Validate TEI Embedding Service
+
+```bash
+curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
+    -X POST \
+    -d '{"inputs":"What is Deep Learning?"}' \
+    -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to text:
+
+```textmate
+[[0.00037115702,-0.06356819,0.0024758505,..................,0.022725677,0.016026087,-0.02125421,-0.02984927,-0.0049473033]]
+```
+
+If the service response has a meaningful response in the value,
+then we consider the TEI Embedding Service to be successfully launched
+
+### 2. Validate Retriever Microservice
+
+```bash
+export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
+  -X POST \
+  -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{ "id": "e191846168aed1f80b2ea12df80844d2", "retrieved_docs": [], "initial_query": "test", "top_n": 1, "metadata": [] }
+```
+
+If the response corresponds to the form of the provided JSON, then we consider the
+Retriever Microservice verification successful.
+
+### 3. Validate TEI Reranking Service
+
+```bash
+curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
+    -X POST \
+    -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
+    -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+[
+  { "index": 1, "score": 0.94238955 },
+  { "index": 0, "score": 0.120219156 }
+]
+```
+
+If the response corresponds to the form of the provided JSON, then we consider the TEI Reranking Service
+verification successful.
+
+### 4. Validate the vLLM/TGI Service
+
+#### If you use vLLM:
+
+```bash
+DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
+'"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
+
+curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "id": "chatcmpl-91003647d1c7469a89e399958f390f67",
+  "object": "chat.completion",
+  "created": 1742877228,
+  "model": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Deep Learning ( DL) is a subfield of Machine Learning (ML) that focuses on the design of algorithms and architectures inspired by the structure and function of the human brain. These algorithms are designed to analyze and interpret data that is presented in the form of patterns or signals, and they often mimic the way the human brain",
+        "tool_calls": []
+      },
+      "logprobs": null,
+      "finish_reason": "length",
+      "stop_reason": null
+    }
+  ],
+  "usage": { "prompt_tokens": 16, "total_tokens": 80, "completion_tokens": 64, "prompt_tokens_details": null },
+  "prompt_logprobs": null
+}
+```
+
+If the service response has a meaningful response in the value of the "choices.message.content" key,
+then we consider the vLLM service to be successfully launched
+
+#### If you use TGI:
+
+```bash
+DATA='{"inputs":"What is a Deep Learning?",'\
+'"parameters":{"max_new_tokens":64,"do_sample": true}}'
+
+curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "generated_text": " What is its application in Computer Vision?\nWhat is a Deep Learning?\nDeep learning is a subfield of machine learning that involves the use of artificial neural networks to model high-level abstractions in data. It involves the use of deep neural networks, which are composed of multiple layers, to learn complex patterns in data. The"
+}
+```
+
+If the service response has a meaningful response in the value of the "generated_text" key,
+then we consider the TGI service to be successfully launched
+
+### 5. Validate the LLM Service (if your used application with FaqGen)
+
+```bash
+DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
+'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
+'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
+
+curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "id": "58f0632f5f03af31471b895b0d0d397b",
+  "text": " Q: What is Text Embeddings Inference (TEI)?\n         A: TEI is a toolkit for deploying and serving open source text embeddings and sequence classification models.\n\n         Q: What models does TEI support?\n         A: TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.\n\n         Q: What is the purpose of TEI?\n         A: The purpose of TEI is to enable high-performance extraction for text embeddings and sequence classification models.\n\n         Q: What are the benefits of using TEI?\n         A: The benefits of using TEI include high",
+  "prompt": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
+}
+```
+
+If the service response has a meaningful response in the value of the "text" key,
+then we consider the LLM service to be successfully launched
+
+### 6. Validate the MegaService
+
+```bash
+curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
+  -H "Content-Type: application/json" \
+  -d '{"messages": "What is the revenue of Nike in 2023?"}'
+```
+
+Checking the response from the service. The response should be similar to text:
+
+```textmate
+data: b' What'
+data: b' is'
+data: b' the'
+data: b' revenue'
+data: b' of'
+data: b' Nike'
+data: b' in'
+data: b' '
+data: b'202'
+data: b'3'
+data: b'?\n'
+data: b'        '
+data: b' Answer'
+data: b':'
+data: b' According'
+data: b' to'
+data: b' the'
+data: b' search'
+data: b' results'
+data: b','
+data: b' the'
+data: b' revenue'
+data: b' of'
+data: b''
+
+data: [DONE]
+
+```
+
+If the output lines in the "data" keys contain words (tokens) containing meaning, then the service
+is considered launched successfully.
+
+### 7. Validate the Frontend (UI)
+
+To access the UI, use the URL - http://${EXTERNAL_HOST_IP}:${CHATQNA_NGINX_PORT}
+A page should open when you click through to this address:
+
+![UI start page](../../../../assets/img/ui-starting-page.png)
+
+If a page of this type has opened, then we believe that the service is running and responding,
+and we can proceed to functional UI testing.
+
+Let's enter the task for the service in the "Enter prompt here" field.
+For example, "What is a Deep Learning?" and press Enter.
+After that, a page with the result of the task should open:
+
+#### If used application without FaqGen
+
+![UI result page](../../../../assets/img/ui-result-page.png)
+
+#### If used application with FaqGen
+
+![UI result page](../../../../assets/img/ui-result-page-faqgen.png)
+
+If the result shown on the page is correct, then we consider the verification of the UI service to be successful.
+
+### 5. Stop application
+
+#### If you use vLLM
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose_vllm.yaml down
+```
+
+#### If you use vLLM with FaqGen
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose_faqgen_vllm.yaml down
+```
+
+#### If you use TGI
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose.yaml down
+```
+
+#### If you use TGI with FaqGen
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose_faqgen.yaml down
+```

From 6903d8abb78f0f3b4a035b83528504a83818b37c Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Tue, 22 Apr 2025 00:50:49 +0700
Subject: [PATCH 10/17] ChatQnA - refactoring README.md

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md | 174 ++++++++++++------
 1 file changed, 120 insertions(+), 54 deletions(-)

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index 9c4887aaed..8f84441e4b 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -97,110 +97,176 @@ After running docker compose, check if all the containers launched via docker co
 docker ps -a
 ```
 
-For the default deployment, the following 5 containers should have started:
+For the default deployment with TGI, the following 9 containers should have started:
 
 ```
-1c67e44c39d2   opea/chatqna-ui:latest   "docker-entrypoint.s…"   About a minute ago   Up About a minute             0.0.0.0:5173->5173/tcp, :::5173->5173/tcp   audioqna-xeon-ui-server
-833a42677247   opea/chatqna:latest      "python audioqna.py"     About a minute ago   Up About a minute             0.0.0.0:3008->8888/tcp, :::3008->8888/tcp   audioqna-xeon-backend-server
-5dc4eb9bf499   opea/speecht5:latest      "python speecht5_ser…"   About a minute ago   Up About a minute             0.0.0.0:7055->7055/tcp, :::7055->7055/tcp   speecht5-service
-814e6efb1166   opea/vllm:latest          "python3 -m vllm.ent…"   About a minute ago   Up About a minute (healthy)   0.0.0.0:3006->80/tcp, :::3006->80/tcp       vllm-service
-46f7a00f4612   opea/whisper:latest       "python whisper_serv…"   About a minute ago   Up About a minute             0.0.0.0:7066->7066/tcp, :::7066->7066/tcp   whisper-service
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-tgi-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
+```
+
+if used TGI with FaqGen:
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+e0ef1ea67640   opea/llm-faqgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              chatqna-llm-faqgen
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-tgi-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
+```
+
+if used vLLM:
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   opea/vllm-rocm:latest                                   "python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-vllm-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
+```
+
+if used vLLM with FaqGen:
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+e0ef1ea67640   opea/llm-faqgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              chatqna-llm-faqgen
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   opea/vllm-rocm:latest                                   "python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-vllm-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
 ```
 
 If any issues are encountered during deployment, refer to the [Troubleshooting](../../../../README_miscellaneous.md#troubleshooting) section.
 
 ### Validate the Pipeline
 
-Once the AudioQnA services are running, test the pipeline using the following command:
+Once the ChatQnA services are running, test the pipeline using the following command:
 
 ```bash
-# Test the AudioQnA megaservice by recording a .wav file, encoding the file into the base64 format, and then sending the base64 string to the megaservice endpoint.
-# The megaservice will return a spoken response as a base64 string. To listen to the response, decode the base64 string and save it as a .wav file.
-wget https://github.com/intel/intel-extension-for-transformers/raw/refs/heads/main/intel_extension_for_transformers/neural_chat/assets/audio/sample_2.wav
-base64_audio=$(base64 -w 0 sample_2.wav)
-
-# if you are using speecht5 as the tts service, voice can be "default" or "male"
-# if you are using gpt-sovits for the tts service, you can set the reference audio following https://github.com/opea-project/GenAIComps/blob/main/comps/third_parties/gpt-sovits/src/README.md
-
-curl http://${host_ip}:3008/v1/audioqna \
-  -X POST \
+curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
   -H "Content-Type: application/json" \
-  -d "{\"audio\": \"${base64_audio}\", \"max_tokens\": 64, \"voice\": \"default\"}" \
-  | sed 's/^"//;s/"$//' | base64 -d > output.wav
+  -d '{"messages": "What is the revenue of Nike in 2023?"}'
 ```
 
-**Note** : Access the AudioQnA UI by web browser through this URL: `http://${host_ip}:5173`. Please confirm the `5173` port is opened in the firewall. To validate each microservice used in the pipeline refer to the [Validate Microservices](#validate-microservices) section.
+**Note** : Access the ChatQnA UI by web browser through this URL: `http://${HOST_IP_EXTERNAL}:${CHATQNA_NGINX_PORT}`
 
 ### Cleanup the Deployment
 
 To stop the containers associated with the deployment, execute the following command:
 
 ```bash
+# if used TGI
 docker compose -f compose.yaml down
+# if used TGI with FaqGen
+# docker compose -f compose_faqgen.yaml down
+# if used vLLM
+# docker compose -f compose_vllm.yaml down
+# if used vLLM with FaqGen
+# docker compose -f compose_faqgen_vllm.yaml down
+
+
 ```
 
-## AudioQnA Docker Compose Files
+## ChatQnA Docker Compose Files
 
 In the context of deploying an AudioQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
 
-| File                                               | Description                                                                               |
-| -------------------------------------------------- | ----------------------------------------------------------------------------------------- |
-| [compose.yaml](./compose.yaml)                     | Default compose file using vllm as serving framework and redis as vector database         |
-| [compose_tgi.yaml](./compose_tgi.yaml)             | The LLM serving framework is TGI. All other configurations remain the same as the default |
-| [compose_multilang.yaml](./compose_multilang.yaml) | The TTS component is GPT-SoVITS. All other configurations remain the same as the default  |
+| File                                                   | Description                                                                                                              |
+|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
+| [compose.yaml](./compose.yaml)                         | The LLM serving framework is TGI. Default compose file using TGI as serving framework and redis as vector database       |
+| [compose_faqgen.yaml](./compose_faqgen.yaml)           | The LLM serving framework is TGI with FaqGen. All other configurations remain the same as the default                    |
+| [compose_vllm.yaml](./compose_vllm.yaml)               | The LLM serving framework is vLLM. Compose file using vllm as serving framework and redis as vector database             |
+| [compose_faqgen_vllm.yaml](./compose_faqgen_vllm.yaml) | The LLM serving framework is vLLM with FaqGen. Compose file using vllm as serving framework and redis as vector database |
 
 ## Validate MicroServices
 
-1. Whisper Service
+1. TEI Embedding Service
 
    ```bash
-   wget https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample.wav
-   curl http://${host_ip}:${WHISPER_SERVER_PORT}/v1/audio/transcriptions \
-     -H "Content-Type: multipart/form-data" \
-     -F file="@./sample.wav" \
-     -F model="openai/whisper-small"
+   curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
+   -X POST \
+   -d '{"inputs":"What is Deep Learning?"}' \
+   -H 'Content-Type: application/json'
    ```
 
-2. LLM backend Service
-
-   In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready and the container (`vllm-service` or `tgi-service`) status shown via `docker ps` will be `healthy`. Before that, the status will be `health: starting`.
-
-   Or try the command below to check whether the LLM serving is ready.
+2. Retriever Microservice
 
    ```bash
-   # vLLM service
-   docker logs vllm-service 2>&1 | grep complete
-   # If the service is ready, you will get the response like below.
-   INFO:     Application startup complete.
+   export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+   curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
+     -X POST \
+     -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
+     -H 'Content-Type: application/json'
    ```
+3. TEI Reranking Service
 
    ```bash
-   # TGI service
-   docker logs tgi-service | grep Connected
-   # If the service is ready, you will get the response like below.
-   2024-09-03T02:47:53.402023Z  INFO text_generation_router::server: router/src/server.rs:2311: Connected
+   curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
+   -X POST \
+   -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
+   -H 'Content-Type: application/json'
    ```
 
-   Then try the `cURL` command below to validate services.
+4. vLLM/TGI Service
 
+   If you use vLLM:
+   
    ```bash
-   # either vLLM or TGI service
-   curl http://${host_ip}:${LLM_SERVER_PORT}/v1/chat/completions \
+   DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
+   '"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
+   
+   curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
      -X POST \
-     -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \
+     -d "$DATA" \
      -H 'Content-Type: application/json'
    ```
 
-3. TTS Service
+   If you use TGI:
 
    ```bash
-   # speecht5 service
-   curl http://${host_ip}:${SPEECHT5_SERVER_PORT}/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3
+   DATA='{"inputs":"What is a Deep Learning?",'\
+   '"parameters":{"max_new_tokens":64,"do_sample": true}}'
+   
+   curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
+   -X POST \
+   -d "$DATA" \
+   -H 'Content-Type: application/json'
+   ```
+
+5. LLM Service (if your used application with FaqGen)
 
-   # gpt-sovits service (optional)
-   curl http://${host_ip}:${GPT_SOVITS_SERVER_PORT}/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3
+   ```bash
+   DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
+   'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
+   'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
+   
+   curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
+     -X POST \
+     -d "$DATA" \
+     -H 'Content-Type: application/json'
    ```
 
+
 ## Conclusion
 
 This guide should enable developers to deploy the default configuration or any of the other compose yaml files for different configurations. It also highlights the configurable parameters that can be set before deployment.
\ No newline at end of file

From a16515998560fad96ce09e387656dd8167623fe2 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Tue, 22 Apr 2025 01:04:35 +0700
Subject: [PATCH 11/17] ChatQnA - refactoring README.md

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md | 30 +++++++++++--------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index 8f84441e4b..da45f1ab54 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -41,10 +41,10 @@ git checkout v1.2
 
 To set up environment variables for deploying ChatQnA services, set up some parameters specific to the deployment environment and source the `set_env_*.sh` script in this directory:
 
-if used vLLM - set_env_vllm.sh
-if used vLLM with FaqGen - set_env_faqgen_vllm.sh
-if used TGI - set_env.sh
-if used TGI with FaqGen - set_env_faqgen.sh
+- if used vLLM - set_env_vllm.sh
+- if used vLLM with FaqGen - set_env_faqgen_vllm.sh
+- if used TGI - set_env.sh
+- if used TGI with FaqGen - set_env_faqgen.sh
 
 ```bash
 export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
@@ -79,15 +79,19 @@ docker compose -f compose.yaml up -d
 
 Please refer to the table below to build different microservices from source:
 
-| Microservice | Deployment Guide                                                                                                                  |
-| ------------ | --------------------------------------------------------------------------------------------------------------------------------- |
-| vLLM         | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker)                    |
-| LLM          | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                                                |
-| WHISPER      | [Whisper build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/asr/src#211-whisper-server-image)                |
-| SPEECHT5     | [SpeechT5 build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/tts/src#211-speecht5-server-image)              |
-| GPT-SOVITS   | [GPT-SOVITS build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/gpt-sovits/src#build-the-image) |
-| MegaService  | [MegaService build guide](../../../../README_miscellaneous.md#build-megaservice-docker-image)                                     |
-| UI           | [Basic UI build guide](../../../../README_miscellaneous.md#build-ui-docker-image)                                                 |
+| Microservice    | Deployment Guide                                                                                                    |
+|-----------------|---------------------------------------------------------------------------------------------------------------------|
+| vLLM            | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker)      |
+| TGI             | [TGI project](https://github.com/huggingface/text-generation-inference.git)                                         |
+| LLM             | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                                  |
+| Redis Vector DB | [Redis](https://github.com/redis/redis.git)                                                                         |
+| Dataprep        | [Dataprep build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/dataprep/src/README_redis.md)     |
+| TEI Embedding   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                           |
+| Retriever       | [Retriever build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/retrievers/src/README_redis.md ) |
+| TEI Reranking   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                           |
+| MegaService     | [MegaService guide](../../../../README.md)                                                                          |
+| UI              | [UI guide](../../../../ui/react/README.md)                                                                          |
+| Nginx           | [Nginx guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/nginx)                       |
 
 ### Check the Deployment Status
 

From a612b4a639fc3bd71cbb40d235e1aa40b8efba91 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Tue, 22 Apr 2025 01:07:06 +0700
Subject: [PATCH 12/17] ChatQnA - refactoring README.md

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md |   2 +-
 .../docker_compose/amd/gpu/rocm/README_old.md | 621 ------------------
 2 files changed, 1 insertion(+), 622 deletions(-)
 delete mode 100644 ChatQnA/docker_compose/amd/gpu/rocm/README_old.md

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index da45f1ab54..861585c14d 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -193,7 +193,7 @@ docker compose -f compose.yaml down
 
 ## ChatQnA Docker Compose Files
 
-In the context of deploying an AudioQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
+In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
 
 | File                                                   | Description                                                                                                              |
 |--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md b/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
deleted file mode 100644
index 0edcf44141..0000000000
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
+++ /dev/null
@@ -1,621 +0,0 @@
-# Build and Deploy ChatQnA Application on AMD GPU (ROCm)
-
-## Build Docker Images
-
-### 1. Build Docker Image
-
-- #### Create application install directory and go to it:
-
-  ```bash
-  mkdir ~/chatqna-install && cd chatqna-install
-  ```
-
-- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git
-  ```
-
-  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
-  ```
-
-  We remind you that when using a specific version of the code, you need to use the README from this version:
-
-- #### Go to build directory:
-
-  ```bash
-  cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_image_build
-  ```
-
-- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
-  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
-
-  ```bash
-  echo Y | rm -R GenAIComps
-  ```
-
-- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git
-  ```
-
-  If you use a specific tag of the GenAIExamples repository,
-  then you should also use the corresponding tag for GenAIComps. (v1.3 replace with its own value):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout v1.3
-  ```
-
-  We remind you that when using a specific version of the code, you need to use the README from this version.
-
-- #### Setting the list of images for the build (from the build file.yaml)
-
-  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
-
-  #### vLLM-based application
-
-  ```bash
-  service_list="dataprep retriever vllm-rocm chatqna chatqna-ui nginx"
-  ```
-
-  #### vLLM-based application with FaqGen
-
-  ```bash
-  service_list="dataprep retriever vllm-rocm llm-faqgen chatqna chatqna-ui nginx"
-  ```
-
-  #### TGI-based application
-
-  ```bash
-  service_list="dataprep retriever chatqna chatqna-ui nginx"
-  ```
-
-  #### TGI-based application with FaqGen
-
-  ```bash
-  service_list="dataprep retriever llm-faqgen chatqna chatqna-ui nginx"
-  ```
-
-- #### Pull Docker Images
-
-  ```bash
-  docker pull redis/redis-stack:7.2.0-v9
-  docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  ```
-
-- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
-
-  ```bash
-  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  ```
-
-- #### Build Docker Images
-
-  ```bash
-  docker compose -f build.yaml build ${service_list} --no-cache
-  ```
-
-  After the build, we check the list of images with the command:
-
-  ```bash
-  docker image ls
-  ```
-
-  The list of images should include:
-
-  ##### vLLM-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### vLLM-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
----
-
-## Deploy the ChatQnA Application
-
-### Docker Compose Configuration for AMD GPUs
-
-To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
-
-- compose_vllm.yaml - for vLLM-based application
-- compose_faqgen_vllm.yaml - for vLLM-based application with FaqGen
-- compose.yaml - for TGI-based
-- compose_faqgen.yaml - for TGI-based application with FaqGen
-
-```yaml
-shm_size: 1g
-devices:
-  - /dev/kfd:/dev/kfd
-  - /dev/dri:/dev/dri
-cap_add:
-  - SYS_PTRACE
-group_add:
-  - video
-security_opt:
-  - seccomp:unconfined
-```
-
-This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
-
-```yaml
-shm_size: 1g
-devices:
-  - /dev/kfd:/dev/kfd
-  - /dev/dri/card0:/dev/dri/card0
-  - /dev/dri/render128:/dev/dri/render128
-cap_add:
-  - SYS_PTRACE
-group_add:
-  - video
-security_opt:
-  - seccomp:unconfined
-```
-
-**How to Identify GPU Device IDs:**
-Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
-
-### Set deploy environment variables
-
-#### Setting variables in the operating system environment:
-
-##### Set variable HUGGINGFACEHUB_API_TOKEN:
-
-```bash
-### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
-export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
-```
-
-#### Set variables value in set_env\*\*\*\*.sh file:
-
-Go to Docker Compose directory:
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-```
-
-The example uses the Nano text editor. You can use any convenient text editor:
-
-#### If you use vLLM based application
-
-```bash
-nano set_env_vllm.sh
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-nano set_env_vllm_faqgen.sh
-```
-
-#### If you use TGI based application
-
-```bash
-nano set_env.sh
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-nano set_env_faqgen.sh
-```
-
-If you are in a proxy environment, also set the proxy-related environment variables:
-
-```bash
-export http_proxy="Your_HTTP_Proxy"
-export https_proxy="Your_HTTPs_Proxy"
-```
-
-Set the values of the variables:
-
-- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
-
-  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
-
-  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
-
-  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
-
-  We set these values in the file set_env\*\*\*\*.sh
-
-- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
-  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
-
-#### Set variables with script set_env\*\*\*\*.sh
-
-#### If you use vLLM based application
-
-```bash
-. set_env_vllm.sh
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-. set_env_faqgen_vllm.sh
-```
-
-#### If you use TGI based application
-
-```bash
-. set_env.sh
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-. set_env_faqgen.sh
-```
-
-### Start the services:
-
-#### If you use vLLM based application
-
-```bash
-docker compose -f compose_vllm.yaml up -d
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-docker compose -f compose_faqgen_vllm.yaml up -d
-```
-
-#### If you use TGI based application
-
-```bash
-docker compose -f compose.yaml up -d
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-docker compose -f compose_faqgen.yaml up -d
-```
-
-All containers should be running and should not restart:
-
-##### If you use vLLM based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use vLLM based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use TGI based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
-##### If you use TGI based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
----
-
-## Validate the Services
-
-### 1. Validate TEI Embedding Service
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
-    -X POST \
-    -d '{"inputs":"What is Deep Learning?"}' \
-    -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to text:
-
-```textmate
-[[0.00037115702,-0.06356819,0.0024758505,..................,0.022725677,0.016026087,-0.02125421,-0.02984927,-0.0049473033]]
-```
-
-If the service response has a meaningful response in the value,
-then we consider the TEI Embedding Service to be successfully launched
-
-### 2. Validate Retriever Microservice
-
-```bash
-export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
-curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
-  -X POST \
-  -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{ "id": "e191846168aed1f80b2ea12df80844d2", "retrieved_docs": [], "initial_query": "test", "top_n": 1, "metadata": [] }
-```
-
-If the response corresponds to the form of the provided JSON, then we consider the
-Retriever Microservice verification successful.
-
-### 3. Validate TEI Reranking Service
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
-    -X POST \
-    -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
-    -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-[
-  { "index": 1, "score": 0.94238955 },
-  { "index": 0, "score": 0.120219156 }
-]
-```
-
-If the response corresponds to the form of the provided JSON, then we consider the TEI Reranking Service
-verification successful.
-
-### 4. Validate the vLLM/TGI Service
-
-#### If you use vLLM:
-
-```bash
-DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
-'"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
-
-curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "id": "chatcmpl-91003647d1c7469a89e399958f390f67",
-  "object": "chat.completion",
-  "created": 1742877228,
-  "model": "meta-llama/Meta-Llama-3-8B-Instruct",
-  "choices": [
-    {
-      "index": 0,
-      "message": {
-        "role": "assistant",
-        "content": "Deep Learning ( DL) is a subfield of Machine Learning (ML) that focuses on the design of algorithms and architectures inspired by the structure and function of the human brain. These algorithms are designed to analyze and interpret data that is presented in the form of patterns or signals, and they often mimic the way the human brain",
-        "tool_calls": []
-      },
-      "logprobs": null,
-      "finish_reason": "length",
-      "stop_reason": null
-    }
-  ],
-  "usage": { "prompt_tokens": 16, "total_tokens": 80, "completion_tokens": 64, "prompt_tokens_details": null },
-  "prompt_logprobs": null
-}
-```
-
-If the service response has a meaningful response in the value of the "choices.message.content" key,
-then we consider the vLLM service to be successfully launched
-
-#### If you use TGI:
-
-```bash
-DATA='{"inputs":"What is a Deep Learning?",'\
-'"parameters":{"max_new_tokens":64,"do_sample": true}}'
-
-curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "generated_text": " What is its application in Computer Vision?\nWhat is a Deep Learning?\nDeep learning is a subfield of machine learning that involves the use of artificial neural networks to model high-level abstractions in data. It involves the use of deep neural networks, which are composed of multiple layers, to learn complex patterns in data. The"
-}
-```
-
-If the service response has a meaningful response in the value of the "generated_text" key,
-then we consider the TGI service to be successfully launched
-
-### 5. Validate the LLM Service (if your used application with FaqGen)
-
-```bash
-DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
-'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
-'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
-
-curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "id": "58f0632f5f03af31471b895b0d0d397b",
-  "text": " Q: What is Text Embeddings Inference (TEI)?\n         A: TEI is a toolkit for deploying and serving open source text embeddings and sequence classification models.\n\n         Q: What models does TEI support?\n         A: TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.\n\n         Q: What is the purpose of TEI?\n         A: The purpose of TEI is to enable high-performance extraction for text embeddings and sequence classification models.\n\n         Q: What are the benefits of using TEI?\n         A: The benefits of using TEI include high",
-  "prompt": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-}
-```
-
-If the service response has a meaningful response in the value of the "text" key,
-then we consider the LLM service to be successfully launched
-
-### 6. Validate the MegaService
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
-  -H "Content-Type: application/json" \
-  -d '{"messages": "What is the revenue of Nike in 2023?"}'
-```
-
-Checking the response from the service. The response should be similar to text:
-
-```textmate
-data: b' What'
-data: b' is'
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b' Nike'
-data: b' in'
-data: b' '
-data: b'202'
-data: b'3'
-data: b'?\n'
-data: b'        '
-data: b' Answer'
-data: b':'
-data: b' According'
-data: b' to'
-data: b' the'
-data: b' search'
-data: b' results'
-data: b','
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b''
-
-data: [DONE]
-
-```
-
-If the output lines in the "data" keys contain words (tokens) containing meaning, then the service
-is considered launched successfully.
-
-### 7. Validate the Frontend (UI)
-
-To access the UI, use the URL - http://${EXTERNAL_HOST_IP}:${CHATQNA_NGINX_PORT}
-A page should open when you click through to this address:
-
-![UI start page](../../../../assets/img/ui-starting-page.png)
-
-If a page of this type has opened, then we believe that the service is running and responding,
-and we can proceed to functional UI testing.
-
-Let's enter the task for the service in the "Enter prompt here" field.
-For example, "What is a Deep Learning?" and press Enter.
-After that, a page with the result of the task should open:
-
-#### If used application without FaqGen
-
-![UI result page](../../../../assets/img/ui-result-page.png)
-
-#### If used application with FaqGen
-
-![UI result page](../../../../assets/img/ui-result-page-faqgen.png)
-
-If the result shown on the page is correct, then we consider the verification of the UI service to be successful.
-
-### 5. Stop application
-
-#### If you use vLLM
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_vllm.yaml down
-```
-
-#### If you use vLLM with FaqGen
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen_vllm.yaml down
-```
-
-#### If you use TGI
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose.yaml down
-```
-
-#### If you use TGI with FaqGen
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen.yaml down
-```

From e01589ca0b37a911f28b76c1c3910440f612b9bc Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Tue, 22 Apr 2025 01:11:05 +0700
Subject: [PATCH 13/17] ChatQnA - refactoring README.md

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md |  53 ++
 .../docker_compose/amd/gpu/rocm/README_old.md | 621 ++++++++++++++++++
 2 files changed, 674 insertions(+)
 create mode 100644 ChatQnA/docker_compose/amd/gpu/rocm/README_old.md

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index 861585c14d..d09579fd7c 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -46,6 +46,22 @@ To set up environment variables for deploying ChatQnA services, set up some para
 - if used TGI - set_env.sh
 - if used TGI with FaqGen - set_env_faqgen.sh
 
+Set the values of the variables:
+
+- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
+
+  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
+
+  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
+
+  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
+
+  We set these values in the file set_env\*\*\*\*.sh
+
+- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
+  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
+
+Setting variables in the operating system environment:
 ```bash
 export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
 source ./set_env_*.sh # replace the script name with the appropriate one
@@ -67,10 +83,47 @@ docker compose -f compose.yaml up -d
 # docker compose -f compose_vllm.yaml up -d
 # if used vLLM with FaqGen
 # docker compose -f compose_faqgen_vllm.yaml up -d
+```
 
+To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
+
+- compose_vllm.yaml - for vLLM-based application
+- compose_faqgen_vllm.yaml - for vLLM-based application with FaqGen
+- compose.yaml - for TGI-based
+- compose_faqgen.yaml - for TGI-based application with FaqGen
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri:/dev/dri
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
 
+This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri/card0:/dev/dri/card0
+  - /dev/dri/render128:/dev/dri/render128
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
 ```
 
+**How to Identify GPU Device IDs:**
+Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
+
 > **Note**: developers should build docker image from source when:
 >
 > - Developing off the git main branch (as the container's ports in the repo may be different > from the published docker image).
diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md b/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
new file mode 100644
index 0000000000..0edcf44141
--- /dev/null
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
@@ -0,0 +1,621 @@
+# Build and Deploy ChatQnA Application on AMD GPU (ROCm)
+
+## Build Docker Images
+
+### 1. Build Docker Image
+
+- #### Create application install directory and go to it:
+
+  ```bash
+  mkdir ~/chatqna-install && cd chatqna-install
+  ```
+
+- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIExamples.git
+  ```
+
+  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
+  ```
+
+  We remind you that when using a specific version of the code, you need to use the README from this version:
+
+- #### Go to build directory:
+
+  ```bash
+  cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_image_build
+  ```
+
+- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
+  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
+
+  ```bash
+  echo Y | rm -R GenAIComps
+  ```
+
+- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIComps.git
+  ```
+
+  If you use a specific tag of the GenAIExamples repository,
+  then you should also use the corresponding tag for GenAIComps. (v1.3 replace with its own value):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout v1.3
+  ```
+
+  We remind you that when using a specific version of the code, you need to use the README from this version.
+
+- #### Setting the list of images for the build (from the build file.yaml)
+
+  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
+
+  #### vLLM-based application
+
+  ```bash
+  service_list="dataprep retriever vllm-rocm chatqna chatqna-ui nginx"
+  ```
+
+  #### vLLM-based application with FaqGen
+
+  ```bash
+  service_list="dataprep retriever vllm-rocm llm-faqgen chatqna chatqna-ui nginx"
+  ```
+
+  #### TGI-based application
+
+  ```bash
+  service_list="dataprep retriever chatqna chatqna-ui nginx"
+  ```
+
+  #### TGI-based application with FaqGen
+
+  ```bash
+  service_list="dataprep retriever llm-faqgen chatqna chatqna-ui nginx"
+  ```
+
+- #### Pull Docker Images
+
+  ```bash
+  docker pull redis/redis-stack:7.2.0-v9
+  docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  ```
+
+- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
+
+  ```bash
+  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  ```
+
+- #### Build Docker Images
+
+  ```bash
+  docker compose -f build.yaml build ${service_list} --no-cache
+  ```
+
+  After the build, we check the list of images with the command:
+
+  ```bash
+  docker image ls
+  ```
+
+  The list of images should include:
+
+  ##### vLLM-based application:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - opea/vllm-rocm:latest
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+  ##### vLLM-based application with FaqGen:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - opea/vllm-rocm:latest
+  - opea/llm-faqgen:latest
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+  ##### TGI-based application:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+  ##### TGI-based application with FaqGen:
+
+  - redis/redis-stack:7.2.0-v9
+  - opea/dataprep:latest
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/retriever:latest
+  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  - opea/llm-faqgen:latest
+  - opea/chatqna:latest
+  - opea/chatqna-ui:latest
+  - opea/nginx:latest
+
+---
+
+## Deploy the ChatQnA Application
+
+### Docker Compose Configuration for AMD GPUs
+
+To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
+
+- compose_vllm.yaml - for vLLM-based application
+- compose_faqgen_vllm.yaml - for vLLM-based application with FaqGen
+- compose.yaml - for TGI-based
+- compose_faqgen.yaml - for TGI-based application with FaqGen
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri:/dev/dri
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri/card0:/dev/dri/card0
+  - /dev/dri/render128:/dev/dri/render128
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+**How to Identify GPU Device IDs:**
+Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
+
+### Set deploy environment variables
+
+#### Setting variables in the operating system environment:
+
+##### Set variable HUGGINGFACEHUB_API_TOKEN:
+
+```bash
+### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
+export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
+```
+
+#### Set variables value in set_env\*\*\*\*.sh file:
+
+Go to Docker Compose directory:
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+```
+
+The example uses the Nano text editor. You can use any convenient text editor:
+
+#### If you use vLLM based application
+
+```bash
+nano set_env_vllm.sh
+```
+
+#### If you use vLLM based application with FaqGen
+
+```bash
+nano set_env_vllm_faqgen.sh
+```
+
+#### If you use TGI based application
+
+```bash
+nano set_env.sh
+```
+
+#### If you use TGI based application with FaqGen
+
+```bash
+nano set_env_faqgen.sh
+```
+
+If you are in a proxy environment, also set the proxy-related environment variables:
+
+```bash
+export http_proxy="Your_HTTP_Proxy"
+export https_proxy="Your_HTTPs_Proxy"
+```
+
+Set the values of the variables:
+
+- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
+
+  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
+
+  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
+
+  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
+
+  We set these values in the file set_env\*\*\*\*.sh
+
+- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
+  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
+
+#### Set variables with script set_env\*\*\*\*.sh
+
+#### If you use vLLM based application
+
+```bash
+. set_env_vllm.sh
+```
+
+#### If you use vLLM based application with FaqGen
+
+```bash
+. set_env_faqgen_vllm.sh
+```
+
+#### If you use TGI based application
+
+```bash
+. set_env.sh
+```
+
+#### If you use TGI based application with FaqGen
+
+```bash
+. set_env_faqgen.sh
+```
+
+### Start the services:
+
+#### If you use vLLM based application
+
+```bash
+docker compose -f compose_vllm.yaml up -d
+```
+
+#### If you use vLLM based application with FaqGen
+
+```bash
+docker compose -f compose_faqgen_vllm.yaml up -d
+```
+
+#### If you use TGI based application
+
+```bash
+docker compose -f compose.yaml up -d
+```
+
+#### If you use TGI based application with FaqGen
+
+```bash
+docker compose -f compose_faqgen.yaml up -d
+```
+
+All containers should be running and should not restart:
+
+##### If you use vLLM based application:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-vllm-service
+- chatqna-backend-server
+- chatqna-ui-server
+- chatqna-nginx-server
+
+##### If you use vLLM based application with FaqGen:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-vllm-service
+- chatqna-llm-faqgen
+- chatqna-backend-server
+- chatqna-ui-server
+- chatqna-nginx-server
+
+##### If you use TGI based application:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-tgi-service
+- chatqna-backend-server
+- chatqna-ui-server
+- chaqna-nginx-server
+
+##### If you use TGI based application with FaqGen:
+
+- chatqna-redis-vector-db
+- chatqna-dataprep-service
+- chatqna-tei-embedding-service
+- chatqna-retriever
+- chatqna-tei-reranking-service
+- chatqna-tgi-service
+- chatqna-llm-faqgen
+- chatqna-backend-server
+- chatqna-ui-server
+- chaqna-nginx-server
+
+---
+
+## Validate the Services
+
+### 1. Validate TEI Embedding Service
+
+```bash
+curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
+    -X POST \
+    -d '{"inputs":"What is Deep Learning?"}' \
+    -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to text:
+
+```textmate
+[[0.00037115702,-0.06356819,0.0024758505,..................,0.022725677,0.016026087,-0.02125421,-0.02984927,-0.0049473033]]
+```
+
+If the service response has a meaningful response in the value,
+then we consider the TEI Embedding Service to be successfully launched
+
+### 2. Validate Retriever Microservice
+
+```bash
+export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
+  -X POST \
+  -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{ "id": "e191846168aed1f80b2ea12df80844d2", "retrieved_docs": [], "initial_query": "test", "top_n": 1, "metadata": [] }
+```
+
+If the response corresponds to the form of the provided JSON, then we consider the
+Retriever Microservice verification successful.
+
+### 3. Validate TEI Reranking Service
+
+```bash
+curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
+    -X POST \
+    -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
+    -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+[
+  { "index": 1, "score": 0.94238955 },
+  { "index": 0, "score": 0.120219156 }
+]
+```
+
+If the response corresponds to the form of the provided JSON, then we consider the TEI Reranking Service
+verification successful.
+
+### 4. Validate the vLLM/TGI Service
+
+#### If you use vLLM:
+
+```bash
+DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
+'"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
+
+curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "id": "chatcmpl-91003647d1c7469a89e399958f390f67",
+  "object": "chat.completion",
+  "created": 1742877228,
+  "model": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Deep Learning ( DL) is a subfield of Machine Learning (ML) that focuses on the design of algorithms and architectures inspired by the structure and function of the human brain. These algorithms are designed to analyze and interpret data that is presented in the form of patterns or signals, and they often mimic the way the human brain",
+        "tool_calls": []
+      },
+      "logprobs": null,
+      "finish_reason": "length",
+      "stop_reason": null
+    }
+  ],
+  "usage": { "prompt_tokens": 16, "total_tokens": 80, "completion_tokens": 64, "prompt_tokens_details": null },
+  "prompt_logprobs": null
+}
+```
+
+If the service response has a meaningful response in the value of the "choices.message.content" key,
+then we consider the vLLM service to be successfully launched
+
+#### If you use TGI:
+
+```bash
+DATA='{"inputs":"What is a Deep Learning?",'\
+'"parameters":{"max_new_tokens":64,"do_sample": true}}'
+
+curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "generated_text": " What is its application in Computer Vision?\nWhat is a Deep Learning?\nDeep learning is a subfield of machine learning that involves the use of artificial neural networks to model high-level abstractions in data. It involves the use of deep neural networks, which are composed of multiple layers, to learn complex patterns in data. The"
+}
+```
+
+If the service response has a meaningful response in the value of the "generated_text" key,
+then we consider the TGI service to be successfully launched
+
+### 5. Validate the LLM Service (if your used application with FaqGen)
+
+```bash
+DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
+'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
+'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
+
+curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "id": "58f0632f5f03af31471b895b0d0d397b",
+  "text": " Q: What is Text Embeddings Inference (TEI)?\n         A: TEI is a toolkit for deploying and serving open source text embeddings and sequence classification models.\n\n         Q: What models does TEI support?\n         A: TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.\n\n         Q: What is the purpose of TEI?\n         A: The purpose of TEI is to enable high-performance extraction for text embeddings and sequence classification models.\n\n         Q: What are the benefits of using TEI?\n         A: The benefits of using TEI include high",
+  "prompt": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
+}
+```
+
+If the service response has a meaningful response in the value of the "text" key,
+then we consider the LLM service to be successfully launched
+
+### 6. Validate the MegaService
+
+```bash
+curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
+  -H "Content-Type: application/json" \
+  -d '{"messages": "What is the revenue of Nike in 2023?"}'
+```
+
+Checking the response from the service. The response should be similar to text:
+
+```textmate
+data: b' What'
+data: b' is'
+data: b' the'
+data: b' revenue'
+data: b' of'
+data: b' Nike'
+data: b' in'
+data: b' '
+data: b'202'
+data: b'3'
+data: b'?\n'
+data: b'        '
+data: b' Answer'
+data: b':'
+data: b' According'
+data: b' to'
+data: b' the'
+data: b' search'
+data: b' results'
+data: b','
+data: b' the'
+data: b' revenue'
+data: b' of'
+data: b''
+
+data: [DONE]
+
+```
+
+If the output lines in the "data" keys contain words (tokens) containing meaning, then the service
+is considered launched successfully.
+
+### 7. Validate the Frontend (UI)
+
+To access the UI, use the URL - http://${EXTERNAL_HOST_IP}:${CHATQNA_NGINX_PORT}
+A page should open when you click through to this address:
+
+![UI start page](../../../../assets/img/ui-starting-page.png)
+
+If a page of this type has opened, then we believe that the service is running and responding,
+and we can proceed to functional UI testing.
+
+Let's enter the task for the service in the "Enter prompt here" field.
+For example, "What is a Deep Learning?" and press Enter.
+After that, a page with the result of the task should open:
+
+#### If used application without FaqGen
+
+![UI result page](../../../../assets/img/ui-result-page.png)
+
+#### If used application with FaqGen
+
+![UI result page](../../../../assets/img/ui-result-page-faqgen.png)
+
+If the result shown on the page is correct, then we consider the verification of the UI service to be successful.
+
+### 5. Stop application
+
+#### If you use vLLM
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose_vllm.yaml down
+```
+
+#### If you use vLLM with FaqGen
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose_faqgen_vllm.yaml down
+```
+
+#### If you use TGI
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose.yaml down
+```
+
+#### If you use TGI with FaqGen
+
+```bash
+cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose_faqgen.yaml down
+```

From ff7a7cc18d53ef8e3148128aee84469c00dcecda Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 21 Apr 2025 18:14:55 +0000
Subject: [PATCH 14/17] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md | 44 ++++++++++---------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index d09579fd7c..3d273f41c1 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -62,6 +62,7 @@ Set the values of the variables:
   The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
 
 Setting variables in the operating system environment:
+
 ```bash
 export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
 source ./set_env_*.sh # replace the script name with the appropriate one
@@ -132,19 +133,19 @@ Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs
 
 Please refer to the table below to build different microservices from source:
 
-| Microservice    | Deployment Guide                                                                                                    |
-|-----------------|---------------------------------------------------------------------------------------------------------------------|
-| vLLM            | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker)      |
-| TGI             | [TGI project](https://github.com/huggingface/text-generation-inference.git)                                         |
-| LLM             | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                                  |
-| Redis Vector DB | [Redis](https://github.com/redis/redis.git)                                                                         |
-| Dataprep        | [Dataprep build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/dataprep/src/README_redis.md)     |
-| TEI Embedding   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                           |
-| Retriever       | [Retriever build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/retrievers/src/README_redis.md ) |
-| TEI Reranking   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                           |
-| MegaService     | [MegaService guide](../../../../README.md)                                                                          |
-| UI              | [UI guide](../../../../ui/react/README.md)                                                                          |
-| Nginx           | [Nginx guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/nginx)                       |
+| Microservice    | Deployment Guide                                                                                                   |
+| --------------- | ------------------------------------------------------------------------------------------------------------------ |
+| vLLM            | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker)     |
+| TGI             | [TGI project](https://github.com/huggingface/text-generation-inference.git)                                        |
+| LLM             | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                                 |
+| Redis Vector DB | [Redis](https://github.com/redis/redis.git)                                                                        |
+| Dataprep        | [Dataprep build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/dataprep/src/README_redis.md)    |
+| TEI Embedding   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                          |
+| Retriever       | [Retriever build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/retrievers/src/README_redis.md) |
+| TEI Reranking   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                          |
+| MegaService     | [MegaService guide](../../../../README.md)                                                                         |
+| UI              | [UI guide](../../../../ui/react/README.md)                                                                         |
+| Nginx           | [Nginx guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/nginx)                      |
 
 ### Check the Deployment Status
 
@@ -170,6 +171,7 @@ eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-emb
 ```
 
 if used TGI with FaqGen:
+
 ```
 CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
 eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
@@ -185,6 +187,7 @@ eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-emb
 ```
 
 if used vLLM:
+
 ```
 CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
 eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
@@ -199,6 +202,7 @@ eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-emb
 ```
 
 if used vLLM with FaqGen:
+
 ```
 CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
 eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
@@ -249,7 +253,7 @@ docker compose -f compose.yaml down
 In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
 
 | File                                                   | Description                                                                                                              |
-|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
+| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------ |
 | [compose.yaml](./compose.yaml)                         | The LLM serving framework is TGI. Default compose file using TGI as serving framework and redis as vector database       |
 | [compose_faqgen.yaml](./compose_faqgen.yaml)           | The LLM serving framework is TGI with FaqGen. All other configurations remain the same as the default                    |
 | [compose_vllm.yaml](./compose_vllm.yaml)               | The LLM serving framework is vLLM. Compose file using vllm as serving framework and redis as vector database             |
@@ -275,6 +279,7 @@ In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, w
      -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
      -H 'Content-Type: application/json'
    ```
+
 3. TEI Reranking Service
 
    ```bash
@@ -287,11 +292,11 @@ In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, w
 4. vLLM/TGI Service
 
    If you use vLLM:
-   
+
    ```bash
    DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
    '"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
-   
+
    curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
      -X POST \
      -d "$DATA" \
@@ -303,7 +308,7 @@ In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, w
    ```bash
    DATA='{"inputs":"What is a Deep Learning?",'\
    '"parameters":{"max_new_tokens":64,"do_sample": true}}'
-   
+
    curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
    -X POST \
    -d "$DATA" \
@@ -316,14 +321,13 @@ In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, w
    DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
    'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
    'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
-   
+
    curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
      -X POST \
      -d "$DATA" \
      -H 'Content-Type: application/json'
    ```
 
-
 ## Conclusion
 
-This guide should enable developers to deploy the default configuration or any of the other compose yaml files for different configurations. It also highlights the configurable parameters that can be set before deployment.
\ No newline at end of file
+This guide should enable developers to deploy the default configuration or any of the other compose yaml files for different configurations. It also highlights the configurable parameters that can be set before deployment.

From 39bb7ae6669b368538eee972df2a494f41a6c341 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Tue, 22 Apr 2025 15:58:41 +0700
Subject: [PATCH 15/17] ChatQnA - refactoring README.md

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 .../docker_compose/amd/gpu/rocm/README_old.md | 621 ------------------
 1 file changed, 621 deletions(-)
 delete mode 100644 ChatQnA/docker_compose/amd/gpu/rocm/README_old.md

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md b/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
deleted file mode 100644
index 0edcf44141..0000000000
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README_old.md
+++ /dev/null
@@ -1,621 +0,0 @@
-# Build and Deploy ChatQnA Application on AMD GPU (ROCm)
-
-## Build Docker Images
-
-### 1. Build Docker Image
-
-- #### Create application install directory and go to it:
-
-  ```bash
-  mkdir ~/chatqna-install && cd chatqna-install
-  ```
-
-- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git
-  ```
-
-  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
-  ```
-
-  We remind you that when using a specific version of the code, you need to use the README from this version:
-
-- #### Go to build directory:
-
-  ```bash
-  cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_image_build
-  ```
-
-- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
-  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
-
-  ```bash
-  echo Y | rm -R GenAIComps
-  ```
-
-- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git
-  ```
-
-  If you use a specific tag of the GenAIExamples repository,
-  then you should also use the corresponding tag for GenAIComps. (v1.3 replace with its own value):
-
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout v1.3
-  ```
-
-  We remind you that when using a specific version of the code, you need to use the README from this version.
-
-- #### Setting the list of images for the build (from the build file.yaml)
-
-  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
-
-  #### vLLM-based application
-
-  ```bash
-  service_list="dataprep retriever vllm-rocm chatqna chatqna-ui nginx"
-  ```
-
-  #### vLLM-based application with FaqGen
-
-  ```bash
-  service_list="dataprep retriever vllm-rocm llm-faqgen chatqna chatqna-ui nginx"
-  ```
-
-  #### TGI-based application
-
-  ```bash
-  service_list="dataprep retriever chatqna chatqna-ui nginx"
-  ```
-
-  #### TGI-based application with FaqGen
-
-  ```bash
-  service_list="dataprep retriever llm-faqgen chatqna chatqna-ui nginx"
-  ```
-
-- #### Pull Docker Images
-
-  ```bash
-  docker pull redis/redis-stack:7.2.0-v9
-  docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  ```
-
-- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
-
-  ```bash
-  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  ```
-
-- #### Build Docker Images
-
-  ```bash
-  docker compose -f build.yaml build ${service_list} --no-cache
-  ```
-
-  After the build, we check the list of images with the command:
-
-  ```bash
-  docker image ls
-  ```
-
-  The list of images should include:
-
-  ##### vLLM-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### vLLM-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
----
-
-## Deploy the ChatQnA Application
-
-### Docker Compose Configuration for AMD GPUs
-
-To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
-
-- compose_vllm.yaml - for vLLM-based application
-- compose_faqgen_vllm.yaml - for vLLM-based application with FaqGen
-- compose.yaml - for TGI-based
-- compose_faqgen.yaml - for TGI-based application with FaqGen
-
-```yaml
-shm_size: 1g
-devices:
-  - /dev/kfd:/dev/kfd
-  - /dev/dri:/dev/dri
-cap_add:
-  - SYS_PTRACE
-group_add:
-  - video
-security_opt:
-  - seccomp:unconfined
-```
-
-This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
-
-```yaml
-shm_size: 1g
-devices:
-  - /dev/kfd:/dev/kfd
-  - /dev/dri/card0:/dev/dri/card0
-  - /dev/dri/render128:/dev/dri/render128
-cap_add:
-  - SYS_PTRACE
-group_add:
-  - video
-security_opt:
-  - seccomp:unconfined
-```
-
-**How to Identify GPU Device IDs:**
-Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
-
-### Set deploy environment variables
-
-#### Setting variables in the operating system environment:
-
-##### Set variable HUGGINGFACEHUB_API_TOKEN:
-
-```bash
-### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
-export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
-```
-
-#### Set variables value in set_env\*\*\*\*.sh file:
-
-Go to Docker Compose directory:
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-```
-
-The example uses the Nano text editor. You can use any convenient text editor:
-
-#### If you use vLLM based application
-
-```bash
-nano set_env_vllm.sh
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-nano set_env_vllm_faqgen.sh
-```
-
-#### If you use TGI based application
-
-```bash
-nano set_env.sh
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-nano set_env_faqgen.sh
-```
-
-If you are in a proxy environment, also set the proxy-related environment variables:
-
-```bash
-export http_proxy="Your_HTTP_Proxy"
-export https_proxy="Your_HTTPs_Proxy"
-```
-
-Set the values of the variables:
-
-- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
-
-  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
-
-  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
-
-  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
-
-  We set these values in the file set_env\*\*\*\*.sh
-
-- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
-  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
-
-#### Set variables with script set_env\*\*\*\*.sh
-
-#### If you use vLLM based application
-
-```bash
-. set_env_vllm.sh
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-. set_env_faqgen_vllm.sh
-```
-
-#### If you use TGI based application
-
-```bash
-. set_env.sh
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-. set_env_faqgen.sh
-```
-
-### Start the services:
-
-#### If you use vLLM based application
-
-```bash
-docker compose -f compose_vllm.yaml up -d
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-docker compose -f compose_faqgen_vllm.yaml up -d
-```
-
-#### If you use TGI based application
-
-```bash
-docker compose -f compose.yaml up -d
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-docker compose -f compose_faqgen.yaml up -d
-```
-
-All containers should be running and should not restart:
-
-##### If you use vLLM based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use vLLM based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use TGI based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
-##### If you use TGI based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
----
-
-## Validate the Services
-
-### 1. Validate TEI Embedding Service
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
-    -X POST \
-    -d '{"inputs":"What is Deep Learning?"}' \
-    -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to text:
-
-```textmate
-[[0.00037115702,-0.06356819,0.0024758505,..................,0.022725677,0.016026087,-0.02125421,-0.02984927,-0.0049473033]]
-```
-
-If the service response has a meaningful response in the value,
-then we consider the TEI Embedding Service to be successfully launched
-
-### 2. Validate Retriever Microservice
-
-```bash
-export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
-curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
-  -X POST \
-  -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{ "id": "e191846168aed1f80b2ea12df80844d2", "retrieved_docs": [], "initial_query": "test", "top_n": 1, "metadata": [] }
-```
-
-If the response corresponds to the form of the provided JSON, then we consider the
-Retriever Microservice verification successful.
-
-### 3. Validate TEI Reranking Service
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
-    -X POST \
-    -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
-    -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-[
-  { "index": 1, "score": 0.94238955 },
-  { "index": 0, "score": 0.120219156 }
-]
-```
-
-If the response corresponds to the form of the provided JSON, then we consider the TEI Reranking Service
-verification successful.
-
-### 4. Validate the vLLM/TGI Service
-
-#### If you use vLLM:
-
-```bash
-DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
-'"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
-
-curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "id": "chatcmpl-91003647d1c7469a89e399958f390f67",
-  "object": "chat.completion",
-  "created": 1742877228,
-  "model": "meta-llama/Meta-Llama-3-8B-Instruct",
-  "choices": [
-    {
-      "index": 0,
-      "message": {
-        "role": "assistant",
-        "content": "Deep Learning ( DL) is a subfield of Machine Learning (ML) that focuses on the design of algorithms and architectures inspired by the structure and function of the human brain. These algorithms are designed to analyze and interpret data that is presented in the form of patterns or signals, and they often mimic the way the human brain",
-        "tool_calls": []
-      },
-      "logprobs": null,
-      "finish_reason": "length",
-      "stop_reason": null
-    }
-  ],
-  "usage": { "prompt_tokens": 16, "total_tokens": 80, "completion_tokens": 64, "prompt_tokens_details": null },
-  "prompt_logprobs": null
-}
-```
-
-If the service response has a meaningful response in the value of the "choices.message.content" key,
-then we consider the vLLM service to be successfully launched
-
-#### If you use TGI:
-
-```bash
-DATA='{"inputs":"What is a Deep Learning?",'\
-'"parameters":{"max_new_tokens":64,"do_sample": true}}'
-
-curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "generated_text": " What is its application in Computer Vision?\nWhat is a Deep Learning?\nDeep learning is a subfield of machine learning that involves the use of artificial neural networks to model high-level abstractions in data. It involves the use of deep neural networks, which are composed of multiple layers, to learn complex patterns in data. The"
-}
-```
-
-If the service response has a meaningful response in the value of the "generated_text" key,
-then we consider the TGI service to be successfully launched
-
-### 5. Validate the LLM Service (if your used application with FaqGen)
-
-```bash
-DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
-'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
-'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
-
-curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "id": "58f0632f5f03af31471b895b0d0d397b",
-  "text": " Q: What is Text Embeddings Inference (TEI)?\n         A: TEI is a toolkit for deploying and serving open source text embeddings and sequence classification models.\n\n         Q: What models does TEI support?\n         A: TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.\n\n         Q: What is the purpose of TEI?\n         A: The purpose of TEI is to enable high-performance extraction for text embeddings and sequence classification models.\n\n         Q: What are the benefits of using TEI?\n         A: The benefits of using TEI include high",
-  "prompt": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-}
-```
-
-If the service response has a meaningful response in the value of the "text" key,
-then we consider the LLM service to be successfully launched
-
-### 6. Validate the MegaService
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
-  -H "Content-Type: application/json" \
-  -d '{"messages": "What is the revenue of Nike in 2023?"}'
-```
-
-Checking the response from the service. The response should be similar to text:
-
-```textmate
-data: b' What'
-data: b' is'
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b' Nike'
-data: b' in'
-data: b' '
-data: b'202'
-data: b'3'
-data: b'?\n'
-data: b'        '
-data: b' Answer'
-data: b':'
-data: b' According'
-data: b' to'
-data: b' the'
-data: b' search'
-data: b' results'
-data: b','
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b''
-
-data: [DONE]
-
-```
-
-If the output lines in the "data" keys contain words (tokens) containing meaning, then the service
-is considered launched successfully.
-
-### 7. Validate the Frontend (UI)
-
-To access the UI, use the URL - http://${EXTERNAL_HOST_IP}:${CHATQNA_NGINX_PORT}
-A page should open when you click through to this address:
-
-![UI start page](../../../../assets/img/ui-starting-page.png)
-
-If a page of this type has opened, then we believe that the service is running and responding,
-and we can proceed to functional UI testing.
-
-Let's enter the task for the service in the "Enter prompt here" field.
-For example, "What is a Deep Learning?" and press Enter.
-After that, a page with the result of the task should open:
-
-#### If used application without FaqGen
-
-![UI result page](../../../../assets/img/ui-result-page.png)
-
-#### If used application with FaqGen
-
-![UI result page](../../../../assets/img/ui-result-page-faqgen.png)
-
-If the result shown on the page is correct, then we consider the verification of the UI service to be successful.
-
-### 5. Stop application
-
-#### If you use vLLM
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_vllm.yaml down
-```
-
-#### If you use vLLM with FaqGen
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen_vllm.yaml down
-```
-
-#### If you use TGI
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose.yaml down
-```
-
-#### If you use TGI with FaqGen
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen.yaml down
-```

From a97f163d3d686bb75821c20310069a18759b142c Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Thu, 24 Apr 2025 13:08:53 +0700
Subject: [PATCH 16/17] ChatQnA - refactoring README.md and fix container names
 in Docker Compose files

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md          | 10 +++++-----
 ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml       |  2 +-
 .../docker_compose/amd/gpu/rocm/compose_faqgen.yaml    |  2 +-
 .../amd/gpu/rocm/compose_faqgen_vllm.yaml              |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index 3d273f41c1..d7a0f43a8d 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -59,7 +59,7 @@ Set the values of the variables:
   We set these values in the file set_env\*\*\*\*.sh
 
 - **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
-  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
+  The values shown in the file set_env.sh or set_env_vllm.sh they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
 
 Setting variables in the operating system environment:
 
@@ -159,7 +159,7 @@ For the default deployment with TGI, the following 9 containers should have star
 
 ```
 CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
-eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
 2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
 613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
 05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
@@ -174,7 +174,7 @@ if used TGI with FaqGen:
 
 ```
 CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
-eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
 2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
 613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
 e0ef1ea67640   opea/llm-faqgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              chatqna-llm-faqgen
@@ -190,7 +190,7 @@ if used vLLM:
 
 ```
 CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
-eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
 2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
 613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
 05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
@@ -205,7 +205,7 @@ if used vLLM with FaqGen:
 
 ```
 CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
-eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
 2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
 613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
 e0ef1ea67640   opea/llm-faqgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              chatqna-llm-faqgen
diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml b/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml
index 078464bb9e..a71fcc830a 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml
@@ -165,7 +165,7 @@ services:
 
   chatqna-nginx-server:
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
-    container_name: chaqna-nginx-server
+    container_name: chatqna-nginx-server
     depends_on:
       - chatqna-backend-server
       - chatqna-ui-server
diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen.yaml b/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen.yaml
index aec3987754..161bb4589f 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen.yaml
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen.yaml
@@ -187,7 +187,7 @@ services:
 
   chatqna-nginx-server:
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
-    container_name: chaqna-nginx-server
+    container_name: chatqna-nginx-server
     depends_on:
       - chatqna-backend-server
       - chatqna-ui-server
diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen_vllm.yaml b/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen_vllm.yaml
index 1c193f7181..b89b367e29 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen_vllm.yaml
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen_vllm.yaml
@@ -192,7 +192,7 @@ services:
 
   chatqna-nginx-server:
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
-    container_name: chaqna-nginx-server
+    container_name: chatqna-nginx-server
     depends_on:
       - chatqna-backend-server
       - chatqna-ui-server

From ad54ac7827b584bc5979a1cd2f7afda699cdf3f8 Mon Sep 17 00:00:00 2001
From: Chingis Yundunov <c.yundunov@datamonsters.com>
Date: Thu, 24 Apr 2025 19:47:37 +0700
Subject: [PATCH 17/17] ChatQnA - refactoring README.md and fix container names
 in Docker Compose files

Signed-off-by: Chingis Yundunov <c.yundunov@datamonsters.com>
---
 ChatQnA/docker_compose/amd/gpu/rocm/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
index d7a0f43a8d..4d968b84eb 100644
--- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md
@@ -31,10 +31,10 @@ git clone https://github.com/opea-project/GenAIExamples.git
 cd GenAIExamples/ChatQnA
 ```
 
-Then checkout a released version, such as v1.2:
+Then checkout a released version, such as v1.3:
 
 ```bash
-git checkout v1.2
+git checkout v1.3
 ```
 
 ### Configure the Deployment Environment
