Skip to content

fixing document service #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kirtimanmishrazipstack wants to merge 1 commit into from
Closed

fixing document service #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions docker/dockerfiles/document.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ ENV PDM_VERSION 2.12.3

RUN DEBIAN_FRONTEND=noninteractive apt-get update; \
apt-get --no-install-recommends install -y \
fonts-dejavu fonts-dejavu-core fonts-dejavu-extra fonts-droid-fallback fonts-dustin \
fonts-f500 fonts-fanwood fonts-freefont-ttf \
fonts-liberation fonts-lmodern fonts-lyx \
fonts-opensymbol fonts-sil-gentium fonts-texgyre fonts-tlwg-purisa \
hyphen-af hyphen-en-us \
libreoffice-common \
python${PYTHON_VERSION} python3-pip \
software-properties-common \
unoconv; \
fonts-dejavu fonts-dejavu-core fonts-dejavu-extra fonts-droid-fallback fonts-dustin \
fonts-f500 fonts-fanwood fonts-freefont-ttf \
fonts-liberation fonts-lmodern fonts-lyx \
fonts-opensymbol fonts-sil-gentium fonts-texgyre fonts-tlwg-purisa \
hyphen-af hyphen-en-us \
libreoffice-common libreoffice-dev libreoffice-script-provider-python python3-uno \
python${PYTHON_VERSION} python3-pip \
software-properties-common \
unoconv; \
apt-get clean && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*; \
\
pip install --no-cache-dir -U pip pdm~=${PDM_VERSION};
Expand All @@ -32,6 +32,8 @@ WORKDIR /app

COPY ${BUILD_CONTEXT_PATH}/ .

RUN pip install --no-cache-dir unoserver==1.5

RUN set -e; \
\
rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \
Expand All @@ -49,7 +51,7 @@ RUN set -e; \

# Creates a non-root user with an explicit UID and adds permission to access the /app folder
# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
RUN adduser -u 5678 --disabled-password --gecos "" unstract;
RUN adduser -u 5678 --disabled-password --gecos "" unstract && chown -R unstract /app;

USER unstract

Expand Down
53 changes: 40 additions & 13 deletions document-service/src/unstract/document_service/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,26 @@
from typing import Any

import redis
from dotenv import load_dotenv
from flask import Flask, request, send_file
from odf import teletype, text
from odf.opendocument import load

load_dotenv()

logging.basicConfig(
level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s : %(message)s"
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s : %(message)s",
)

UPLOAD_FOLDER = os.environ.get("UPLOAD_FOLDER", "/tmp/document_service/upload")
PROCESS_FOLDER = os.environ.get("PROCESS_FOLDER", "/tmp/document_service/process")
PROCESS_FOLDER = os.environ.get(
"PROCESS_FOLDER", "/tmp/document_service/process"
)
LIBREOFFICE_PYTHON = os.environ.get("LIBREOFFICE_PYTHON", "/usr/bin/python3")
MAX_FILE_SIZE = int(os.environ.get("MAX_FILE_SIZE", 10485760)) # 10 * 1024 * 1024
MAX_FILE_SIZE = int(
os.environ.get("MAX_FILE_SIZE", 10485760)
) # 10 * 1024 * 1024
SERVICE_API_TOKEN = os.environ.get("SERVICE_API_TOKEN", "")

app = Flask("document_service")
Expand Down Expand Up @@ -99,7 +107,9 @@ def upload_file():
redis_host = os.environ.get("REDIS_HOST")
redis_port = int(os.environ.get("REDIS_PORT"))
redis_password = os.environ.get("REDIS_PASSWORD")
r = redis.Redis(host=redis_host, port=redis_port, password=redis_password)
r = redis.Redis(
host=redis_host, port=redis_port, password=redis_password
)
# TODO: Create a file reaper process to look at uploaded time and delete
redis_key = f"upload_time:{account_id}_{file_name}"
current_timestamp = int(time.time())
Expand All @@ -123,7 +133,9 @@ def find_and_replace():
output_format = request.args.get("output_format").lower()
find_and_replace_text = request.json

app.logger.info(f"Find and replace for file {file_name} for account {account_id}")
app.logger.info(
f"Find and replace for file {file_name} for account {account_id}"
)
app.logger.info(f"Output format: {output_format}")

if output_format not in ["pdf"]:
Expand All @@ -141,9 +153,12 @@ def find_and_replace():
file_name_odt = f"{account_id}_{file_name}.odt"
file_name_odt = os.path.join(PROCESS_FOLDER, file_name_odt)
try:
command = f"{LIBREOFFICE_PYTHON} -m unoserver.converter --convert-to odt \
command = f"{LIBREOFFICE_PYTHON} -m unoserver.converter \
--convert-to odt \
--filter writer8 {file_namex} {file_name_odt}"
result = subprocess.run(command, shell=True, capture_output=True, text=True)
result = subprocess.run(
command, shell=True, capture_output=True, text=True
)
app.logger.info(result)
if result.returncode != 0:
app.logger.error(
Expand All @@ -155,7 +170,9 @@ def find_and_replace():
app.logger.info(
f"File converted to ODT format successfully! {file_name_odt}"
)
app.logger.info(f"ODT convertion result: {result.stdout} | {result.stderr}")
app.logger.info(
f"ODT convertion result: {result.stdout} | {result.stderr}"
)
except Exception as e:
app.logger.error(f"Error while converting file to ODT format: {e}")
return "Error while converting file to ODT format!", 500
Expand All @@ -169,9 +186,13 @@ def find_and_replace():
replace_str = find_and_replace_text[find_str]
for element in doc.getElementsByType(text.Span):
if find_str in teletype.extractText(element):
app.logger.info(f"Found {find_str} in {teletype.extractText(element)}")
app.logger.info(
f"Found {find_str} in {teletype.extractText(element)}"
)
new_element = text.Span()
new_element.setAttribute("stylename", element.getAttribute("stylename"))
new_element.setAttribute(
"stylename", element.getAttribute("stylename")
)
t = teletype.extractText(element)
t = t.replace(find_str, replace_str)
new_element.addText(t)
Expand All @@ -188,7 +209,9 @@ def find_and_replace():
f"{LIBREOFFICE_PYTHON} -m unoserver.converter --convert-to pdf "
f"--filter writer_pdf_Export {file_name_odt} {file_name_output}"
)
result = subprocess.run(command, shell=True, capture_output=True, text=True)
result = subprocess.run(
command, shell=True, capture_output=True, text=True
)
if result.returncode != 0:
app.logger.error(
f"Failed to convert file to {output_format} format: "
Expand All @@ -200,9 +223,13 @@ def find_and_replace():
f"File converted to {output_format} format successfully! "
f"{file_name_output}"
)
app.logger.info(f"ODT convertion result: {result.stdout} | {result.stderr}")
app.logger.info(
f"ODT convertion result: {result.stdout} | {result.stderr}"
)
except Exception as e:
app.logger.error(f"Error while converting file to {output_format} format: {e}")
app.logger.error(
f"Error while converting file to {output_format} format: {e}"
)
return f"Error while converting file to {output_format} format!", 500
return send_file(file_name_output, as_attachment=True)

Expand Down