diff --git a/Dockerfile b/Dockerfile index 1ecc800..1c1903f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ FROM python:3.9 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app +RUN apt-get update && apt-get install python3-pil tesseract-ocr libtesseract-dev tesseract-ocr-eng tesseract-ocr-slv tesseract-ocr-script-latn ffmpeg libsm6 libxext6 libgl1 -y + COPY requirements.txt /usr/src/app/ # RUN pip install --upgrade pip diff --git a/requirements.txt b/requirements.txt index 67765bc..ea9e7a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,7 @@ pandas==1.3.3 python-docx==0.8.11 lxml==4.8.0 PyPDF2==2.10.4 -requests \ No newline at end of file +requests +pytesseract==0.3.10 +opencv-python==4.5.2.54 +numpy==1.20.3 diff --git a/swagger_server/controllers/doc2text_controller.py b/swagger_server/controllers/doc2text_controller.py index 7b69dd0..2b39edc 100644 --- a/swagger_server/controllers/doc2text_controller.py +++ b/swagger_server/controllers/doc2text_controller.py @@ -1,6 +1,8 @@ +import os.path from tempfile import TemporaryFile import connexion +import pytesseract import six import requests import json @@ -12,13 +14,60 @@ import xml.etree.ElementTree as ET from PyPDF2 import PdfReader from swagger_server.classla import cl_utils import traceback +import cv2 +import numpy as np + +tika_server = "http://tika2:9999/tika_NAROBENURLZANALASC" -tika_server = "http://tika2:9999/tika" # endpoint below to be used only for development purposes (don't need to run docker) # tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika" +def extract_text_if_no_tika(file): + if file is None: + return "No file provided", 400 + elif "openxmlformats-officedocument.wordprocessingml.document" in file.content_type: + content = [p.text for p in docx.Document(file).paragraphs] + elif "application/pdf" in file.content_type: + reader = PdfReader(file) + content = '\n'.join([p.extract_text() for p in reader.pages]) + content = "ZACASNO UPORABLJEN DRUGI BRALEC KOT TIKA, TA BO SE DODANA KASNEJE...\n\n" + content + elif "text/xml" in file.content_type: + root = ET.parse(file).getroot() + plainText = root.findall('PlainText') + if len(plainText) == 0: + return "Didn't find anything in PlainText", 400 + content = '\n'.join([pt.text for pt in plainText]) + # elif "text/plain" in file.content_type: + else: + content = file.read().decode('utf-8') + + return content, 200 + + +def ocr_text_if_no_tika(file): + win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe" + if os.path.exists(win_p): + pytesseract.pytesseract.tesseract_cmd = win_p + + # convert string data to numpy array + file_bytes = np.fromstring(file.read(), np.uint8) + # convert numpy array to image + img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) + + conf = '-l eng+slv' + return pytesseract.image_to_string(img, config=conf), 200 + + +def tika_responding(): + try: + ret = requests.get(tika_server) + return ret.status_code == 200 + except: + return False + + def datoteka_v_besedilo_post(file=None): # noqa: E501 """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo @@ -32,6 +81,8 @@ def datoteka_v_besedilo_post(file=None): # noqa: E501 if file is None: return "No file provided", 400 try: + if not tika_responding(): + return extract_text_if_no_tika(file) response = requests.put(tika_server, data=file) return response.text, 200 except Exception as e: @@ -53,6 +104,8 @@ def get_text_ocr(file=None): # noqa: E501 if file is None: return "No file provided", 400 try: + if not tika_responding(): + return ocr_text_if_no_tika(file) response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) return response.text, 200 @@ -75,8 +128,12 @@ def datoteka_v_besedilo_in_classla(file=None): # noqa: E501 if file is None: return "No file provided", 400 try: - response = requests.put(tika_server, data=file) - return cl_utils.raw_text_to_conllu(response.text) + if tika_responding(): + response = requests.put(tika_server, data=file) + return cl_utils.raw_text_to_conllu(response.text) + else: + txt, _ = extract_text_if_no_tika(file) + return cl_utils.raw_text_to_conllu(txt) except Exception as e: return str(e), 500 @@ -96,8 +153,12 @@ def get_conllu_ocr(file=None): # noqa: E501 if file is None: return "No file provided", 400 try: - response = requests.put(tika_server, data=file, - headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) - return cl_utils.raw_text_to_conllu(response.text) + if tika_responding(): + response = requests.put(tika_server, data=file, + headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) + return cl_utils.raw_text_to_conllu(response.text) + else: + txt, _ = ocr_text_if_no_tika(file) + return cl_utils.raw_text_to_conllu(txt) except Exception as e: return str(e), 500