updated endpoint definitions/names and reformatted some files and code

swagger_server/utils/db_utils.py

@@ -0,0 +1,54 @@
+import mariadb
+import sys
+from decouple import config
+
+
+database_info = {
+    'database': config('MDB_DATABASE'),
+    'host': config('MDB_HOST'),
+    'port': config('MDB_PORT', cast=int),
+    'user': config('MDB_USER'),
+    'password': config('MDB_PASSWORD')
+}
+
+cur = None
+
+# Connect to MariaDB Platform
+try:
+    conn = mariadb.connect(**database_info)
+    # Get cursor
+    cur = conn.cursor()
+except mariadb.Error as e:
+    print(f"Error connecting to MariaDB Platform: {e}")
+    # sys.exit(1)
+
+
+# class BaseModel(Model):
+#     class Meta:
+#         database = db
+#
+#
+# class os2022_ngrams(BaseModel):
+#     file_id = IntegerField()
+#     sent_id = FloatField()
+#     ngram_len = IntegerField()
+#     frequency_g_t = IntegerField()
+#     gram_text = TextField()
+#     lemma_text = TextField()
+#     xpos_text = TextField()
+#     upos_text = TextField()
+#
+# db.connect()
+
+
+class Ngrams_Manager:
+    @staticmethod
+    def get_by_file_id(file_id):
+        try:
+            # cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}')
+            # cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
+            # return list(cur)
+            return 1
+        except Exception as e:
+            print(e, 'EXC')
+        return 0

swagger_server/utils/txt_utils.py

@@ -0,0 +1,65 @@
+import os.path
+
+import pytesseract
+import requests
+import docx
+import xml.etree.ElementTree as ET
+from PyPDF2 import PdfReader
+from swagger_server.classla import cl_utils
+import cv2
+import numpy as np
+
+tika_server = "http://tika2:9999/tika"
+
+
+# endpoint below to be used only for development purposes (don't need to run docker)
+# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
+
+def extract_text_prepResp(file):
+    if tika_responding():
+        response = requests.put(tika_server, data=file)
+        return response.text, 200
+    if "openxmlformats-officedocument.wordprocessingml.document" in file.content_type:
+        content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
+    elif "application/pdf" in file.content_type:
+        reader = PdfReader(file)
+        content = '\n'.join([p.extract_text() for p in reader.pages])
+        content = content
+    elif "text/xml" in file.content_type:
+        root = ET.parse(file).getroot()
+        plainText = root.findall('PlainText')
+        if len(plainText) == 0:
+            return "Didn't find anything in PlainText", 400
+        content = '\n'.join([pt.text for pt in plainText])
+    # elif "text/plain" in file.content_type:
+    else:
+        content = file.read().decode('utf-8')
+
+    return content, 200
+
+
+def ocr_text_prepResp(file):
+    if tika_responding():
+        response = requests.put(tika_server, data=file,
+                                headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
+        return response.text, 200
+
+    win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"
+    if os.path.exists(win_p):
+        pytesseract.pytesseract.tesseract_cmd = win_p
+
+    # convert string data to numpy array
+    file_bytes = np.fromstring(file.read(), np.uint8)
+    # convert numpy array to image
+    img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
+
+    conf = '-l eng+slv'
+    return pytesseract.image_to_string(img, config=conf), 200
+
+
+def tika_responding():
+    try:
+        ret = requests.get(tika_server)
+        return ret.status_code == 200
+    except:
+        return False