Dodani popravki za iskanje kandidatov in števila besedil. Dodana logika za klicanje ocrja. Popravljen bug za timeout v ateapi-ju. Popravljen swager.

swagger_server/utils/txt_utils.py

@@ -9,7 +9,7 @@ from swagger_server.utils import cl_utils
 import cv2
 import numpy as np
 import magic
-
+import re
 tika_server = "http://tika2:9999/tika"
 
 # endpoint below to be used only for development purposes (don't need to run docker)
@@ -26,28 +26,54 @@ def extract_text_prepResp(file, content_type=""):
         try:
             response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
             content = response.text
+            #preveri če je pretvorba uspešna
+            
+            # original string
+            res = re.findall(r'\w+', content)
+
+            #preveri, če imamo vsaj 10 besed in če je povprečna dolžina >3 in < 12
+            #če to drži, idi v ocr
+            reslen=map(lambda n:len(n),res)
+            print(f"Število besed je {len(res)}")
+
+            if len(res)>0 :
+                avglen=sum(reslen)/len(res)
+            else:
+                avglen=0
+            
+            print(f"Povprečna dolžina besede je {avglen}")
+            
+            if(len(res)<10 or avglen<4 or avglen>11):
+                print("Besedilo je sumljivo, gremo v OCR in damo file na začetek!")
+                file.seek(0)
+                response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
+                                             "Accept": "text/plain; charset=UTF-8"})
+                content = response.text
+
+            #odstranim še vse prelome vrstic, ker imamo s tem probleme
+            content=' '.join(content.splitlines())
         except:
             content = "ERROR - something went wrong when reading file with tika"
 
-    if content == "":
-        if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
-            content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
-        elif "application/pdf" in content_type:
-            reader = PdfReader(file)
-            content = '\n'.join([p.extract_text() for p in reader.pages])
-            content = content
-        elif "text/xml" in content_type:
-            root = ET.parse(file).getroot()
-            plainText = root.findall('PlainText')
-            if len(plainText) == 0:
-                return "Didn't find anything in PlainText", 400
-            content = '\n'.join([pt.text for pt in plainText])
-        # elif "text/plain" in file.content_type:
-        else:
-            try:
-                content = file.read().decode('utf-8')
-            except:
-                content = "ERROR - something went wrong when reading file with not-tika method!"
+    #if content == "":
+    #    if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
+    #        content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
+    #    elif "application/pdf" in content_type:
+    #        reader = PdfReader(file)
+    #        content = '\n'.join([p.extract_text() for p in reader.pages])
+    #       content = content
+    #    elif "text/xml" in content_type:
+    #        root = ET.parse(file).getroot()
+    #        plainText = root.findall('PlainText')
+    #        if len(plainText) == 0:
+    #            return "Didn't find anything in PlainText", 400
+    #        content = '\n'.join([pt.text for pt in plainText])
+    #   # elif "text/plain" in file.content_type:
+    #    else:
+    #       try:
+    #           content = file.read().decode('utf-8')
+    #       except:
+    #           content = "ERROR - something went wrong when reading file with not-tika method!"
 
     return content, 200