Dodani popravki za iskanje kandidatov in števila besedil. Dodana logika za klicanje ocrja. Popravljen bug za timeout v ateapi-ju. Popravljen swager.

ATEapi/Dockerfile

@@ -28,4 +28,4 @@ RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /
 USER appuser
 
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD ["gunicorn", "--bind", "0.0.0.0:5000", "main:app"]
+CMD ["gunicorn", "-t 0", "--bind", "0.0.0.0:5000", "main:app"]

docker-compose.yml

@@ -11,7 +11,7 @@ services:
       - ../classla/classla_resources:/root/classla_resources
     environment:
       - PYTHONUNBUFFERED=1
-      - MDB_DATABASE=corpus_120k
+      - MDB_DATABASE=conllus_150k
       - MDB_HOST=164.8.252.72
       - MDB_PORT=3306
       - MDB_USER=rsdo5

swagger_server/controllers/oss_controller.py

@@ -43,7 +43,9 @@ def get_extracted_words(leta=None, vrste=None, kljucnebesede=None, udk=None):  #
 
     :rtype: List[TerminoloskiKandidat]
     """
-    return 'do some magic!5'
+    files = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk)
+    return files, 200
+
 
 
 def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501

swagger_server/swagger/swagger.yaml

@@ -348,7 +348,7 @@ paths:
       parameters:
       - name: leta
         in: query
-        required: true
+        required: false
         style: form
         explode: true
         schema:
@@ -358,7 +358,7 @@ paths:
             format: int64
       - name: vrste
         in: query
-        required: true
+        required: false
         style: form
         explode: true
         schema:
@@ -368,7 +368,7 @@ paths:
             format: int64
       - name: kljucnebesede
         in: query
-        required: true
+        required: false
         style: form
         explode: true
         schema:
@@ -377,7 +377,7 @@ paths:
             type: string
       - name: udk
         in: query
-        required: true
+        required: false
         style: form
         explode: true
         schema:

swagger_server/utils/db_utils.py

@@ -78,6 +78,8 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
         
         print(sql)
         print(params)
+
+
         
         cur.execute(sql,params)
         
@@ -89,6 +91,81 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
     return ret
 
 
+def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
+    ret = []
+  
+    try:
+        print(database_info)
+        conn = mariadb.connect(**database_info)
+        cur = conn.cursor()
+        
+        
+        
+        sql = "select distinct document_id from metadata"
+        where=""
+        params=[]
+        if (udk):
+            where_in_udk = ','.join(['%s'] * len(udk))
+            where=" udk IN (%s) " % (where_in_udk)
+            params=udk
+        
+        if (leta):
+
+            where_in_leta = ','.join(['%s'] * len(leta))
+            if (where):
+                where=where+ " AND "
+            where=where + " leto IN (%s) " % (where_in_leta)
+            params=params+leta
+        
+        if (vrste):
+            if (where):
+                where=where+ " AND "
+            where_in_vrste = ','.join(['%s'] * len(vrste))
+            where=where + " tipologija IN (%s) " % (where_in_vrste)
+            params=params+vrste
+
+        if (kljucnebesede):
+            if (where):
+                where=where+ " AND "
+            where_in_kb = ','.join(['%s'] * len(kljucnebesede))
+            where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
+            params=params+kljucnebesede
+
+        if(where):
+            sql=sql+" where " + where
+        
+        print(sql)
+        print(params)
+
+        sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf from (
+                    SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf
+                    FROM ngrams_upos_tf tf, documents d,
+                        (
+                        Select ngram, upos, count(*) as df from ngrams_upos_tf TF
+                        where document_id in
+                    ({sql})
+                    group by TF.ngram, TF.upos
+                    ) dff, ngrams_upos_df df
+                    where
+                    tf.document_id=d.document_id and
+                    df.ngram=tf.ngram AND df.upos=tf.upos and
+                    dff.ngram=tf.ngram AND dff.upos=tf.upos
+                    ) X
+                    group by ngram,upos
+                    order by tfidf desc
+                    limit 100"""
+        
+        print (sqltk)
+        
+        cur.execute(sqltk,params)
+        
+        ret = list(cur)
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB Platform: {e}")
+    
+
+    return ret
+
 # class BaseModel(Model):
 #     class Meta:
 #         database = db

swagger_server/utils/txt_utils.py

@@ -9,7 +9,7 @@ from swagger_server.utils import cl_utils
 import cv2
 import numpy as np
 import magic
-
+import re
 tika_server = "http://tika2:9999/tika"
 
 # endpoint below to be used only for development purposes (don't need to run docker)
@@ -26,28 +26,54 @@ def extract_text_prepResp(file, content_type=""):
         try:
             response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
             content = response.text
+            #preveri če je pretvorba uspešna
+            
+            # original string
+            res = re.findall(r'\w+', content)
+
+            #preveri, če imamo vsaj 10 besed in če je povprečna dolžina >3 in < 12
+            #če to drži, idi v ocr
+            reslen=map(lambda n:len(n),res)
+            print(f"Število besed je {len(res)}")
+
+            if len(res)>0 :
+                avglen=sum(reslen)/len(res)
+            else:
+                avglen=0
+            
+            print(f"Povprečna dolžina besede je {avglen}")
+            
+            if(len(res)<10 or avglen<4 or avglen>11):
+                print("Besedilo je sumljivo, gremo v OCR in damo file na začetek!")
+                file.seek(0)
+                response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
+                                             "Accept": "text/plain; charset=UTF-8"})
+                content = response.text
+
+            #odstranim še vse prelome vrstic, ker imamo s tem probleme
+            content=' '.join(content.splitlines())
         except:
             content = "ERROR - something went wrong when reading file with tika"
 
-    if content == "":
+    #if content == "":
-        if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
+    #    if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
-            content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
+    #        content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
-        elif "application/pdf" in content_type:
+    #    elif "application/pdf" in content_type:
-            reader = PdfReader(file)
+    #        reader = PdfReader(file)
-            content = '\n'.join([p.extract_text() for p in reader.pages])
+    #        content = '\n'.join([p.extract_text() for p in reader.pages])
-            content = content
+    #       content = content
-        elif "text/xml" in content_type:
+    #    elif "text/xml" in content_type:
-            root = ET.parse(file).getroot()
+    #        root = ET.parse(file).getroot()
-            plainText = root.findall('PlainText')
+    #        plainText = root.findall('PlainText')
-            if len(plainText) == 0:
+    #        if len(plainText) == 0:
-                return "Didn't find anything in PlainText", 400
+    #            return "Didn't find anything in PlainText", 400
-            content = '\n'.join([pt.text for pt in plainText])
+    #        content = '\n'.join([pt.text for pt in plainText])
-        # elif "text/plain" in file.content_type:
+    #   # elif "text/plain" in file.content_type:
-        else:
+    #    else:
-            try:
+    #       try:
-                content = file.read().decode('utf-8')
+    #           content = file.read().decode('utf-8')
-            except:
+    #       except:
-                content = "ERROR - something went wrong when reading file with not-tika method!"
+    #           content = "ERROR - something went wrong when reading file with not-tika method!"
 
     return content, 200