Working fix pre merge

swagger_server/controllers/extract_controller.py

@@ -8,21 +8,25 @@ from swagger_server.models.izlusci_async_body import IzlusciAsyncBody  # noqa: E
 from swagger_server.models.izlusci_sync_body import IzlusciSyncBody  # noqa: E501
 from swagger_server.requets_db.models.vrsta import JobManager
 from swagger_server.utils import cl_utils
+from swagger_server.utils import txt_utils
 from swagger_server.util import get_random_filename, create_random_file_in_tmp_folder
 import requests
 from werkzeug.utils import secure_filename
 
 ATEapi_endpoint = "http://ate-api:5000/predict"
 
+
 # endpoint below to be used only for development purposes (don't need to run docker)
 # ATEapi_endpoint = "http://localhost:5000/predict"
 
 
-def do_izlusci(conllus, prepovedane_besede):
+def do_izlusci(conllus, prepovedane_besede,definicije=False):
     tmp_file_path = ""
+    print(definicije);
     try:
         big_conllu = cl_utils.multipla_conllus_to_one_from_conllus_arr(conllus)
         tmp_file_path = create_random_file_in_tmp_folder(big_conllu, ".conllu")
+
         fp = open(tmp_file_path, 'rb')
         try:
             files = [
@@ -30,26 +34,38 @@ def do_izlusci(conllus, prepovedane_besede):
             ]
             res = requests.post(ATEapi_endpoint, files=files)
             data = json.loads(res.text)
+        
+           
+            ret = {'terminoloski_kandidati': [
+                {
+                    'POSoznake': tk['term_example_pos'],
+                    'kandidat': tk['lemma'],  # more to bit lemma al terms?
+                    'kanonicnaoblika': tk['canonical'],
+                    'ranking': tk['ranking'],
+                    'podporneutezi': [
+                        0.0,  # ????????
+                        0.0  # ??????
+                    ],
+                    'pogostostpojavljanja': [tk['frequency'], 0]  # ???????
+                }
+                for tk in data if tk['canonical'] not in prepovedane_besede
+            ]}
+
+          
+
+
+            if definicije:
+                print("grem po definicije!!!")
+                ret = txt_utils.extract_definition_sentences(tmp_file_path,ret);    
+               
+        except Exception as e: print(e)
         finally:
             fp.close()
             os.remove(tmp_file_path)
 
-        ret = {'terminoloski_kandidati': [
-            {
-                'POSoznake': tk['msd'],
-                'kandidat': tk['terms'],  # more to bit lemma al terms?
-                'kanonicnaoblika': tk['canonical'],
-                'ranking': tk['ranking'],
-                'podporneutezi': [
-                    0.0,  # ????????
-                    0.0  # ??????
-                ],
-                'pogostostpojavljanja': [0, 0]  # ???????
-            }
-            for tk in data if tk['terms'] not in prepovedane_besede
-        ]}
         return ret, 200
     except Exception as e:
+        print(e)
         return str(e), 500
 
 
@@ -84,5 +100,5 @@ def get_candidates_sync(body):  # noqa: E501
     """
     if connexion.request.is_json:
         body = IzlusciSyncBody.from_dict(connexion.request.get_json())  # noqa: E501
-
-    return do_izlusci(body.conllus, body.prepovedane_besede)
+    print(body)
+    return do_izlusci(body.conllus, body.prepovedane_besede,body.definicije)

swagger_server/controllers/jobs_controller.py

@@ -6,28 +6,54 @@ import traceback
 import peewee
 import asyncio
 import concurrent.futures as cf
-
+from flask import Response
 from swagger_server.controllers.extract_controller import do_izlusci
 from swagger_server.models.job_response import JobResponse  # noqa: E501
 from swagger_server.requets_db.models.vrsta import (Job)
 from threading import Thread
-from swagger_server.utils import cl_utils
+from swagger_server.utils import cl_utils, db_utils
 from swagger_server.utils import txt_utils
 from werkzeug.datastructures import FileStorage
 import threading
 import time
 
-CLASSLA_CONCURANCE_LIMIT = 3
-DOC2TEXT_CONCURANCE_LIMIT = 3
-ATEAPI_CONCURANCE_LIMIT = 2
+CLASSLA_SMALL_SIZE_LIMIT = 500 * 1e3  # 500 KB, aka.: 500 * 10^3
 
-classla_sem = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT)
-doc2text_sem = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT)
-ateapi_sem = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT)
+DOC2TEXT_SMALL_SIZE_LIMIT = 10 * 1e6  # 10 MB
+
+ATEAPI_SMALL_SIZE_LIMIT = 2 * 1e6  # 2 MB
+
+########################################
+CLASSLA_CONCURANCE_LIMIT_BIG = 2
+CLASSLA_CONCURANCE_LIMIT_SMALL = 2
+
+DOC2TEXT_CONCURANCE_LIMIT_BIG = 2
+DOC2TEXT_CONCURANCE_LIMIT_SMALL = 2
+
+ATEAPI_CONCURANCE_LIMIT_BIG = 2
+ATEAPI_CONCURANCE_LIMIT_SMALL = 2
+
+IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT = 4
+########################################
+classla_sem_big = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT_BIG)
+classla_sem_small = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT_SMALL)
+
+doc2text_sem_big = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT_BIG)
+doc2text_sem_small = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT_SMALL)
+
+ateapi_sem_big = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT_BIG)
+ateapi_sem_small = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT_SMALL)
+
+izluscipoiskanju_sem = threading.Semaphore(IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT)
+
+running_threads = {}  # <--- dict currently not used, was trying to figure out how to cancel workers mid execution,
+
+
+# no luck with that yet
 
 
 def delete_job(job_id):  # noqa: E501
-    """Izbriše job
+    """Izbriše job
 
      # noqa: E501
 
@@ -36,7 +62,14 @@ def delete_job(job_id):  # noqa: E501
 
     :rtype: str
     """
-    return 'Endpoint currently disabled'
+    try:
+        job = Job.get_by_id(job_id)
+        if job.started_on is not None and job.finished_on is None:
+            return Response("Cancelling ongoing jobs currently not implemented.", 400)
+        job.delete_instance()
+        return f"Job with the ID {job_id} was removed."
+    except peewee.DoesNotExist:
+        return Response("Job with this ID does not exist", 404)
 
 
 def get_job_status(job_id):  # noqa: E501
@@ -56,8 +89,19 @@ def get_job_status(job_id):  # noqa: E501
         if job.started_on is not None and job.finished_on is None:
             return JobResponse(job_status="currently processing", created_on=job.created_on,
                                started_on=job.started_on), 200
-        if job.started_on is not None and job.finished_on is not None:
-            return JobResponse(job_status="finished processing", created_on=job.created_on, started_on=job.started_on,
+        if job.started_on is not None and job.finished_on is not None and not job.job_output.startswith("ERROR -"):
+            res = job.job_output
+            if job.job_type in [4, 5]:
+                try:
+                    res = json.loads(job.job_output)
+                except:
+                    pass
+            return JobResponse(job_status="finished processing (OK)", created_on=job.created_on,
+                               started_on=job.started_on,
+                               finished_on=job.finished_on, job_result=res), 200
+        if job.started_on is not None and job.finished_on is not None and job.job_output.startswith("ERROR -"):
+            return JobResponse(job_status="finished processing (ERROR)", created_on=job.created_on,
+                               started_on=job.started_on,
                                finished_on=job.finished_on, job_result=job.job_output), 200
     except peewee.DoesNotExist:
         return "Job with this ID does not exist", 404
@@ -75,22 +119,65 @@ def clear_up_unfinished_jobs():
 
 
 async def try_do_jobs():
-    with cf.ThreadPoolExecutor(max_workers=3) as ex:
+    with cf.ThreadPoolExecutor(max_workers=4) as ex:
         ex.submit(try_do_jobs_classla)
         ex.submit(try_do_jobs_doc2text)
         ex.submit(try_do_jobs_ateapi)
+        ex.submit(try_do_jobs_izluscipoiskanju)
 
 
-### Job looping
+# to pe je pod ex.submit
+# sub = ex.submit(execute_ateapi_job, job)
+# running_threads[job.id] = sub
+# time.sleep(1)
+# running_threads[job.id]
+# preveri ce je done: ```running_threads[job.id].done()``` (vrne true false)
+
+# was_canceled = running_threads[job.id].cancel()
+# Todo: Mogoce kaksna druga opcija? Ampak verjetno ne, ne vidim (še?) kak prekicat ONGOING job
+# To zgoraj preklice samo job, ki se se ni zacel, kar pa ni za ta use case uporabno.
+
+
+### Picking jobs for looping
+def try_do_jobs_izluscipoiskanju():
+    while True:
+        try:
+            if izluscipoiskanju_sem._value > 0:
+                unfinished_jobs = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 5) \
+                    .limit(izluscipoiskanju_sem._value)
+                with cf.ThreadPoolExecutor(max_workers=IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT) as ex:
+                    [ex.submit(execute_izluscipoiskanju_job, job, izluscipoiskanju_sem) for job in unfinished_jobs]
+        except Exception as e:
+            print(f"Exception in try_do_jobs_izluscipoiskanju")
+            traceback.print_exc()
+        finally:
+            time.sleep(3)
+
+
+### Picking jobs for looping
 def try_do_jobs_ateapi():
     while True:
         try:
-            if ateapi_sem._value > 0:
-                unfinished_jobs = Job.select() \
-                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 4) \
-                    .limit(ateapi_sem._value)
-                with cf.ThreadPoolExecutor(max_workers=ATEAPI_CONCURANCE_LIMIT) as ex:
-                    [ex.submit(execute_ateapi_job, job) for job in unfinished_jobs]
+            unfinished_jobs_big = []
+            unfinished_jobs_small = []
+            if ateapi_sem_big._value > 0:
+                unfinished_jobs_big.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
+                                                              Job.job_type == 4,
+                                                              Job.input_size > ATEAPI_SMALL_SIZE_LIMIT) \
+                                           .limit(ateapi_sem_big._value))
+            if ateapi_sem_small._value > 0:
+                unfinished_jobs_small.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
+                                                                Job.job_type == 4,
+                                                                Job.input_size <= ATEAPI_SMALL_SIZE_LIMIT) \
+                                             .limit(ateapi_sem_small._value))
+
+            if len(unfinished_jobs_big) > 0:
+                with cf.ThreadPoolExecutor(max_workers=ATEAPI_CONCURANCE_LIMIT_BIG) as ex:
+                    [ex.submit(execute_ateapi_job, job, ateapi_sem_big) for job in unfinished_jobs_big]
+            if len(unfinished_jobs_small) > 0:
+                with cf.ThreadPoolExecutor(max_workers=ATEAPI_CONCURANCE_LIMIT_SMALL) as ex:
+                    [ex.submit(execute_ateapi_job, job, ateapi_sem_small) for job in unfinished_jobs_small]
         except Exception as e:
             print(f"Exception in try_do_jobs_ateapi")
             traceback.print_exc()
@@ -98,27 +185,51 @@ def try_do_jobs_ateapi():
             time.sleep(3)
 
 
-### Job looping
+### Picking jobs for looping
 def try_do_jobs_classla():
     time.sleep(15)  # wait for tokenizers to load for classla ...
     while True:
         try:
-            if cl_utils.nlp_loaded:
-                if classla_sem._value > 0:
-                    unfinished_jobs_txt = Job.select() \
-                        .where(Job.finished_on.is_null(), Job.job_type == 2,
-                               Job.input_file.is_null(False)) \
-                        .limit(classla_sem._value)
+            if not cl_utils.nlp_loaded:
+                raise Exception("NLP utils not loaded yet.")
 
-                    unfinished_jobs_no_txt = Job.select() \
-                        .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
-                               Job.input_file.is_null()) \
-                        .limit(classla_sem._value)
+            unfinished_jobs_big = []
+            unfinished_jobs_small = []
 
-                    unfinished_jobs = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
-                    unfinished_jobs = unfinished_jobs[:classla_sem._value]
-                    with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT) as ex:
-                        [ex.submit(execute_classla_job, job) for job in unfinished_jobs]
+            if classla_sem_big._value > 0:
+                unfinished_jobs_txt = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.job_type == 2,
+                           Job.input_file.is_null(False), Job.input_size > CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_big._value)
+
+                unfinished_jobs_no_txt = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
+                           Job.input_file.is_null(), Job.input_size > CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_big._value)
+
+                unfinished_jobs_big = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
+                unfinished_jobs_big = unfinished_jobs_big[:classla_sem_big._value]
+
+            if classla_sem_small._value > 0:
+                unfinished_jobs_txt = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.job_type == 2,
+                           Job.input_file.is_null(False), Job.input_size <= CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_small._value)
+
+                unfinished_jobs_no_txt = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
+                           Job.input_file.is_null(), Job.input_size <= CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_small._value)
+
+                unfinished_jobs_small = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
+                unfinished_jobs_small = unfinished_jobs_small[:classla_sem_small._value]
+
+            if len(unfinished_jobs_big) > 0:
+                with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT_BIG) as ex:
+                    [ex.submit(execute_classla_job, job, doc2text_sem_big) for job in unfinished_jobs_big]
+            if len(unfinished_jobs_small) > 0:
+                with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT_SMALL) as ex:
+                    [ex.submit(execute_classla_job, job, doc2text_sem_small) for job in unfinished_jobs_small]
 
         except Exception as e:
             print(f"Exception in try_do_jobs_classla")
@@ -127,16 +238,29 @@ def try_do_jobs_classla():
             time.sleep(3)
 
 
-### Job looping
+### Picking jobs for looping
 def try_do_jobs_doc2text():
     while True:
         try:
-            if doc2text_sem._value > 0:
-                unfinished_jobs = Job.select() \
-                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type << [1, 12, 3, 32]) \
-                    .limit(doc2text_sem._value)
-                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT) as ex:
-                    [ex.submit(execute_doc2text_job, job) for job in unfinished_jobs]
+            unfinished_jobs_big = []
+            unfinished_jobs_small = []
+            if doc2text_sem_big._value > 0:
+                unfinished_jobs_big.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
+                                                              Job.job_type << [1, 12, 3, 32],
+                                                              Job.input_size > DOC2TEXT_SMALL_SIZE_LIMIT) \
+                                           .limit(doc2text_sem_big._value))
+            if doc2text_sem_small._value > 0:
+                unfinished_jobs_small.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
+                                                                Job.job_type << [1, 12, 3, 32],
+                                                                Job.input_size <= DOC2TEXT_SMALL_SIZE_LIMIT) \
+                                             .limit(doc2text_sem_small._value))
+
+            if len(unfinished_jobs_big) > 0:
+                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT_BIG) as ex:
+                    [ex.submit(execute_doc2text_job, job, doc2text_sem_big) for job in unfinished_jobs_big]
+            if len(unfinished_jobs_small) > 0:
+                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT_SMALL) as ex:
+                    [ex.submit(execute_doc2text_job, job, doc2text_sem_small) for job in unfinished_jobs_small]
         except Exception as e:
             print(f"Exception in try_do_jobs_doc2text")
             traceback.print_exc()
@@ -144,13 +268,14 @@ def try_do_jobs_doc2text():
             time.sleep(3)
 
 
-async def prep_jobs(tasks):
-    await asyncio.gather(*tasks)
+# async def prep_jobs(tasks):
+#     await asyncio.gather(*tasks)
 
 
-def execute_doc2text_job(job: Job):
+####### JOB EXECUTION LOGIC
+def execute_doc2text_job(job: Job, sem: threading.Semaphore):
     try:
-        doc2text_sem.acquire()
+        sem.acquire()
         del_file = False
         job.started_on = datetime.datetime.utcnow()
         job.save()
@@ -158,7 +283,7 @@ def execute_doc2text_job(job: Job):
         tmp_file_path = job.input_file
         if not os.path.exists(tmp_file_path):
             job.finished_on = datetime.datetime.utcnow()
-            job.job_output = "ERROR - Temporary file went missing, couldn't properly finish job"
+            job.job_output = "ERROR - Temporary file went missing, couldn't properly finish job. Please try executing the job again."
             job.save()
             return
 
@@ -190,34 +315,85 @@ def execute_doc2text_job(job: Job):
         job.started_on = None
         job.save()
     finally:
-        doc2text_sem.release()
+        sem.release()
 
 
-def execute_classla_job(job: Job):
+####### JOB EXECUTION LOGIC
+def execute_classla_job(job: Job, sem: threading.Semaphore):
     try:
-        classla_sem.acquire()
+        sem.acquire()
         job.started_on = datetime.datetime.utcnow()
         job.save()
-        conllu, _ = cl_utils.raw_text_to_conllu(job.job_input)
+        conllu, status = cl_utils.raw_text_to_conllu(job.job_input)
+        if status != 200:
+            conllu = f'ERROR - {conllu}'
         job.job_output = conllu
         job.finished_on = datetime.datetime.utcnow()
         job.save()
+    except:
+        job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+        print(f"Unexpected error at job {job.id}")
     finally:
-        classla_sem.release()
+        sem.release()
 
 
-def execute_ateapi_job(job: Job):
+####### JOB EXECUTION LOGIC
+def execute_ateapi_job(job: Job, sem: threading.Semaphore):
     try:
-        ateapi_sem.acquire()
+        sem.acquire()
         job.started_on = datetime.datetime.utcnow()
         job.save()
         info = json.loads(job.job_input)
-        ret_json, _ = do_izlusci(info['conllus'], info['prepovedane_besede'])
-        job.job_output = ret_json
+        _res = do_izlusci(info['conllus'], info['prepovedane_besede'],info['definicije'])
+        if type(_res) is tuple:
+            ret_json = _res[0]
+        else:
+            try:
+                if type(_res.response) is dict:
+                    ret_json = str(_res.response)
+                else:
+                    try:
+                        ret_json = _res.response[0].decode('utf-8')
+                    except:
+                        ret_json = "ERROR - Unknown exception."
+
+                if _res.status_code != 200:
+                    ret_json = f'ERROR - {ret_json}'
+            except:
+                ret_json = "ERROR - Unknown exception."
+        job.job_output = json.dumps(ret_json, ensure_ascii=False)
         job.finished_on = datetime.datetime.utcnow()
         job.save()
+    except:
+        job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+        print(f"Unexpected error at job {job.id}")
     finally:
-        ateapi_sem.release()
+        sem.release()
+
+
+####### JOB EXECUTION LOGIC
+def execute_izluscipoiskanju_job(job: Job, sem: threading.Semaphore):
+    try:
+        sem.acquire()
+        job.started_on = datetime.datetime.utcnow()
+        job.save()
+        info = json.loads(job.job_input)
+        terKand = db_utils.vrni_oss_terminoloske_kandidate(info['leta'], info['vrste'], info['kljucne_besede'],
+                                                           info['prepovedane_besede'], info['udk'],info['definicije'])
+        job.job_output = json.dumps(terKand, ensure_ascii=False)
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+    except:
+        job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+        print(f"Unexpected error at job {job.id}")
+    finally:
+        sem.release()
 
 
 clear_up_unfinished_jobs()

swagger_server/controllers/oss_controller.py

@@ -1,55 +1,88 @@
+import json
+
+import connexion
+
+from swagger_server.requets_db.models.vrsta import JobManager
 from swagger_server.utils import db_utils
 from swagger_server import util
 from flask import send_file
 
 
-def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
-    """Vrne seznam CoNNL-U-jev glede na iskalne pogoje
+def get_conllus(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
+    """"Vrne seznam CoNNL-U-jev glede na iskalne pogoje
 
      # noqa: E501
 
-    :param leta: 
+    :param leta:
     :type leta: List[int]
-    :param vrste: 
-    :type vrste: List[str]
-    :param kljucnebesede: 
-    :type kljucnebesede: List[str]
-    :param cerifpodrocja: 
-    :type cerifpodrocja: List[int]
+    :param vrste:
+    :type vrste: List[int]
+    :param kljucne_besede:
+    :type kljucne_besede: List[str]
+    :param udk:
+    :type udk: List[str]
 
     :rtype: List[str]
     """
-    if not kljucnebesede:
+    if not kljucne_besede:
         return "Manjkajo kljucne besede", 400
-    #zaenkrat ne potrebujemo te storitve
-    files = db_utils.get_files_by_udc(kljucnebesede)
+    files = db_utils.get_files_by_udc(kljucne_besede)
     if not files:
         return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
     return ' '.join(files), 200
 
 
-def get_extracted_words(leta=None, vrste=None, kljucnebesede=None, udk=None):  # noqa: E501
-    """Vrne terminloške kandidate glede na 
+def get_extracted_words(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None, udk=None,definicije=False):  # noqa: E501
+    """Vrne terminloške kandidate glede na ... (sync)
 
      # noqa: E501
 
-    :param leta: 
+    :param leta:
     :type leta: List[int]
-    :param vrste: 
-    :type vrste: List[str]
-    :param kljucnebesede: 
-    :type kljucnebesede: List[str]
-    :param cerifpodrocja: 
-    :type cerifpodrocja: List[int]
+    :param vrste:
+    :type vrste: List[int]
+    :param kljucne_besede:
+    :type kljucne_besede: List[str]
+    :param prepovedane_besede:
+    :type prepovedane_besede: List[str]
+    :param udk:
+    :type udk: List[str]
 
     :rtype: List[TerminoloskiKandidat]
     """
-    files = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk)
-    return files, 200
+    terKand = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucne_besede, prepovedane_besede, udk,definicije)
+    return terKand, 200
 
 
+def get_extracted_words_async(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None,
+                              udk=None,definicije=False):  # noqa: E501
+    """Vrne terminloške kandidate glede na ... (async)
 
-def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
+     # noqa: E501
+
+    :param leta:
+    :type leta: List[int]
+    :param vrste:
+    :type vrste: List[int]
+    :param kljucne_besede:
+    :type kljucne_besede: List[str]
+    :param prepovedane_besede:
+    :type prepovedane_besede: List[str]
+    :param udk:
+    :type udk: List[str]
+
+    :rtype: str
+    """
+    job, is_old_job = JobManager.create_job(5, json.dumps(
+        {'leta': leta, 'vrste': vrste, 'kljucne_besede': kljucne_besede, 'prepovedane_besede': prepovedane_besede,
+         'udk': udk,'definicije':definicije}))
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+    return ret, 200
+
+
+def get_files(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
     """Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje
 
      # noqa: E501
@@ -57,66 +90,64 @@ def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
     :param leta: 
     :type leta: List[int]
     :param vrste: 
-    :type vrste: List[str]
-    :param kljucnebesede: 
-    :type kljucnebesede: List[str]
-    :param cerifpodrocja: 
-    :type cerifpodrocja: List[int]
+    :type vrste: List[int]
+    :param kljucne_besede:
+    :type kljucne_besede: List[str]
+    :param udk:
+    :type udk: List[str]
 
-    :rtype: List[List[bytearray]]
+    :rtype: str
     """
-    if not kljucnebesede:
+    if not kljucne_besede:
         return "Manjkajo kljucne besede", 400
-    #zaenkrat ne potrebujemo te storitve
-    files = db_utils.get_files_by_udc(kljucnebesede)
+    files = db_utils.get_files_by_udc(kljucne_besede)
     if not files:
         return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
     return ' '.join(files), 200
 
 
-def get_number_texts(leta=None, vrste=None, kljucnebesede=None, udk=None):  # noqa: E501
-    """Vrne število besedil glede na iskalne pogoje
+def get_number_texts(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
+    """Vrne število besedil glede na iskalne pogoje
 
      # noqa: E501
 
     :param leta: 
     :type leta: List[int]
     :param vrste: 
-    :type vrste: List[str]
-    :param kljucnebesede: 
-    :type kljucnebesede: List[str]
-    :param cerifpodrocja: 
-    :type udc: List[int]
+    :type vrste: List[int]
+    :param kljucne_besede:
+    :type kljucne_besede: List[str]
+    :param udk:
+    :type udk: List[str]
 
     :rtype: int
     """
-    #if not kljucnebesede:
+    # if not kljucnebesede:
     #    return "Manjkajo kljucne besede", 400
 
-    files = db_utils.vrni_oss_dokumente(leta, vrste, kljucnebesede, udk)
+    files = db_utils.vrni_oss_dokumente(leta, vrste, kljucne_besede, udk)
     return len(files), 200
 
 
-def get_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
+def get_texts(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
     """Vrne seznam besedil glede na iskalne pogoje
 
      # noqa: E501
 
-    :param leta: 
+    :param leta:
     :type leta: List[int]
-    :param vrste: 
-    :type vrste: List[str]
-    :param kljucnebesede: 
-    :type kljucnebesede: List[str]
-    :param cerifpodrocja: 
-    :type cerifpodrocja: List[int]
+    :param vrste:
+    :type vrste: List[int]
+    :param kljucne_besede:
+    :type kljucne_besede: List[str]
+    :param udk:
+    :type udk: List[str]
 
     :rtype: List[str]
     """
-    if not kljucnebesede:
+    if not kljucne_besede:
         return "Manjkajo kljucne besede", 400
-    #zaenkrat ne potrebujemo te storitve
-    files = db_utils.get_files_by_udc(kljucnebesede)
+    files = db_utils.get_files_by_udc(kljucne_besede)
     if not files:
         return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
     return ' '.join(files), 200
@@ -166,7 +197,7 @@ def oss_besedilo_po_id_get(file_id):  # noqa: E501
     """
     try:
         f = util.get_original_file_path_by_id(file_id)
-        print(f) # for debugging purposes on the server, delete this later
+        print(f)  # for debugging purposes on the server, delete this later
         return send_file(util.get_original_file_path_by_id(file_id), download_name=f'{file_id}.xml')
     except FileNotFoundError as e:
         return "The file with this ID doesn't exist.", 404

swagger_server/models/izlusci_async_body.py

@@ -14,7 +14,7 @@ class IzlusciAsyncBody(Model):
 
     Do not edit the class manually.
     """
-    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
+    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None, definicje: bool=False):  # noqa: E501
         """IzlusciAsyncBody - a model defined in Swagger
 
         :param conllus: The conllus of this IzlusciAsyncBody.  # noqa: E501
@@ -24,15 +24,18 @@ class IzlusciAsyncBody(Model):
         """
         self.swagger_types = {
             'conllus': List[str],
-            'prepovedane_besede': List[str]
+            'prepovedane_besede': List[str],
+            'definicije':bool
         }
 
         self.attribute_map = {
             'conllus': 'conllus',
-            'prepovedane_besede': 'prepovedaneBesede'
+            'prepovedane_besede': 'prepovedaneBesede',
+            'definicije':'definicije'
         }
         self._conllus = conllus
         self._prepovedane_besede = prepovedane_besede
+        self._definicije=definicje
 
     @classmethod
     def from_dict(cls, dikt) -> 'IzlusciAsyncBody':
@@ -45,6 +48,30 @@ class IzlusciAsyncBody(Model):
         """
         return util.deserialize_model(dikt, cls)
 
+
+
+    @property
+    def definicije(self) -> bool:
+        """Gets the conllus of this IzlusciSyncBody.
+
+
+        :return: The conllus of this IzlusciSyncBody.
+        :rtype: List[str]
+        """
+        return self._definicije
+
+    @definicije.setter
+    def definicije(self, definicije: bool):
+        """Sets the conllus of this IzlusciSyncBody.
+
+
+        :param conllus: The conllus of this IzlusciSyncBody.
+        :type conllus: List[str]
+        """
+
+        self._definicije = definicije
+
+
     @property
     def conllus(self) -> List[str]:
         """Gets the conllus of this IzlusciAsyncBody.

swagger_server/models/izlusci_sync_body.py

@@ -14,7 +14,7 @@ class IzlusciSyncBody(Model):
 
     Do not edit the class manually.
     """
-    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
+    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None, definicje: bool=False):  # noqa: E501
         """IzlusciSyncBody - a model defined in Swagger
 
         :param conllus: The conllus of this IzlusciSyncBody.  # noqa: E501
@@ -24,15 +24,18 @@ class IzlusciSyncBody(Model):
         """
         self.swagger_types = {
             'conllus': List[str],
-            'prepovedane_besede': List[str]
+            'prepovedane_besede': List[str],
+            'definicije':bool
         }
 
         self.attribute_map = {
             'conllus': 'conllus',
-            'prepovedane_besede': 'prepovedaneBesede'
+            'prepovedane_besede': 'prepovedaneBesede',
+            'definicije':'definicije'
         }
         self._conllus = conllus
         self._prepovedane_besede = prepovedane_besede
+        self._definicije = definicje
 
     @classmethod
     def from_dict(cls, dikt) -> 'IzlusciSyncBody':
@@ -66,6 +69,27 @@ class IzlusciSyncBody(Model):
 
         self._conllus = conllus
 
+    @property
+    def definicije(self) -> bool:
+        """Gets the conllus of this IzlusciSyncBody.
+
+
+        :return: The conllus of this IzlusciSyncBody.
+        :rtype: List[str]
+        """
+        return self._definicije
+
+    @definicije.setter
+    def definicije(self, definicije: bool):
+        """Sets the conllus of this IzlusciSyncBody.
+
+
+        :param conllus: The conllus of this IzlusciSyncBody.
+        :type conllus: List[str]
+        """
+
+        self._definicije = definicije
+
     @property
     def prepovedane_besede(self) -> List[str]:
         """Gets the prepovedane_besede of this IzlusciSyncBody.

swagger_server/models/terminoloski_kandidat.py

@@ -14,7 +14,7 @@ class TerminoloskiKandidat(Model):
 
     Do not edit the class manually.
     """
-    def __init__(self, kandidat: str=None, kanonicnaoblika: str=None, po_soznake: str=None, nosilnautez: float=None, podporneutezi: List[float]=None, pogostostpojavljanja: List[int]=None):  # noqa: E501
+    def __init__(self, kandidat: str=None,  definicija: str=None, kanonicnaoblika: str=None, po_soznake: str=None, nosilnautez: float=None, podporneutezi: List[float]=None, pogostostpojavljanja: List[int]=None):  # noqa: E501
         """TerminoloskiKandidat - a model defined in Swagger
 
         :param kandidat: The kandidat of this TerminoloskiKandidat.  # noqa: E501
@@ -32,6 +32,7 @@ class TerminoloskiKandidat(Model):
         """
         self.swagger_types = {
             'kandidat': str,
+            'definicja': str,
             'kanonicnaoblika': str,
             'po_soznake': str,
             'nosilnautez': float,
@@ -41,6 +42,7 @@ class TerminoloskiKandidat(Model):
 
         self.attribute_map = {
             'kandidat': 'kandidat',
+            'definicija': 'definicija',
             'kanonicnaoblika': 'kanonicnaoblika',
             'po_soznake': 'POSoznake',
             'nosilnautez': 'nosilnautez',
@@ -48,6 +50,7 @@ class TerminoloskiKandidat(Model):
             'pogostostpojavljanja': 'pogostostpojavljanja'
         }
         self._kandidat = kandidat
+        self._definicija = definicija
         self._kanonicnaoblika = kanonicnaoblika
         self._po_soznake = po_soznake
         self._nosilnautez = nosilnautez
@@ -86,6 +89,27 @@ class TerminoloskiKandidat(Model):
 
         self._kandidat = kandidat
 
+    @property
+    def definicija(self) -> str:
+        """Gets the definicija of this TerminoloskiKandidat.
+
+
+        :return: The definicija of this TerminoloskiKandidat.
+        :rtype: str
+        """
+        return self._definicija
+
+    @definicija.setter
+    def definicija(self, definicija: str):
+        """Sets the kandidat of this TerminoloskiKandidat.
+
+
+        :param kandidat: The kandidat of this TerminoloskiKandidat.
+        :type kandidat: str
+        """
+
+        self._definicija = definicija
+
     @property
     def kanonicnaoblika(self) -> str:
         """Gets the kanonicnaoblika of this TerminoloskiKandidat.

swagger_server/swagger/swagger.yaml

@@ -343,7 +343,7 @@ paths:
     get:
       tags:
       - oss
-      summary: 'Vrne terminloške kandidate glede na '
+      summary: 'Vrne terminloške kandidate glede na iskalne pogoje'
       operationId: get_extracted_words
       parameters:
       - name: leta
@@ -384,6 +384,12 @@ paths:
           type: array
           items:
             type: string
+      - name: definicije
+        in: query
+        required: false
+        style: form
+        schema:
+          type: boolean
       responses:
         "200":
           description: OK
@@ -651,6 +657,8 @@ components:
           type: string
         kanonicnaoblika:
           type: string
+        definition:
+          type: string
         POSoznake:
           type: string
         nosilnautez:
@@ -753,6 +761,8 @@ components:
           type: array
           items:
             type: string
+        definicije:
+          type: boolean
     izlusciAsync_body:
       type: object
       properties:
@@ -764,6 +774,8 @@ components:
           type: array
           items:
             type: string
+        definicije:
+          type: boolean
     datotekaVBesediloSync_body:
       required:
       - file

swagger_server/utils/db_utils.py

@@ -3,12 +3,13 @@ import os
 import sys
 import requests
 import json
+import time
 
 
 database_info = {
     'database': os.getenv("MDB_DATABASE", "oss"),
     'host': os.getenv("MDB_HOST", "localhost"),
-    'port': int(os.getenv("PORT", 3306)) ,
+    'port': int(os.getenv("MDB_PORT", 3306)) ,
     'user': os.getenv("MDB_USER", "root"),
     'password': os.getenv("MDB_PASSWORD", "root"),
 }
@@ -96,13 +97,13 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
     return ret
 
 
-def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
+def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_besede, udk,definicije=False):
     ret = []
   
     try:
         print(database_info)
         conn = mariadb.connect(**database_info)
-        cur = conn.cursor()
+        cur = conn.cursor(dictionary=True)
         
         
         
@@ -142,7 +143,7 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
         print(sql)
         print(params)
 
-        sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf, sum(tf) as tf from (
+        sqltk=f"""Select ngram,upos,convert(avg(tfidf),FLOAT) as tfidf, convert(sum(tf),INT) as tf from (
                     SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf
                     FROM ngrams_upos_tf tf, documents d,
                         (
@@ -160,34 +161,55 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
                     order by tfidf desc
                     limit 1000;"""
                     #
+        #sqltk=f"""select ngram,upos,convert(1.0,float) as tfidf,%s as tf from ngrams_upos_tf limit 10;"""
+
         print (sqltk)
-        
+        #še prepovedane besede ven
+        start_time = time.time()
         cur.execute(sqltk,params)
         terms=cur.fetchall()
+        print("Čas poizbedbe je %.2f sekund" % (time.time() - start_time))
+        print (terms);
         #ret = list(cur)
         can = {'forms':[
-            ngram 
+            ngram["ngram"]
             for ngram in terms
             ]
         }
-        res = requests.post(ATEapi_endpoint, json=can)
-        data = res.json().canonical_forms
+        print (can);
+        res = requests.post(canonapi_endpoint, json=can)
+        
+        data = res.json()
+        print (data);
+        print (data.get("canonical_forms"));
+        print (terms);
+        print(zip(data.get("canonical_forms"),terms))
+
+
+
 
         ret = {'terminoloski_kandidati': [
             {
-                'POSoznake': x.upos,
-                'kandidat': x.ngram,  # more to bit lemma al terms?
+                'POSoznake': x.get("upos"),
+                'kandidat': x.get("ngram"),  # more to bit lemma al terms?
+                'definicija': None,
                 'kanonicnaoblika': d,
-                'ranking': x.tfidf,
+                'ranking': x.get('tfidf'),
                 'podporneutezi': [
                     0.0,  # ????????
                     0.0  # ??????
                 ],
-                'pogostostpojavljanja': [tf, 0]  # ???????
+                'pogostostpojavljanja': [x.get('tf'), 0]  # ???????
             }
-            for d,x in zip(data,cur) 
+            for (d,x) in zip(data.get("canonical_forms"),terms) 
         ]}
-            
+    
+        #if definicije
+        #idi z variablo sql po id-je dokumentov, preberi conlluje iz diska
+        #naredi en vlki conllu
+        #pokliči metodo
+
+
     except mariadb.Error as e:
         print(f"Error connecting to MariaDB Platform: {e}")
     

swagger_server/utils/txt_utils.py

@@ -7,15 +7,51 @@ import xml.etree.ElementTree as ET
 from PyPDF2 import PdfReader
 from swagger_server.utils import cl_utils
 import cv2
+import json 
 import numpy as np
 import magic
 import re
 #to še mora v env 
 tika_server = "http://tika2:9999/tika"
+definicije_endpoint = "http://definitions:5000/DefExAPI/definition_sentence_extraction"
+
 
 # endpoint below to be used only for development purposes (don't need to run docker)
 # tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
 
+#rabim conllu -> file
+# lematizirane besede ->lematized terms
+#file je touple z vsebino
+#torej ('temp_1.conllu', fp, 'application/octet-stream')
+def extract_definition_sentences(filePath="", lemmatized_terms=[]):
+
+    try:
+        fp = open(filePath, 'rb')
+        can = {'lemmatized_terms':[
+            w["kandidat"]
+            for w in lemmatized_terms["terminoloski_kandidati"]
+            ]
+        }
+        terms=json.dumps(can)
+        headers = {'accept': 'application/json'}
+        #,'Content-Type': 'multipart/form-data'}
+
+        res = requests.post(definicije_endpoint,headers=headers, files={'terms': (None, terms),'conllu_file': fp})
+        data = res.json()
+        print(data);
+        for i in lemmatized_terms["terminoloski_kandidati"]:
+            i["definicija"]=next((x["definicija"] for x in data["definition_candidates"] if x["term"] == i["kandidat"]), None)   
+        #apend to lematized terms
+        print(lemmatized_terms)
+    except Exception as e: print(e)
+    finally:
+        fp.close();
+
+        
+
+    return lemmatized_terms
+
+
 
 def extract_text_prepResp(file, content_type=""):
     content_type = file.content_type