Improved extract method to work properly (sync)

rsdo5.json

@@ -265,13 +265,13 @@
                 }
             }
         },
-        "/izlusci": {
+        "/izlusciSync": {
             "post": {
                 "tags": [
                     "extract"
                 ],
-                "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki",
+                "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki [sihrono, rezultat v sami zahtevi]",
-                "operationId": "getCandidates",
+                "operationId": "getCandidates_sync",
                 "requestBody": {
                     "content": {
                         "application/json": {
@@ -313,6 +313,51 @@
                 }
             }
         },
+        "/izlusciAsync": {
+            "post": {
+                "tags": [
+                    "extract"
+                ],
+                "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki [asinhrono, ustvari novi job]",
+                "operationId": "getCandidates_async",
+                "requestBody": {
+                    "content": {
+                        "application/json": {
+                            "schema": {
+                                "type": "object",
+                                "properties": {
+                                    "conllus": {
+                                        "type": "array",
+                                        "items": {
+                                            "type": "string"
+                                        }
+                                    },
+                                    "prepovedaneBesede": {
+                                        "type": "array",
+                                        "items": {
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    "required": true
+                },
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "content": {
+                            "*/*": {
+                                "schema": {
+                                    "type": "string"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
         "/datotekaVBesediloSync": {
             "post": {
                 "tags": [

swagger_server/controllers/extract_controller.py

@@ -1,18 +1,74 @@
+import codecs
+import os
+
 import connexion
 import json
 from pathlib import Path
-from swagger_server.models.izlusci_body import IzlusciBody  # noqa: E501
+from swagger_server.models.izlusci_async_body import IzlusciAsyncBody  # noqa: E501
+from swagger_server.models.izlusci_sync_body import IzlusciSyncBody  # noqa: E501
 from swagger_server.utils import cl_utils
+from swagger_server.util import get_random_filename, create_random_file_in_tmp_folder
 import requests
+from werkzeug.utils import secure_filename
 
-# ATEapi_endpoint = "http://localhost:5000/predict"
+ATEapi_endpoint = "http://localhost:5000/predict"
 
 
-ATEapi_endpoint = "http://ate-api:5000/predict"
+# ATEapi_endpoint = "http://ate-api:5000/predict"
 
 
-def get_candidates(body):  # noqa: E501
+def get_candidates_async(body):  # noqa: E501
-    """Izlusci terminološke kandidate iz seznama besedil v conllu obliki
+    """Izlusci terminološke kandidate iz seznama besedil v conllu obliki [asinhrono, ustvari novi job]
+
+     # noqa: E501
+
+    :param body:
+    :type body: dict | bytes
+
+    :rtype: str
+    """
+    if connexion.request.is_json:
+        body = IzlusciAsyncBody.from_dict(connexion.request.get_json())  # noqa: E501
+    return 'do some magic!'
+
+
+def do_izlusci(conllus, prepovedane_besede):
+    tmp_file_path = ""
+    try:
+        big_conllu = cl_utils.multipla_conllus_to_one_from_conllus_arr(conllus)
+        tmp_file_path = create_random_file_in_tmp_folder(big_conllu, ".conllu")
+        fp = open(tmp_file_path, 'rb')
+        try:
+            files = [
+                ('file', ('temp_1.conllu', fp, 'application/octet-stream'))
+            ]
+            res = requests.post(ATEapi_endpoint, files=files)
+            data = json.loads(res.text)
+        finally:
+            fp.close()
+            os.remove(tmp_file_path)
+
+        ret = {'terminoloski_kandidati': [
+            {
+                'POSoznake': tk['msd'],
+                'kandidat': tk['terms'],  # more to bit lemma al terms?
+                'kanonicnaoblika': tk['canonical'],
+                'ranking': tk['ranking'],
+                'podporneutezi': [
+                    0.0,  # ????????
+                    0.0  # ??????
+                ],
+                'pogostostpojavljanja': [0, 0]  # ???????
+            }
+            for tk in data if tk['terms'] not in prepovedane_besede
+        ]}
+        return ret, 200
+    except Exception as e:
+        return str(e), 500
+
+
+def get_candidates_sync(body):  # noqa: E501
+    """Izlusci terminološke kandidate iz seznama besedil v conllu obliki [sihrono, rezultat v sami zahtevi]
 
      # noqa: E501
 
@@ -22,93 +78,6 @@ def get_candidates(body):  # noqa: E501
     :rtype: List[TerminoloskiKandidat]
     """
     if connexion.request.is_json:
-        body = IzlusciBody.from_dict(connexion.request.get_json())  # noqa: E501
+        body = IzlusciSyncBody.from_dict(connexion.request.get_json())  # noqa: E501
 
-    # Todo: What is "conllus" in body input anyway??
+    return do_izlusci(body.conllus, body.prepovedane_besede)
-    # Todo: Kaj je s prepovedanimi besedami?
-    # file_ids = [10000, 10001, 10002, 10003]
-    # big_conllu = cl_utils.multipla_conllus_to_one_from_file_ids(file_ids)
-
-    # txt = Path('../ATEapi/temp_1.conllu').read_text('utf-8')  # LOCAL ONLY
-
-    # # Todo: does this close the file after the request?
-    # files = [
-    #     ('file', ('temp_1.conllu', open('../ATEapi/temp_1.conllu', 'rb'),
-    #               'application/octet-stream'))
-    # ]
-    # res = requests.post(ATEapi_endpoint, files=files)
-    # # res.status_code
-    # # res.text
-    #
-    # return res.text, res.status_code
-    #
-    # example_data = ""
-    # with open("C:\\Users\\Kiki\\Desktop\\Untitled-2.json", "r", encoding='utf-8') as f:
-    #     example_data = json.loads(f.read())
-    #
-    # d = 0
-    #
-    # ret = {'terminoloski_kandidati': [
-    #     {
-    #         'POSoznake': tk['msd'],
-    #         'kandidat': tk['terms'],  # more to bit lemma al terms?
-    #         'kanonicnaoblika': tk['canonical'],
-    #         'ranking': tk['ranking'],
-    #         'podporneutezi': [
-    #             6.0274563,  # ????????
-    #             6.0274563  # ??????
-    #         ],
-    #         'pogostostpojavljanja': [101, 71]  # ???????
-    #     }
-    #     for tk in example_data
-    # ]}
-
-    data = {
-        "terminoloski_kandidati": [
-            {
-                "POSoznake": "Ncmsn",
-                "kandidat": "vpliv",
-                "kanonicnaoblika": "vpliv",
-                "nosilnautez": 0.8008282,
-                "podporneutezi": [
-                    6.0274563,
-                    6.0274563
-                ],
-                "pogostostpojavljanja": [101, 71]
-            },
-            {
-                "POSoznake": "Agpnsg Ncnsg",
-                "kandidat": "bivalen okolje",
-                "kanonicnaoblika": "bivalno okolje",
-                "nosilnautez": 0.60254,
-                "podporneutezi": [
-                    3.263,
-                    2.134
-                ],
-                "pogostostpojavljanja": [27, 11]
-            },
-            {
-                "POSoznake": "Agpmsny Ncmsn Ncmpg",
-                "kandidat": "motorični status otrok",
-                "kanonicnaoblika": "motorični status otrok",
-                "nosilnautez": 0.3324,
-                "podporneutezi": [
-                    1.221,
-                    3.323
-                ],
-                "pogostostpojavljanja": [31, 51]
-            },
-            {
-                "POSoznake": "Agpnsn Ncnsn",
-                "kandidat": "diplomsko delo",
-                "kanonicnaoblika": "diplomsko delo",
-                "nosilnautez": 0.2008282,
-                "podporneutezi": [
-                    0.883,
-                    1.02
-                ],
-                "pogostostpojavljanja": [1241, 111]
-            }
-        ]
-    }
-    return data

swagger_server/models/__init__.py

@@ -11,7 +11,8 @@ from swagger_server.models.datoteka_v_conllu_async_body import DatotekaVConlluAs
 from swagger_server.models.datoteka_v_conllu_async_ocr_body import DatotekaVConlluAsyncOcrBody
 from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody
 from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody
-from swagger_server.models.izlusci_body import IzlusciBody
 from swagger_server.models.job_response import JobResponse
 from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody
 from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
+from swagger_server.models.izlusci_async_body import IzlusciAsyncBody
+from swagger_server.models.izlusci_sync_body import IzlusciSyncBody

swagger_server/models/izlusci_async_body.py

@@ -0,0 +1,88 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class IzlusciAsyncBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
+        """IzlusciAsyncBody - a model defined in Swagger
+
+        :param conllus: The conllus of this IzlusciAsyncBody.  # noqa: E501
+        :type conllus: List[str]
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody.  # noqa: E501
+        :type prepovedane_besede: List[str]
+        """
+        self.swagger_types = {
+            'conllus': List[str],
+            'prepovedane_besede': List[str]
+        }
+
+        self.attribute_map = {
+            'conllus': 'conllus',
+            'prepovedane_besede': 'prepovedaneBesede'
+        }
+        self._conllus = conllus
+        self._prepovedane_besede = prepovedane_besede
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'IzlusciAsyncBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The izlusciAsync_body of this IzlusciAsyncBody.  # noqa: E501
+        :rtype: IzlusciAsyncBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def conllus(self) -> List[str]:
+        """Gets the conllus of this IzlusciAsyncBody.
+
+
+        :return: The conllus of this IzlusciAsyncBody.
+        :rtype: List[str]
+        """
+        return self._conllus
+
+    @conllus.setter
+    def conllus(self, conllus: List[str]):
+        """Sets the conllus of this IzlusciAsyncBody.
+
+
+        :param conllus: The conllus of this IzlusciAsyncBody.
+        :type conllus: List[str]
+        """
+
+        self._conllus = conllus
+
+    @property
+    def prepovedane_besede(self) -> List[str]:
+        """Gets the prepovedane_besede of this IzlusciAsyncBody.
+
+
+        :return: The prepovedane_besede of this IzlusciAsyncBody.
+        :rtype: List[str]
+        """
+        return self._prepovedane_besede
+
+    @prepovedane_besede.setter
+    def prepovedane_besede(self, prepovedane_besede: List[str]):
+        """Sets the prepovedane_besede of this IzlusciAsyncBody.
+
+
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody.
+        :type prepovedane_besede: List[str]
+        """
+
+        self._prepovedane_besede = prepovedane_besede

swagger_server/models/izlusci_sync_body.py

@@ -9,17 +9,17 @@ from swagger_server.models.base_model_ import Model
 from swagger_server import util
 
 
-class IzlusciBody(Model):
+class IzlusciSyncBody(Model):
     """NOTE: This class is auto generated by the swagger code generator program.
 
     Do not edit the class manually.
     """
     def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
-        """IzlusciBody - a model defined in Swagger
+        """IzlusciSyncBody - a model defined in Swagger
 
-        :param conllus: The conllus of this IzlusciBody.  # noqa: E501
+        :param conllus: The conllus of this IzlusciSyncBody.  # noqa: E501
         :type conllus: List[str]
-        :param prepovedane_besede: The prepovedane_besede of this IzlusciBody.  # noqa: E501
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody.  # noqa: E501
         :type prepovedane_besede: List[str]
         """
         self.swagger_types = {
@@ -35,32 +35,32 @@ class IzlusciBody(Model):
         self._prepovedane_besede = prepovedane_besede
 
     @classmethod
-    def from_dict(cls, dikt) -> 'IzlusciBody':
+    def from_dict(cls, dikt) -> 'IzlusciSyncBody':
         """Returns the dict as a model
 
         :param dikt: A dict.
         :type: dict
-        :return: The izlusci_body of this IzlusciBody.  # noqa: E501
+        :return: The izlusciSync_body of this IzlusciSyncBody.  # noqa: E501
-        :rtype: IzlusciBody
+        :rtype: IzlusciSyncBody
         """
         return util.deserialize_model(dikt, cls)
 
     @property
     def conllus(self) -> List[str]:
-        """Gets the conllus of this IzlusciBody.
+        """Gets the conllus of this IzlusciSyncBody.
 
 
-        :return: The conllus of this IzlusciBody.
+        :return: The conllus of this IzlusciSyncBody.
         :rtype: List[str]
         """
         return self._conllus
 
     @conllus.setter
     def conllus(self, conllus: List[str]):
-        """Sets the conllus of this IzlusciBody.
+        """Sets the conllus of this IzlusciSyncBody.
 
 
-        :param conllus: The conllus of this IzlusciBody.
+        :param conllus: The conllus of this IzlusciSyncBody.
         :type conllus: List[str]
         """
 
@@ -68,20 +68,20 @@ class IzlusciBody(Model):
 
     @property
     def prepovedane_besede(self) -> List[str]:
-        """Gets the prepovedane_besede of this IzlusciBody.
+        """Gets the prepovedane_besede of this IzlusciSyncBody.
 
 
-        :return: The prepovedane_besede of this IzlusciBody.
+        :return: The prepovedane_besede of this IzlusciSyncBody.
         :rtype: List[str]
         """
         return self._prepovedane_besede
 
     @prepovedane_besede.setter
     def prepovedane_besede(self, prepovedane_besede: List[str]):
-        """Sets the prepovedane_besede of this IzlusciBody.
+        """Sets the prepovedane_besede of this IzlusciSyncBody.
 
 
-        :param prepovedane_besede: The prepovedane_besede of this IzlusciBody.
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody.
         :type prepovedane_besede: List[str]
         """
 

swagger_server/requets_db/models/vrsta.py

@@ -55,6 +55,7 @@ class JobManager:
         :possibilities:
         # 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 12 = oboje
         # 3 = pretvori dat v besedilo OCR, 2 = oznaci besedilo, 32 = oboje
+        # 4 = izlusci async
 
         :return: Job object, Did already exist boolean
         """

swagger_server/swagger/swagger.yaml

@@ -156,17 +156,18 @@ paths:
                 type: string
                 x-content-type: '*/*'
       x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
-  /izlusci:
+  /izlusciSync:
     post:
       tags:
       - extract
-      summary: Izlusci terminološke kandidate iz seznama besedil v conllu obliki
+      summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\
-      operationId: get_candidates
+        \ [sihrono, rezultat v sami zahtevi]"
+      operationId: get_candidates_sync
       requestBody:
         content:
           application/json:
             schema:
-              $ref: '#/components/schemas/izlusci_body'
+              $ref: '#/components/schemas/izlusciSync_body'
         required: true
       responses:
         "200":
@@ -179,6 +180,28 @@ paths:
                   $ref: '#/components/schemas/TerminoloskiKandidat'
                 x-content-type: '*/*'
       x-openapi-router-controller: swagger_server.controllers.extract_controller
+  /izlusciAsync:
+    post:
+      tags:
+      - extract
+      summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\
+        \ [asinhrono, ustvari novi job]"
+      operationId: get_candidates_async
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/izlusciAsync_body'
+        required: true
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.extract_controller
   /datotekaVBesediloSync:
     post:
       tags:
@@ -677,7 +700,7 @@ components:
         job_result:
           type: string
       example:
-        job_status: finished processing
+        job_status: waiting in que
         started_on: 2000-01-23T04:56:07.000+00:00
         created_on: 2000-01-23T04:56:07.000+00:00
         finished_on: 2000-01-23T04:56:07.000+00:00
@@ -719,7 +742,18 @@ components:
         file:
           type: string
           format: binary
-    izlusci_body:
+    izlusciSync_body:
+      type: object
+      properties:
+        conllus:
+          type: array
+          items:
+            type: string
+        prepovedaneBesede:
+          type: array
+          items:
+            type: string
+    izlusciAsync_body:
       type: object
       properties:
         conllus:

swagger_server/util.py

@@ -1,8 +1,14 @@
+import codecs
 import datetime
 import os.path
+import pathlib
 
 import six
 import typing
+
+import werkzeug.datastructures
+from werkzeug.utils import secure_filename
+
 from swagger_server import type_util
 import pandas as pd
 import string
@@ -195,3 +201,15 @@ def get_random_filename():
     extra = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
     return f'{ts}_{extra}'
 
+
+def create_random_file_in_tmp_folder(fill_content, extension=""):
+    pathlib.Path('tmp').mkdir(exist_ok=True)
+    tmp_file = ""
+    while True:
+        # just in case a VERY rare chance of a same generate name happens
+        tmp_file = "tmp/" + secure_filename(get_random_filename() + extension)
+        if not os.path.exists(tmp_file):
+            break
+    with codecs.open(tmp_file, 'w', 'utf-8') as f:
+        f.write(fill_content)
+    return tmp_file

swagger_server/utils/cl_utils.py

@@ -32,3 +32,16 @@ def multipla_conllus_to_one_from_file_ids(list_file_ids):
             sent_cnt += 1
 
     return ret
+
+
+def multipla_conllus_to_one_from_conllus_arr(list_conllus):
+    sent_cnt = 1
+    ret = "# newpar id = 1\n"
+    for conllu in list_conllus:
+        conllu = conllu.replace('\r\n', '\n')
+        matches = sent_extractor.finditer(conllu)
+        for match in matches:
+            ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
+            sent_cnt += 1
+
+    return ret