diff --git a/rsdo5.json b/rsdo5.json index a260847..57babd8 100644 --- a/rsdo5.json +++ b/rsdo5.json @@ -265,13 +265,13 @@ } } }, - "/izlusci": { + "/izlusciSync": { "post": { "tags": [ "extract" ], - "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki", - "operationId": "getCandidates", + "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki [sihrono, rezultat v sami zahtevi]", + "operationId": "getCandidates_sync", "requestBody": { "content": { "application/json": { @@ -313,6 +313,51 @@ } } }, + "/izlusciAsync": { + "post": { + "tags": [ + "extract" + ], + "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki [asinhrono, ustvari novi job]", + "operationId": "getCandidates_async", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "conllus": { + "type": "array", + "items": { + "type": "string" + } + }, + "prepovedaneBesede": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/datotekaVBesediloSync": { "post": { "tags": [ diff --git a/swagger_server/controllers/extract_controller.py b/swagger_server/controllers/extract_controller.py index b53df7d..fdd0e47 100644 --- a/swagger_server/controllers/extract_controller.py +++ b/swagger_server/controllers/extract_controller.py @@ -1,114 +1,83 @@ +import codecs +import os + import connexion import json from pathlib import Path -from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501 +from swagger_server.models.izlusci_async_body import IzlusciAsyncBody # noqa: E501 +from swagger_server.models.izlusci_sync_body import IzlusciSyncBody # noqa: E501 from swagger_server.utils import cl_utils +from swagger_server.util import get_random_filename, create_random_file_in_tmp_folder import requests +from werkzeug.utils import secure_filename -# ATEapi_endpoint = "http://localhost:5000/predict" +ATEapi_endpoint = "http://localhost:5000/predict" -ATEapi_endpoint = "http://ate-api:5000/predict" +# ATEapi_endpoint = "http://ate-api:5000/predict" -def get_candidates(body): # noqa: E501 - """Izlusci terminološke kandidate iz seznama besedil v conllu obliki +def get_candidates_async(body): # noqa: E501 + """Izlusci terminološke kandidate iz seznama besedil v conllu obliki [asinhrono, ustvari novi job] # noqa: E501 - :param body: + :param body: + :type body: dict | bytes + + :rtype: str + """ + if connexion.request.is_json: + body = IzlusciAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501 + return 'do some magic!' + + +def do_izlusci(conllus, prepovedane_besede): + tmp_file_path = "" + try: + big_conllu = cl_utils.multipla_conllus_to_one_from_conllus_arr(conllus) + tmp_file_path = create_random_file_in_tmp_folder(big_conllu, ".conllu") + fp = open(tmp_file_path, 'rb') + try: + files = [ + ('file', ('temp_1.conllu', fp, 'application/octet-stream')) + ] + res = requests.post(ATEapi_endpoint, files=files) + data = json.loads(res.text) + finally: + fp.close() + os.remove(tmp_file_path) + + ret = {'terminoloski_kandidati': [ + { + 'POSoznake': tk['msd'], + 'kandidat': tk['terms'], # more to bit lemma al terms? + 'kanonicnaoblika': tk['canonical'], + 'ranking': tk['ranking'], + 'podporneutezi': [ + 0.0, # ???????? + 0.0 # ?????? + ], + 'pogostostpojavljanja': [0, 0] # ??????? + } + for tk in data if tk['terms'] not in prepovedane_besede + ]} + return ret, 200 + except Exception as e: + return str(e), 500 + + +def get_candidates_sync(body): # noqa: E501 + """Izlusci terminološke kandidate iz seznama besedil v conllu obliki [sihrono, rezultat v sami zahtevi] + + # noqa: E501 + + :param body: :type body: dict | bytes :rtype: List[TerminoloskiKandidat] """ if connexion.request.is_json: - body = IzlusciBody.from_dict(connexion.request.get_json()) # noqa: E501 + body = IzlusciSyncBody.from_dict(connexion.request.get_json()) # noqa: E501 - # Todo: What is "conllus" in body input anyway?? - # Todo: Kaj je s prepovedanimi besedami? - # file_ids = [10000, 10001, 10002, 10003] - # big_conllu = cl_utils.multipla_conllus_to_one_from_file_ids(file_ids) - - # txt = Path('../ATEapi/temp_1.conllu').read_text('utf-8') # LOCAL ONLY - - # # Todo: does this close the file after the request? - # files = [ - # ('file', ('temp_1.conllu', open('../ATEapi/temp_1.conllu', 'rb'), - # 'application/octet-stream')) - # ] - # res = requests.post(ATEapi_endpoint, files=files) - # # res.status_code - # # res.text - # - # return res.text, res.status_code - # - # example_data = "" - # with open("C:\\Users\\Kiki\\Desktop\\Untitled-2.json", "r", encoding='utf-8') as f: - # example_data = json.loads(f.read()) - # - # d = 0 - # - # ret = {'terminoloski_kandidati': [ - # { - # 'POSoznake': tk['msd'], - # 'kandidat': tk['terms'], # more to bit lemma al terms? - # 'kanonicnaoblika': tk['canonical'], - # 'ranking': tk['ranking'], - # 'podporneutezi': [ - # 6.0274563, # ???????? - # 6.0274563 # ?????? - # ], - # 'pogostostpojavljanja': [101, 71] # ??????? - # } - # for tk in example_data - # ]} - - data = { - "terminoloski_kandidati": [ - { - "POSoznake": "Ncmsn", - "kandidat": "vpliv", - "kanonicnaoblika": "vpliv", - "nosilnautez": 0.8008282, - "podporneutezi": [ - 6.0274563, - 6.0274563 - ], - "pogostostpojavljanja": [101, 71] - }, - { - "POSoznake": "Agpnsg Ncnsg", - "kandidat": "bivalen okolje", - "kanonicnaoblika": "bivalno okolje", - "nosilnautez": 0.60254, - "podporneutezi": [ - 3.263, - 2.134 - ], - "pogostostpojavljanja": [27, 11] - }, - { - "POSoznake": "Agpmsny Ncmsn Ncmpg", - "kandidat": "motorični status otrok", - "kanonicnaoblika": "motorični status otrok", - "nosilnautez": 0.3324, - "podporneutezi": [ - 1.221, - 3.323 - ], - "pogostostpojavljanja": [31, 51] - }, - { - "POSoznake": "Agpnsn Ncnsn", - "kandidat": "diplomsko delo", - "kanonicnaoblika": "diplomsko delo", - "nosilnautez": 0.2008282, - "podporneutezi": [ - 0.883, - 1.02 - ], - "pogostostpojavljanja": [1241, 111] - } - ] - } - return data + return do_izlusci(body.conllus, body.prepovedane_besede) diff --git a/swagger_server/models/__init__.py b/swagger_server/models/__init__.py index c7bfe56..43e30da 100644 --- a/swagger_server/models/__init__.py +++ b/swagger_server/models/__init__.py @@ -11,7 +11,8 @@ from swagger_server.models.datoteka_v_conllu_async_body import DatotekaVConlluAs from swagger_server.models.datoteka_v_conllu_async_ocr_body import DatotekaVConlluAsyncOcrBody from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody -from swagger_server.models.izlusci_body import IzlusciBody from swagger_server.models.job_response import JobResponse from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat +from swagger_server.models.izlusci_async_body import IzlusciAsyncBody +from swagger_server.models.izlusci_sync_body import IzlusciSyncBody diff --git a/swagger_server/models/izlusci_async_body.py b/swagger_server/models/izlusci_async_body.py new file mode 100644 index 0000000..fd41ef7 --- /dev/null +++ b/swagger_server/models/izlusci_async_body.py @@ -0,0 +1,88 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from swagger_server.models.base_model_ import Model +from swagger_server import util + + +class IzlusciAsyncBody(Model): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None): # noqa: E501 + """IzlusciAsyncBody - a model defined in Swagger + + :param conllus: The conllus of this IzlusciAsyncBody. # noqa: E501 + :type conllus: List[str] + :param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody. # noqa: E501 + :type prepovedane_besede: List[str] + """ + self.swagger_types = { + 'conllus': List[str], + 'prepovedane_besede': List[str] + } + + self.attribute_map = { + 'conllus': 'conllus', + 'prepovedane_besede': 'prepovedaneBesede' + } + self._conllus = conllus + self._prepovedane_besede = prepovedane_besede + + @classmethod + def from_dict(cls, dikt) -> 'IzlusciAsyncBody': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The izlusciAsync_body of this IzlusciAsyncBody. # noqa: E501 + :rtype: IzlusciAsyncBody + """ + return util.deserialize_model(dikt, cls) + + @property + def conllus(self) -> List[str]: + """Gets the conllus of this IzlusciAsyncBody. + + + :return: The conllus of this IzlusciAsyncBody. + :rtype: List[str] + """ + return self._conllus + + @conllus.setter + def conllus(self, conllus: List[str]): + """Sets the conllus of this IzlusciAsyncBody. + + + :param conllus: The conllus of this IzlusciAsyncBody. + :type conllus: List[str] + """ + + self._conllus = conllus + + @property + def prepovedane_besede(self) -> List[str]: + """Gets the prepovedane_besede of this IzlusciAsyncBody. + + + :return: The prepovedane_besede of this IzlusciAsyncBody. + :rtype: List[str] + """ + return self._prepovedane_besede + + @prepovedane_besede.setter + def prepovedane_besede(self, prepovedane_besede: List[str]): + """Sets the prepovedane_besede of this IzlusciAsyncBody. + + + :param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody. + :type prepovedane_besede: List[str] + """ + + self._prepovedane_besede = prepovedane_besede diff --git a/swagger_server/models/izlusci_body.py b/swagger_server/models/izlusci_sync_body.py similarity index 69% rename from swagger_server/models/izlusci_body.py rename to swagger_server/models/izlusci_sync_body.py index 8fbfd21..4320649 100644 --- a/swagger_server/models/izlusci_body.py +++ b/swagger_server/models/izlusci_sync_body.py @@ -9,17 +9,17 @@ from swagger_server.models.base_model_ import Model from swagger_server import util -class IzlusciBody(Model): +class IzlusciSyncBody(Model): """NOTE: This class is auto generated by the swagger code generator program. Do not edit the class manually. """ def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None): # noqa: E501 - """IzlusciBody - a model defined in Swagger + """IzlusciSyncBody - a model defined in Swagger - :param conllus: The conllus of this IzlusciBody. # noqa: E501 + :param conllus: The conllus of this IzlusciSyncBody. # noqa: E501 :type conllus: List[str] - :param prepovedane_besede: The prepovedane_besede of this IzlusciBody. # noqa: E501 + :param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody. # noqa: E501 :type prepovedane_besede: List[str] """ self.swagger_types = { @@ -35,32 +35,32 @@ class IzlusciBody(Model): self._prepovedane_besede = prepovedane_besede @classmethod - def from_dict(cls, dikt) -> 'IzlusciBody': + def from_dict(cls, dikt) -> 'IzlusciSyncBody': """Returns the dict as a model :param dikt: A dict. :type: dict - :return: The izlusci_body of this IzlusciBody. # noqa: E501 - :rtype: IzlusciBody + :return: The izlusciSync_body of this IzlusciSyncBody. # noqa: E501 + :rtype: IzlusciSyncBody """ return util.deserialize_model(dikt, cls) @property def conllus(self) -> List[str]: - """Gets the conllus of this IzlusciBody. + """Gets the conllus of this IzlusciSyncBody. - :return: The conllus of this IzlusciBody. + :return: The conllus of this IzlusciSyncBody. :rtype: List[str] """ return self._conllus @conllus.setter def conllus(self, conllus: List[str]): - """Sets the conllus of this IzlusciBody. + """Sets the conllus of this IzlusciSyncBody. - :param conllus: The conllus of this IzlusciBody. + :param conllus: The conllus of this IzlusciSyncBody. :type conllus: List[str] """ @@ -68,20 +68,20 @@ class IzlusciBody(Model): @property def prepovedane_besede(self) -> List[str]: - """Gets the prepovedane_besede of this IzlusciBody. + """Gets the prepovedane_besede of this IzlusciSyncBody. - :return: The prepovedane_besede of this IzlusciBody. + :return: The prepovedane_besede of this IzlusciSyncBody. :rtype: List[str] """ return self._prepovedane_besede @prepovedane_besede.setter def prepovedane_besede(self, prepovedane_besede: List[str]): - """Sets the prepovedane_besede of this IzlusciBody. + """Sets the prepovedane_besede of this IzlusciSyncBody. - :param prepovedane_besede: The prepovedane_besede of this IzlusciBody. + :param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody. :type prepovedane_besede: List[str] """ diff --git a/swagger_server/requets_db/models/vrsta.py b/swagger_server/requets_db/models/vrsta.py index 3c9fcb7..4d75031 100644 --- a/swagger_server/requets_db/models/vrsta.py +++ b/swagger_server/requets_db/models/vrsta.py @@ -55,6 +55,7 @@ class JobManager: :possibilities: # 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 12 = oboje # 3 = pretvori dat v besedilo OCR, 2 = oznaci besedilo, 32 = oboje + # 4 = izlusci async :return: Job object, Did already exist boolean """ diff --git a/swagger_server/swagger/swagger.yaml b/swagger_server/swagger/swagger.yaml index 63445a5..2052337 100644 --- a/swagger_server/swagger/swagger.yaml +++ b/swagger_server/swagger/swagger.yaml @@ -156,17 +156,18 @@ paths: type: string x-content-type: '*/*' x-openapi-router-controller: swagger_server.controllers.marktext_async_controller - /izlusci: + /izlusciSync: post: tags: - extract - summary: Izlusci terminološke kandidate iz seznama besedil v conllu obliki - operationId: get_candidates + summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\ + \ [sihrono, rezultat v sami zahtevi]" + operationId: get_candidates_sync requestBody: content: application/json: schema: - $ref: '#/components/schemas/izlusci_body' + $ref: '#/components/schemas/izlusciSync_body' required: true responses: "200": @@ -179,6 +180,28 @@ paths: $ref: '#/components/schemas/TerminoloskiKandidat' x-content-type: '*/*' x-openapi-router-controller: swagger_server.controllers.extract_controller + /izlusciAsync: + post: + tags: + - extract + summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\ + \ [asinhrono, ustvari novi job]" + operationId: get_candidates_async + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/izlusciAsync_body' + required: true + responses: + "200": + description: OK + content: + '*/*': + schema: + type: string + x-content-type: '*/*' + x-openapi-router-controller: swagger_server.controllers.extract_controller /datotekaVBesediloSync: post: tags: @@ -677,7 +700,7 @@ components: job_result: type: string example: - job_status: finished processing + job_status: waiting in que started_on: 2000-01-23T04:56:07.000+00:00 created_on: 2000-01-23T04:56:07.000+00:00 finished_on: 2000-01-23T04:56:07.000+00:00 @@ -719,7 +742,18 @@ components: file: type: string format: binary - izlusci_body: + izlusciSync_body: + type: object + properties: + conllus: + type: array + items: + type: string + prepovedaneBesede: + type: array + items: + type: string + izlusciAsync_body: type: object properties: conllus: diff --git a/swagger_server/util.py b/swagger_server/util.py index 206d4a0..e067ab1 100644 --- a/swagger_server/util.py +++ b/swagger_server/util.py @@ -1,8 +1,14 @@ +import codecs import datetime import os.path +import pathlib import six import typing + +import werkzeug.datastructures +from werkzeug.utils import secure_filename + from swagger_server import type_util import pandas as pd import string @@ -195,3 +201,15 @@ def get_random_filename(): extra = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) return f'{ts}_{extra}' + +def create_random_file_in_tmp_folder(fill_content, extension=""): + pathlib.Path('tmp').mkdir(exist_ok=True) + tmp_file = "" + while True: + # just in case a VERY rare chance of a same generate name happens + tmp_file = "tmp/" + secure_filename(get_random_filename() + extension) + if not os.path.exists(tmp_file): + break + with codecs.open(tmp_file, 'w', 'utf-8') as f: + f.write(fill_content) + return tmp_file diff --git a/swagger_server/utils/cl_utils.py b/swagger_server/utils/cl_utils.py index d1507f9..d9a75e1 100644 --- a/swagger_server/utils/cl_utils.py +++ b/swagger_server/utils/cl_utils.py @@ -32,3 +32,16 @@ def multipla_conllus_to_one_from_file_ids(list_file_ids): sent_cnt += 1 return ret + + +def multipla_conllus_to_one_from_conllus_arr(list_conllus): + sent_cnt = 1 + ret = "# newpar id = 1\n" + for conllu in list_conllus: + conllu = conllu.replace('\r\n', '\n') + matches = sent_extractor.finditer(conllu) + for match in matches: + ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n' + sent_cnt += 1 + + return ret