webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved extract method to work properly (sync)

Browse Source
This commit is contained in:
Kikimanox
2022-10-18 19:08:41 +02:00
parent b483e1643f
commit 673a83691b
9 changed files with 289 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swagger_server/controllers/extract_controller.py
+64 -95
View File
@@ -1,114 +1,83 @@
import codecs
import os
import connexion
import json
from pathlib import Path
from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501
from swagger_server.models.izlusci_async_body import IzlusciAsyncBody # noqa: E501
from swagger_server.models.izlusci_sync_body import IzlusciSyncBody # noqa: E501
from swagger_server.utils import cl_utils
from swagger_server.util import get_random_filename, create_random_file_in_tmp_folder
import requests
from werkzeug.utils import secure_filename
# ATEapi_endpoint = "http://localhost:5000/predict"
ATEapi_endpoint = "http://localhost:5000/predict"
ATEapi_endpoint = "http://ate-api:5000/predict"
# ATEapi_endpoint = "http://ate-api:5000/predict"
def get_candidates(body): # noqa: E501
"""Izlusci terminološke kandidate iz seznama besedil v conllu obliki
def get_candidates_async(body): # noqa: E501
"""Izlusci terminološke kandidate iz seznama besedil v conllu obliki [asinhrono, ustvari novi job]
# noqa: E501
:param body:
:param body:
:type body: dict | bytes
:rtype: str
"""
if connexion.request.is_json:
body = IzlusciAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501
return 'do some magic!'
def do_izlusci(conllus, prepovedane_besede):
tmp_file_path = ""
try:
big_conllu = cl_utils.multipla_conllus_to_one_from_conllus_arr(conllus)
tmp_file_path = create_random_file_in_tmp_folder(big_conllu, ".conllu")
fp = open(tmp_file_path, 'rb')
try:
files = [
('file', ('temp_1.conllu', fp, 'application/octet-stream'))
]
res = requests.post(ATEapi_endpoint, files=files)
data = json.loads(res.text)
finally:
fp.close()
os.remove(tmp_file_path)
ret = {'terminoloski_kandidati': [
{
'POSoznake': tk['msd'],
'kandidat': tk['terms'], # more to bit lemma al terms?
'kanonicnaoblika': tk['canonical'],
'ranking': tk['ranking'],
'podporneutezi': [
0.0, # ????????
0.0 # ??????
],
'pogostostpojavljanja': [0, 0] # ???????
}
for tk in data if tk['terms'] not in prepovedane_besede
]}
return ret, 200
except Exception as e:
return str(e), 500
def get_candidates_sync(body): # noqa: E501
"""Izlusci terminološke kandidate iz seznama besedil v conllu obliki [sihrono, rezultat v sami zahtevi]
# noqa: E501
:param body:
:type body: dict | bytes
:rtype: List[TerminoloskiKandidat]
"""
if connexion.request.is_json:
body = IzlusciBody.from_dict(connexion.request.get_json()) # noqa: E501
body = IzlusciSyncBody.from_dict(connexion.request.get_json()) # noqa: E501
# Todo: What is "conllus" in body input anyway??
# Todo: Kaj je s prepovedanimi besedami?
# file_ids = [10000, 10001, 10002, 10003]
# big_conllu = cl_utils.multipla_conllus_to_one_from_file_ids(file_ids)
# txt = Path('../ATEapi/temp_1.conllu').read_text('utf-8') # LOCAL ONLY
# # Todo: does this close the file after the request?
# files = [
# ('file', ('temp_1.conllu', open('../ATEapi/temp_1.conllu', 'rb'),
# 'application/octet-stream'))
# ]
# res = requests.post(ATEapi_endpoint, files=files)
# # res.status_code
# # res.text
#
# return res.text, res.status_code
#
# example_data = ""
# with open("C:\\Users\\Kiki\\Desktop\\Untitled-2.json", "r", encoding='utf-8') as f:
# example_data = json.loads(f.read())
#
# d = 0
#
# ret = {'terminoloski_kandidati': [
# {
# 'POSoznake': tk['msd'],
# 'kandidat': tk['terms'], # more to bit lemma al terms?
# 'kanonicnaoblika': tk['canonical'],
# 'ranking': tk['ranking'],
# 'podporneutezi': [
# 6.0274563, # ????????
# 6.0274563 # ??????
# ],
# 'pogostostpojavljanja': [101, 71] # ???????
# }
# for tk in example_data
# ]}
data = {
"terminoloski_kandidati": [
{
"POSoznake": "Ncmsn",
"kandidat": "vpliv",
"kanonicnaoblika": "vpliv",
"nosilnautez": 0.8008282,
"podporneutezi": [
6.0274563,
6.0274563
],
"pogostostpojavljanja": [101, 71]
},
{
"POSoznake": "Agpnsg Ncnsg",
"kandidat": "bivalen okolje",
"kanonicnaoblika": "bivalno okolje",
"nosilnautez": 0.60254,
"podporneutezi": [
3.263,
2.134
],
"pogostostpojavljanja": [27, 11]
},
{
"POSoznake": "Agpmsny Ncmsn Ncmpg",
"kandidat": "motorični status otrok",
"kanonicnaoblika": "motorični status otrok",
"nosilnautez": 0.3324,
"podporneutezi": [
1.221,
3.323
],
"pogostostpojavljanja": [31, 51]
},
{
"POSoznake": "Agpnsn Ncnsn",
"kandidat": "diplomsko delo",
"kanonicnaoblika": "diplomsko delo",
"nosilnautez": 0.2008282,
"podporneutezi": [
0.883,
1.02
],
"pogostostpojavljanja": [1241, 111]
}
]
}
return data
return do_izlusci(body.conllus, body.prepovedane_besede)
swagger_server/models/__init__.py
+2 -1
View File
@@ -11,7 +11,8 @@ from swagger_server.models.datoteka_v_conllu_async_body import DatotekaVConlluAs
from swagger_server.models.datoteka_v_conllu_async_ocr_body import DatotekaVConlluAsyncOcrBody
from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody
from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody
from swagger_server.models.izlusci_body import IzlusciBody
from swagger_server.models.job_response import JobResponse
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
from swagger_server.models.izlusci_async_body import IzlusciAsyncBody
from swagger_server.models.izlusci_sync_body import IzlusciSyncBody
swagger_server/models/izlusci_async_body.py
+88
View File
@@ -0,0 +1,88 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class IzlusciAsyncBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None): # noqa: E501
"""IzlusciAsyncBody - a model defined in Swagger
:param conllus: The conllus of this IzlusciAsyncBody. # noqa: E501
:type conllus: List[str]
:param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody. # noqa: E501
:type prepovedane_besede: List[str]
"""
self.swagger_types = {
'conllus': List[str],
'prepovedane_besede': List[str]
}
self.attribute_map = {
'conllus': 'conllus',
'prepovedane_besede': 'prepovedaneBesede'
}
self._conllus = conllus
self._prepovedane_besede = prepovedane_besede
@classmethod
def from_dict(cls, dikt) -> 'IzlusciAsyncBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The izlusciAsync_body of this IzlusciAsyncBody. # noqa: E501
:rtype: IzlusciAsyncBody
"""
return util.deserialize_model(dikt, cls)
@property
def conllus(self) -> List[str]:
"""Gets the conllus of this IzlusciAsyncBody.
:return: The conllus of this IzlusciAsyncBody.
:rtype: List[str]
"""
return self._conllus
@conllus.setter
def conllus(self, conllus: List[str]):
"""Sets the conllus of this IzlusciAsyncBody.
:param conllus: The conllus of this IzlusciAsyncBody.
:type conllus: List[str]
"""
self._conllus = conllus
@property
def prepovedane_besede(self) -> List[str]:
"""Gets the prepovedane_besede of this IzlusciAsyncBody.
:return: The prepovedane_besede of this IzlusciAsyncBody.
:rtype: List[str]
"""
return self._prepovedane_besede
@prepovedane_besede.setter
def prepovedane_besede(self, prepovedane_besede: List[str]):
"""Sets the prepovedane_besede of this IzlusciAsyncBody.
:param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody.
:type prepovedane_besede: List[str]
"""
self._prepovedane_besede = prepovedane_besede
swagger_server/models/izlusci_body.py → swagger_server/models/izlusci_sync_body.py
+15 -15
View File
@@ -9,17 +9,17 @@ from swagger_server.models.base_model_ import Model
from swagger_server import util
class IzlusciBody(Model):
class IzlusciSyncBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None): # noqa: E501
"""IzlusciBody - a model defined in Swagger
"""IzlusciSyncBody - a model defined in Swagger
:param conllus: The conllus of this IzlusciBody. # noqa: E501
:param conllus: The conllus of this IzlusciSyncBody. # noqa: E501
:type conllus: List[str]
:param prepovedane_besede: The prepovedane_besede of this IzlusciBody. # noqa: E501
:param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody. # noqa: E501
:type prepovedane_besede: List[str]
"""
self.swagger_types = {
@@ -35,32 +35,32 @@ class IzlusciBody(Model):
self._prepovedane_besede = prepovedane_besede
@classmethod
def from_dict(cls, dikt) -> 'IzlusciBody':
def from_dict(cls, dikt) -> 'IzlusciSyncBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The izlusci_body of this IzlusciBody. # noqa: E501
:rtype: IzlusciBody
:return: The izlusciSync_body of this IzlusciSyncBody. # noqa: E501
:rtype: IzlusciSyncBody
"""
return util.deserialize_model(dikt, cls)
@property
def conllus(self) -> List[str]:
"""Gets the conllus of this IzlusciBody.
"""Gets the conllus of this IzlusciSyncBody.
:return: The conllus of this IzlusciBody.
:return: The conllus of this IzlusciSyncBody.
:rtype: List[str]
"""
return self._conllus
@conllus.setter
def conllus(self, conllus: List[str]):
"""Sets the conllus of this IzlusciBody.
"""Sets the conllus of this IzlusciSyncBody.
:param conllus: The conllus of this IzlusciBody.
:param conllus: The conllus of this IzlusciSyncBody.
:type conllus: List[str]
"""
@@ -68,20 +68,20 @@ class IzlusciBody(Model):
@property
def prepovedane_besede(self) -> List[str]:
"""Gets the prepovedane_besede of this IzlusciBody.
"""Gets the prepovedane_besede of this IzlusciSyncBody.
:return: The prepovedane_besede of this IzlusciBody.
:return: The prepovedane_besede of this IzlusciSyncBody.
:rtype: List[str]
"""
return self._prepovedane_besede
@prepovedane_besede.setter
def prepovedane_besede(self, prepovedane_besede: List[str]):
"""Sets the prepovedane_besede of this IzlusciBody.
"""Sets the prepovedane_besede of this IzlusciSyncBody.
:param prepovedane_besede: The prepovedane_besede of this IzlusciBody.
:param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody.
:type prepovedane_besede: List[str]
"""
swagger_server/requets_db/models/vrsta.py
+1
View File
@@ -55,6 +55,7 @@ class JobManager:
:possibilities:
# 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 12 = oboje
# 3 = pretvori dat v besedilo OCR, 2 = oznaci besedilo, 32 = oboje
# 4 = izlusci async
:return: Job object, Did already exist boolean
"""
swagger_server/swagger/swagger.yaml
+40 -6
View File
@@ -156,17 +156,18 @@ paths:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
/izlusci:
/izlusciSync:
post:
tags:
- extract
summary: Izlusci terminološke kandidate iz seznama besedil v conllu obliki
operationId: get_candidates
summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\
\ [sihrono, rezultat v sami zahtevi]"
operationId: get_candidates_sync
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/izlusci_body'
$ref: '#/components/schemas/izlusciSync_body'
required: true
responses:
"200":
@@ -179,6 +180,28 @@ paths:
$ref: '#/components/schemas/TerminoloskiKandidat'
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.extract_controller
/izlusciAsync:
post:
tags:
- extract
summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\
\ [asinhrono, ustvari novi job]"
operationId: get_candidates_async
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/izlusciAsync_body'
required: true
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.extract_controller
/datotekaVBesediloSync:
post:
tags:
@@ -677,7 +700,7 @@ components:
job_result:
type: string
example:
job_status: finished processing
job_status: waiting in que
started_on: 2000-01-23T04:56:07.000+00:00
created_on: 2000-01-23T04:56:07.000+00:00
finished_on: 2000-01-23T04:56:07.000+00:00
@@ -719,7 +742,18 @@ components:
file:
type: string
format: binary
izlusci_body:
izlusciSync_body:
type: object
properties:
conllus:
type: array
items:
type: string
prepovedaneBesede:
type: array
items:
type: string
izlusciAsync_body:
type: object
properties:
conllus:
swagger_server/util.py
+18
View File
@@ -1,8 +1,14 @@
import codecs
import datetime
import os.path
import pathlib
import six
import typing
import werkzeug.datastructures
from werkzeug.utils import secure_filename
from swagger_server import type_util
import pandas as pd
import string
@@ -195,3 +201,15 @@ def get_random_filename():
extra = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
return f'{ts}_{extra}'
def create_random_file_in_tmp_folder(fill_content, extension=""):
pathlib.Path('tmp').mkdir(exist_ok=True)
tmp_file = ""
while True:
# just in case a VERY rare chance of a same generate name happens
tmp_file = "tmp/" + secure_filename(get_random_filename() + extension)
if not os.path.exists(tmp_file):
break
with codecs.open(tmp_file, 'w', 'utf-8') as f:
f.write(fill_content)
return tmp_file
swagger_server/utils/cl_utils.py
+13
View File
@@ -32,3 +32,16 @@ def multipla_conllus_to_one_from_file_ids(list_file_ids):
sent_cnt += 1
return ret
def multipla_conllus_to_one_from_conllus_arr(list_conllus):
sent_cnt = 1
ret = "# newpar id = 1\n"
for conllu in list_conllus:
conllu = conllu.replace('\r\n', '\n')
matches = sent_extractor.finditer(conllu)
for match in matches:
ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
sent_cnt += 1
return ret