webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added mentioned post requests, made it so that the response body is a clear json (as requested) in case the response is a json, fixed decimal bug, updated swagger templates and json

Browse Source
This commit is contained in:
Kikimanox
2022-11-29 17:14:40 +01:00
parent 3c2577a5d6
commit 911bc8f0c9
6 changed files with 802 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swagger_server/controllers/extract_controller.py
+2 -2
View File
@@ -15,7 +15,7 @@ from werkzeug.utils import secure_filename
ATEapi_endpoint = "http://ate-api:5000/predict"
# endpoint below to be used only for development purposes (don't need to run docker)
# ATEapi_endpoint = "http://localhost:5000/predict"
# ATEapi_endpoint = "http://localhost:5001/predict"
def do_izlusci(conllus, prepovedane_besede):
@@ -54,7 +54,7 @@ def do_izlusci(conllus, prepovedane_besede):
0.0 # ??????
],
# 'pogostostpojavljanja': [0, 0] # ???????
'pogostostpojavljanja': tk['frequency'] # ???????
'pogostostpojavljanja': int(tk['frequency']) # ???????
}
for tk in data if tk['lemma'] not in prepovedane_besede
]}
swagger_server/controllers/jobs_controller.py
+22 -1
View File
@@ -68,9 +68,15 @@ def get_job_status(job_id): # noqa: E501
return JobResponse(job_status="currently processing", created_on=job.created_on,
started_on=job.started_on), 200
if job.started_on is not None and job.finished_on is not None and not job.job_output.startswith("ERROR -"):
res = job.job_output
if job.job_type in [4, 5]:
try:
res = json.loads(job.job_output)
except:
pass
return JobResponse(job_status="finished processing (OK)", created_on=job.created_on,
started_on=job.started_on,
finished_on=job.finished_on, job_result=job.job_output), 200
finished_on=job.finished_on, job_result=res), 200
if job.started_on is not None and job.finished_on is not None and job.job_output.startswith("ERROR -"):
return JobResponse(job_status="finished processing (ERROR)", created_on=job.created_on,
started_on=job.started_on,
@@ -249,6 +255,11 @@ def execute_classla_job(job: Job):
job.job_output = conllu
job.finished_on = datetime.datetime.utcnow()
job.save()
except:
job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
job.finished_on = datetime.datetime.utcnow()
job.save()
print(f"Unexpected error at job {job.id}")
finally:
classla_sem.release()
@@ -279,6 +290,11 @@ def execute_ateapi_job(job: Job):
job.job_output = json.dumps(ret_json, ensure_ascii=False)
job.finished_on = datetime.datetime.utcnow()
job.save()
except:
job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
job.finished_on = datetime.datetime.utcnow()
job.save()
print(f"Unexpected error at job {job.id}")
finally:
ateapi_sem.release()
@@ -294,6 +310,11 @@ def execute_izluscipoiskanju_job(job: Job):
job.job_output = json.dumps(terKand, ensure_ascii=False)
job.finished_on = datetime.datetime.utcnow()
job.save()
except:
job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
job.finished_on = datetime.datetime.utcnow()
job.save()
print(f"Unexpected error at job {job.id}")
finally:
izluscipoiskanju_sem.release()
swagger_server/controllers/oss_post_controller.py
+127
View File
@@ -0,0 +1,127 @@
import connexion
import six
from swagger_server.controllers.oss_controller import get_conllus, get_extracted_words, get_extracted_words_async, \
get_files, get_number_texts, get_texts
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util
def get_conllus_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501
"""Vrne seznam CoNNL-U-jev glede na iskalne pogoje
# noqa: E501
:param leta:
:type leta: List[int]
:param vrste:
:type vrste: List[int]
:param kljucne_besede:
:type kljucne_besede: List[str]
:param udk:
:type udk: List[str]
:rtype: List[str]
"""
return get_conllus(leta, vrste, kljucne_besede, udk)
def get_extracted_words_async_post(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None,
udk=None): # noqa: E501
"""Vrne terminloške kandidate glede na ... (async)
# noqa: E501
:param leta:
:type leta: List[int]
:param vrste:
:type vrste: List[int]
:param kljucne_besede:
:type kljucne_besede: List[str]
:param prepovedane_besede:
:type prepovedane_besede: List[str]
:param udk:
:type udk: List[str]
:rtype: str
"""
return get_extracted_words_async(leta, vrste, kljucne_besede, prepovedane_besede, udk)
def get_extracted_words_post(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None,
udk=None): # noqa: E501
"""Vrne terminloške kandidate glede na ... (sync)
# noqa: E501
:param leta:
:type leta: List[int]
:param vrste:
:type vrste: List[int]
:param kljucne_besede:
:type kljucne_besede: List[str]
:param prepovedane_besede:
:type prepovedane_besede: List[str]
:param udk:
:type udk: List[str]
:rtype: List[TerminoloskiKandidat]
"""
return get_extracted_words(leta, vrste, kljucne_besede, prepovedane_besede, udk)
def get_files_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501
"""Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje
# noqa: E501
:param leta:
:type leta: List[int]
:param vrste:
:type vrste: List[int]
:param kljucne_besede:
:type kljucne_besede: List[str]
:param udk:
:type udk: List[str]
:rtype: List[List[bytearray]]
"""
return get_files(leta, vrste, kljucne_besede, udk)
def get_number_texts_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501
"""Vrne število besedil glede na iskalne pogoje
# noqa: E501
:param leta:
:type leta: List[int]
:param vrste:
:type vrste: List[int]
:param kljucne_besede:
:type kljucne_besede: List[str]
:param udk:
:type udk: List[str]
:rtype: int
"""
return get_number_texts(leta, vrste, kljucne_besede, udk)
def get_texts_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501
"""Vrne seznam besedil glede na iskalne pogoje
# noqa: E501
:param leta:
:type leta: List[int]
:param vrste:
:type vrste: List[int]
:param kljucne_besede:
:type kljucne_besede: List[str]
:param udk:
:type udk: List[str]
:rtype: List[str]
"""
return get_texts(leta, vrste, kljucne_besede, udk)
swagger_server/swagger/swagger.yaml
+405 -7
View File
@@ -339,6 +339,60 @@ paths:
format: int64
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_controller
post:
tags:
- oss-post
summary: Vrne število besedil glede na iskalne pogoje
operationId: get_number_texts_post
parameters:
- name: leta
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: kljucneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: udk
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
responses:
"200":
description: OK
content:
'*/*':
schema:
type: integer
format: int64
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_post_controller
/oss/izlusciPoIskanju:
get:
tags:
@@ -404,6 +458,70 @@ paths:
$ref: '#/components/schemas/TerminoloskiKandidat'
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_controller
post:
tags:
- oss-post
summary: Vrne terminloške kandidate glede na ... (sync)
operationId: get_extracted_words_post
parameters:
- name: leta
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: kljucneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: prepovedaneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: udk
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
responses:
"200":
description: OK
content:
'*/*':
schema:
type: array
items:
$ref: '#/components/schemas/TerminoloskiKandidat'
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_post_controller
/oss/izlusciPoIskanjuAsync:
get:
tags:
@@ -467,6 +585,68 @@ paths:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_controller
post:
tags:
- oss-post
summary: Vrne terminloške kandidate glede na ... (async)
operationId: get_extracted_words_async_post
parameters:
- name: leta
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: kljucneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: prepovedaneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: udk
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_post_controller
/oss/datotekePoIskanju:
get:
tags:
@@ -527,6 +707,65 @@ paths:
format: byte
x-content-type: application/octet-stream
x-openapi-router-controller: swagger_server.controllers.oss_controller
post:
tags:
- oss-post
summary: Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne
pogoje
operationId: get_files_post
parameters:
- name: leta
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: kljucneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: udk
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
responses:
"200":
description: OK
content:
application/octet-stream:
schema:
type: array
items:
type: array
items:
type: string
format: byte
x-content-type: application/octet-stream
x-openapi-router-controller: swagger_server.controllers.oss_post_controller
/oss/datotekaPoId:
get:
tags:
@@ -610,6 +849,61 @@ paths:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_controller
post:
tags:
- oss-post
summary: Vrne seznam CoNNL-U-jev glede na iskalne pogoje
operationId: get_conllus_post
parameters:
- name: leta
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: kljucneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: udk
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
responses:
"200":
description: OK
content:
'*/*':
schema:
type: array
items:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_post_controller
/oss/conlluPoId:
get:
tags:
@@ -714,6 +1008,61 @@ paths:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_controller
post:
tags:
- oss-post
summary: Vrne seznam besedil glede na iskalne pogoje
operationId: get_texts_post
parameters:
- name: leta
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: kljucneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
- name: udk
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
responses:
"200":
description: OK
content:
'*/*':
schema:
type: array
items:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.oss_post_controller
components:
schemas:
TerminoloskiKandidat:
@@ -734,17 +1083,13 @@ components:
type: number
format: float
pogostostpojavljanja:
type: array
items:
type: integer
format: int64
type: number
format: int64
example:
podporneutezi:
- 6.0274563
- 6.0274563
pogostostpojavljanja:
- 1
- 1
pogostostpojavljanja: 1.46581298050294517310021547018550336360931396484375
POSoznake: POSoznake
kanonicnaoblika: kanonicnaoblika
nosilnautez: 0.8008282
@@ -869,4 +1214,57 @@ components:
file:
type: string
format: binary
parameters:
leta:
name: leta
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
vrste:
name: vrste
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
kljucneBesede:
name: kljucneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
prepovedaneBesede:
name: prepovedaneBesede
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
udk:
name: udk
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
swagger_server/utils/db_utils.py
+2 -1
View File
@@ -172,7 +172,8 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucne_besede, prepovedane_bes
0.0, # ????????
0.0 # ??????
],
'pogostostpojavljanja': [tf, 0] # ???????
# 'pogostostpojavljanja': [tf 0.0] # ???????
'pogostostpojavljanja': int(tf)
}
for upos,ngram,tfidf,tf in cur
]}