webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added async version of izlusciPoIskanju

Browse Source
This commit is contained in:
Kikimanox
2022-10-19 11:24:59 +02:00
parent 7682cd0e0a
commit 21fa8542c9
3 changed files with 57 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swagger_server/controllers/jobs_controller.py
+40 -5
View File
@@ -11,19 +11,21 @@ from swagger_server.controllers.extract_controller import do_izlusci
from swagger_server.models.job_response import JobResponse # noqa: E501
from swagger_server.requets_db.models.vrsta import (Job)
from threading import Thread
from swagger_server.utils import cl_utils
from swagger_server.utils import cl_utils, db_utils
from swagger_server.utils import txt_utils
from werkzeug.datastructures import FileStorage
import threading
import time
CLASSLA_CONCURANCE_LIMIT = 3
DOC2TEXT_CONCURANCE_LIMIT = 3
ATEAPI_CONCURANCE_LIMIT = 2
CLASSLA_CONCURANCE_LIMIT = 4
DOC2TEXT_CONCURANCE_LIMIT = 4
ATEAPI_CONCURANCE_LIMIT = 4
IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT = 4
classla_sem = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT)
doc2text_sem = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT)
ateapi_sem = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT)
izluscipoiskanju_sem = threading.Semaphore(IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT)
def delete_job(job_id): # noqa: E501
@@ -75,10 +77,28 @@ def clear_up_unfinished_jobs():
async def try_do_jobs():
with cf.ThreadPoolExecutor(max_workers=3) as ex:
with cf.ThreadPoolExecutor(max_workers=4) as ex:
ex.submit(try_do_jobs_classla)
ex.submit(try_do_jobs_doc2text)
ex.submit(try_do_jobs_ateapi)
ex.submit(try_do_jobs_izluscipoiskanju)
### Job looping
def try_do_jobs_izluscipoiskanju():
while True:
try:
if izluscipoiskanju_sem._value > 0:
unfinished_jobs = Job.select() \
.where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 5) \
.limit(izluscipoiskanju_sem._value)
with cf.ThreadPoolExecutor(max_workers=IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT) as ex:
[ex.submit(execute_izluscipoiskanju_job, job) for job in unfinished_jobs]
except Exception as e:
print(f"Exception in try_do_jobs_ateapi")
traceback.print_exc()
finally:
time.sleep(3)
### Job looping
@@ -220,6 +240,21 @@ def execute_ateapi_job(job: Job):
ateapi_sem.release()
def execute_izluscipoiskanju_job(job: Job):
try:
izluscipoiskanju_sem.acquire()
job.started_on = datetime.datetime.utcnow()
job.save()
info = json.loads(job.job_input)
terKand = db_utils.vrni_oss_terminoloske_kandidate(info['leta'], info['vrste'], info['kljucnebesede'],
info['udk'])
job.job_output = terKand
job.finished_on = datetime.datetime.utcnow()
job.save()
finally:
izluscipoiskanju_sem.release()
clear_up_unfinished_jobs()
loop = asyncio.get_event_loop()