implemented the rest of the async functions and changed the way multithreading works

swagger_server/controllers/jobs_controller.py

@@ -1,15 +1,23 @@
 import datetime
+import os.path
 
 import peewee
 import asyncio
-
+import concurrent.futures as cf
 from swagger_server.models.job_response import JobResponse  # noqa: E501
 from swagger_server.requets_db.models.vrsta import (Job)
 from threading import Thread
 from swagger_server.classla import cl_utils
+from swagger_server.utils import txt_utils
+from werkzeug.datastructures import FileStorage
+import threading
+import time
 
-CLASSLA_CONCURANCE_LIMIT = 4
-classla_sem = asyncio.Semaphore(CLASSLA_CONCURANCE_LIMIT)
+CLASSLA_CONCURANCE_LIMIT = 3
+DOC2TEXT_CONCURANCE_LIMIT = 4
+
+classla_sem = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT)
+doc2text_sem = asyncio.Semaphore(DOC2TEXT_CONCURANCE_LIMIT)
 
 
 def delete_job(job_id):  # noqa: E501
@@ -53,51 +61,123 @@ def clear_up_unfinished_jobs():
     """
     In case server crashed while jobs were in queue...
     """
-    # tu more but !=, ne deluje ce je "is not"
-    Job.update(started_on=None).where(Job.started_on != None, Job.finished_on == None).execute()
+    Job.update(started_on=None).where(Job.started_on.is_null(False), Job.finished_on.is_null()).execute()
+    if os.path.exists('tmp'):
+        for tmp_file in os.listdir('tmp'):
+            if not Job.select().where(Job.input_file == tmp_file).exists():
+                os.remove(f'tmp/{tmp_file}')
+
+
+async def try_do_jobs():
+    with cf.ThreadPoolExecutor(max_workers=2) as ex:
+        ex.submit(try_do_jobs_classla)
+        ex.submit(try_do_jobs_doc2text)
 
 
 ### Job looping
-async def try_do_jobs():
-    await asyncio.sleep(15)  # wait for tokenizers to load for classla ...
+def try_do_jobs_classla():
+    time.sleep(15)  # wait for tokenizers to load for classla ...
     while True:
         try:
             if cl_utils.nlp_loaded:
-                # print(cl_utils.raw_text_to_conllu("Danes je lep soncen dan. Res je!"))
                 if classla_sem._value > 0:
-                    # classla
-                    unfinished_jobs = Job.select() \
-                        .where(Job.finished_on == None, Job.started_on == None, Job.job_type == 2) \
+                    unfinished_jobs_txt = Job.select() \
+                        .where(Job.finished_on.is_null(), Job.job_type == 2,
+                               Job.input_file.is_null(False)) \
                         .limit(classla_sem._value)
-                    tasks = [
-                        asyncio.ensure_future(execute_classla_job(job))
-                        for job in unfinished_jobs
-                    ]
-                    asyncio.get_event_loop().create_task(prep_classla_jobs(tasks))
-                    # await asyncio.gather(*tasks)
 
+                    unfinished_jobs_no_txt = Job.select() \
+                        .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
+                               Job.input_file.is_null()) \
+                        .limit(classla_sem._value)
 
+                    unfinished_jobs = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
+                    unfinished_jobs = unfinished_jobs[:classla_sem._value]
+                    with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT) as ex:
+                        [ex.submit(execute_classla_job, job) for job in unfinished_jobs]
 
-            else:
-                pass
         except Exception as e:
-            print(f"Exception in ... {e}")
+            print(f"Exception in {__name__}: {e}")
         finally:
-            await asyncio.sleep(3)
+            time.sleep(3)
 
 
-async def prep_classla_jobs(tasks):
+### Job looping
+def try_do_jobs_doc2text():
+    while True:
+        try:
+            if doc2text_sem._value > 0:
+                unfinished_jobs = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type << [1, 12, 3, 32]) \
+                    .limit(doc2text_sem._value)
+                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT) as ex:
+                    [ex.submit(execute_doc2text_job, job) for job in unfinished_jobs]
+        except Exception as e:
+            print(f"Exception in {__name__}: {e}")
+        finally:
+            time.sleep(3)
+
+
+async def prep_jobs(tasks):
     await asyncio.gather(*tasks)
 
 
-async def execute_classla_job(job: Job):
-    async with classla_sem:
+def execute_doc2text_job(job: Job):
+    try:
+        doc2text_sem.acquire()
+        del_file = False
+        job.started_on = datetime.datetime.utcnow()
+        job.save()
+
+        tmp_file_path = job.input_file
+        if not os.path.exists(tmp_file_path):
+            job.finished_on = datetime.datetime.utcnow()
+            job.job_output = "ERROR - Temporary file went missing, couldn't properly finish job"
+            job.save()
+            return
+
+        with open(tmp_file_path, 'rb+') as f:
+            file = FileStorage(f)
+            jtype = job.job_type
+            text = ""
+            if jtype in [1, 12]:
+                text = txt_utils.extract_text_prepResp(file)
+            elif jtype in [3, 32]:
+                text = txt_utils.ocr_text_prepResp(file)
+
+            if jtype in [1, 3]:
+                job.job_output = text
+                job.finished_on = datetime.datetime.utcnow()
+            elif jtype in [12, 32]:
+                job.job_input = text
+                job.job_type = 2
+                job.input_size = len(text)
+                del_file = True
+            job.save()
+        if del_file:
+            try:
+                os.remove(tmp_file_path)
+            except:
+                pass
+
+    except:
+        job.started_on = None
+        job.save()
+    finally:
+        doc2text_sem.release()
+
+
+def execute_classla_job(job: Job):
+    try:
+        classla_sem.acquire()
         job.started_on = datetime.datetime.utcnow()
         job.save()
         conllu, _ = cl_utils.raw_text_to_conllu(job.job_input)
         job.job_output = conllu
         job.finished_on = datetime.datetime.utcnow()
         job.save()
+    finally:
+        classla_sem.release()
 
 
 clear_up_unfinished_jobs()
@@ -111,7 +191,3 @@ def loop_in_thread(loop):
 
 t = Thread(target=loop_in_thread, args=(loop,))
 t.start()
-
-# clear_up_unfinished_jobs()
-# loop = asyncio.get_event_loop()
-# loop.run_until_complete(try_do_jobs()) this version seems more at home, but it blocks the thread, fix that?