Merge remote-tracking branch 'origin/main'

Merge popravkov vrste in celega brancha z definicijami, kanonizirano obliko ter kodo ijs-ja

docker-compose.yml

@@ -5,7 +5,7 @@ services:
     restart: always
     image: flask-server
     ports:
-      - "8081:8080"
+      - "8080:8080"
     volumes: # when pushing to server, remove the first dot here in front of /mnt
       - /mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022
       - ../classla/classla_resources:/root/classla_resources

swagger_server/DB/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

swagger_server/__main__.py

@@ -7,6 +7,7 @@ from swagger_server import encoder
 def main():
     app = connexion.App(__name__, specification_dir='./swagger/')
     app.app.json_encoder = encoder.JSONEncoder
+    app.app.config['JSON_SORT_KEYS'] = False
     app.add_api('swagger.yaml', arguments={'title': 'OpenAPI definition'}, pythonic_params=True)
     app.run(port=8080)
 

swagger_server/controllers/oss_post_controller.py

@@ -0,0 +1,127 @@
+import connexion
+import six
+
+from swagger_server.controllers.oss_controller import get_conllus, get_extracted_words, get_extracted_words_async, \
+    get_files, get_number_texts, get_texts
+from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat  # noqa: E501
+from swagger_server import util
+
+
+def get_conllus_post(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
+    """Vrne seznam CoNNL-U-jev glede na iskalne pogoje
+
+     # noqa: E501
+
+    :param leta: 
+    :type leta: List[int]
+    :param vrste: 
+    :type vrste: List[int]
+    :param kljucne_besede: 
+    :type kljucne_besede: List[str]
+    :param udk: 
+    :type udk: List[str]
+
+    :rtype: List[str]
+    """
+    return get_conllus(leta, vrste, kljucne_besede, udk)
+
+
+def get_extracted_words_async_post(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None,
+                                   udk=None,definicije=False):  # noqa: E501
+    """Vrne terminloške kandidate glede na ... (async)
+
+     # noqa: E501
+
+    :param leta: 
+    :type leta: List[int]
+    :param vrste: 
+    :type vrste: List[int]
+    :param kljucne_besede: 
+    :type kljucne_besede: List[str]
+    :param prepovedane_besede: 
+    :type prepovedane_besede: List[str]
+    :param udk: 
+    :type udk: List[str]
+
+    :rtype: str
+    """
+    return get_extracted_words_async(leta, vrste, kljucne_besede, prepovedane_besede, udk,,definicije)
+
+
+def get_extracted_words_post(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None,
+                             udk=None,definicije=False):  # noqa: E501
+    """Vrne terminloške kandidate glede na ... (sync)
+
+     # noqa: E501
+
+    :param leta: 
+    :type leta: List[int]
+    :param vrste: 
+    :type vrste: List[int]
+    :param kljucne_besede: 
+    :type kljucne_besede: List[str]
+    :param prepovedane_besede: 
+    :type prepovedane_besede: List[str]
+    :param udk: 
+    :type udk: List[str]
+
+    :rtype: List[TerminoloskiKandidat]
+    """
+    return get_extracted_words(leta, vrste, kljucne_besede, prepovedane_besede, udk,definicije)
+
+
+def get_files_post(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
+    """Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje
+
+     # noqa: E501
+
+    :param leta: 
+    :type leta: List[int]
+    :param vrste: 
+    :type vrste: List[int]
+    :param kljucne_besede: 
+    :type kljucne_besede: List[str]
+    :param udk: 
+    :type udk: List[str]
+
+    :rtype: List[List[bytearray]]
+    """
+    return get_files(leta, vrste, kljucne_besede, udk)
+
+
+def get_number_texts_post(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
+    """Vrne število besedil glede na iskalne pogoje
+
+     # noqa: E501
+
+    :param leta: 
+    :type leta: List[int]
+    :param vrste: 
+    :type vrste: List[int]
+    :param kljucne_besede: 
+    :type kljucne_besede: List[str]
+    :param udk: 
+    :type udk: List[str]
+
+    :rtype: int
+    """
+    return get_number_texts(leta, vrste, kljucne_besede, udk)
+
+
+def get_texts_post(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
+    """Vrne seznam besedil glede na iskalne pogoje
+
+     # noqa: E501
+
+    :param leta: 
+    :type leta: List[int]
+    :param vrste: 
+    :type vrste: List[int]
+    :param kljucne_besede: 
+    :type kljucne_besede: List[str]
+    :param udk: 
+    :type udk: List[str]
+
+    :rtype: List[str]
+    """
+    return get_texts(leta, vrste, kljucne_besede, udk)

swagger_server/models/job_response.py

@@ -78,7 +78,7 @@ class JobResponse(Model):
         :param job_status: The job_status of this JobResponse.
         :type job_status: str
         """
-        allowed_values = ["waiting in que", "currently processing", "finished processing"]  # noqa: E501
+        allowed_values = ["waiting in que", "currently processing", "finished processing (OK)", "finished processing (ERROR)"]  # noqa: E501
         if job_status not in allowed_values:
             raise ValueError(
                 "Invalid value for `job_status` ({0}), must be one of {1}"

swagger_server/requets_db/models/vrsta.py

@@ -7,22 +7,14 @@ import os
 from swagger_server.util import get_random_filename
 from werkzeug.utils import secure_filename
 
-LOCAL_PATH = "requets_db/dbs"
+DB_Path = 'DB/jobs.db'
-SERVER_PATH = "swagger_server/requets_db/dbs"
+if not os.path.exists('DB'):
-
+    os.makedirs('DB', exist_ok=True)
-if not os.path.exists(LOCAL_PATH):
+db = SqliteDatabase(DB_Path, pragmas={
-    if os.path.exists('requets_db'):
+    'journal_mode': 'wal',
-        os.makedirs(LOCAL_PATH, exist_ok=True)
+    'cache_size': -1 * 256 * 1024,  # 256MB
-    elif os.path.exists('swagger_server/requets_db'):
+    'foreign_keys': 1,
-        os.makedirs(SERVER_PATH, exist_ok=True)
+    'charset': 'utf8'
-
-DB = f'{LOCAL_PATH}/jobs.db'
-if not os.path.exists(LOCAL_PATH):
-    DB = f'{SERVER_PATH}/jobs.db'
-db = SqliteDatabase(DB, pragmas={
-    # 'journal_mode': 'wal',
-    'cache_size': -1 * 128 * 1024,  # 128MB
-    'foreign_keys': 1
 })
 
 
@@ -43,7 +35,7 @@ class Job(BaseModel):
     input_file = TextField(index=True, null=True)
 
 
-db.drop_tables([Job])  # TODO: After pushing this, comment it and push again
+# db.drop_tables([Job])  # If pushing this uncommented, comment it and push again
 db.create_tables([Job])
 
 
@@ -55,12 +47,13 @@ class JobManager:
         :possibilities:
         # 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 12 = oboje
         # 3 = pretvori dat v besedilo OCR, 2 = oznaci besedilo, 32 = oboje
-        # 4 = izlusci async
+        # 4 = izlusci async glede na vhodne conlluje
+        # 5 = izlusci po iskanju async
 
         :return: Job object, Did already exist boolean
         """
         try:
-            if job_type in [2, 4]:
+            if job_type in [2, 4, 5]:
                 job, is_new = Job.get_or_create(job_type=job_type, job_input=job_input, input_size=len(job_input))
             elif job_type in [1, 3, 12, 32]:
                 tmp_file = ""
@@ -72,7 +65,8 @@ class JobManager:
                 pathlib.Path('tmp').mkdir(exist_ok=True)
                 job_input: werkzeug.datastructures.FileStorage
                 job_input.save(tmp_file)
-                job, is_new = Job.get_or_create(job_type=job_type, input_file=tmp_file, input_size=-1)
+                job, is_new = Job.get_or_create(job_type=job_type, input_file=tmp_file,
+                                                input_size=os.stat(tmp_file).st_size)
             return job, is_new
 
         except Exception as e:

swagger_server/utils/cl_utils.py

@@ -10,8 +10,11 @@ nlp_loaded = True
 
 sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL)
 
+
 def raw_text_to_conllu(text):
     try:
+        if text == '' or text is None:
+            raise Exception("Text can not be empty (when trying to make conllu for it)")
         docall = nlpSlo(text)
         docallconllu = docall.to_conll()