Added new logic (temporary logic for getting files by search) and converting from files to txt

swagger_server/util.py

@@ -1,8 +1,10 @@
 import datetime
+import os.path
 
 import six
 import typing
 from swagger_server import type_util
+import pandas as pd
 
 
 def _deserialize(data, klass):
@@ -142,9 +144,43 @@ def _deserialize_dict(data, boxed_type):
             for k, v in six.iteritems(data)}
 
 
-def get_conllu_path_by_id(file_id):
-    return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.plainText.conllu'
+def is_docker() -> bool:
+    # todo: better way of checking if we're on docker or if we're in the develoment enviroment
+    return not os.path.exists('.env')
 
 
-def get_original_file_by_id(file_id):
-    return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.xml'
+def get_conllu_file_path_by_id(file_id):
+    r = f'classla_OS2022/conll/rsdo_doc-{file_id}.plainText.conllu'
+    if is_docker():
+        return r
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_original_file_path_by_id(file_id):
+    r = f'classla_OS2022/besedila/rsdo_doc-{file_id}.xml'
+    if is_docker():
+        return r
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_tei_file_path_by_id(file_id):
+    r = f'classla_OS2022/tei/rsdo_doc-{file_id}.plainText.tei.xml'
+    if is_docker():
+        return r
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_files_by_keywords(kljucnebesede):
+    ret = []
+    kljucnebesede = [k.lower() for k in kljucnebesede]
+    # Temporary solution until connection with mariadb is fixed
+    ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/"
+    for path, dirs, files, in os.walk(ngrams_path):
+        for _file in files:
+            file = f'{ngrams_path}{_file}'
+            data = pd.read_csv(file, sep='\t')
+            amount = len(data[data['ngram_len'] == 1 & data['gram_text'].str.lower().isin(kljucnebesede)])
+            # this should be 1
+            if amount >= 1:
+                ret.append(_file[9:][:-12])
+    return ret