Added new logic (temporary logic for getting files by search) and converting from files to txt

requirements.txt

@@ -7,3 +7,7 @@ peewee==3.15.0
 mariadb==1.0.11
 classla==1.1.0
 python-decouple==3.6
+pandas==1.3.3
+python-docx==0.8.11
+lxml==4.8.0
+PyPDF2==2.10.4

swagger_server/__main__.py

@@ -8,7 +8,6 @@ def main():
     app = connexion.App(__name__, specification_dir='./swagger/')
     app.app.json_encoder = encoder.JSONEncoder
     app.add_api('swagger.yaml', arguments={'title': 'OpenAPI definition'}, pythonic_params=True)
-    print("Starting app4")
     app.run(port=8080)
 
 

swagger_server/controllers/doc2text_controller.py

@@ -1,7 +1,13 @@
 import connexion
 import six
+# import tika
 
 from swagger_server import util
+import pandas as pd
+import docx
+import codecs
+import xml.etree.ElementTree as ET
+from PyPDF2 import PdfReader
 
 
 def datoteka_v_besedilo_post(file=None):  # noqa: E501
@@ -14,7 +20,27 @@ def datoteka_v_besedilo_post(file=None):  # noqa: E501
 
     :rtype: str
     """
-    return 'do some magic!'
+
+    if file is None:
+        return "No file provided", 400
+    elif "openxmlformats-officedocument.wordprocessingml.document" in file.content_type:
+        content = [p.text for p in docx.Document(file).paragraphs]
+    elif "text/plain" in file.content_type:
+        content = file.read().decode('utf-8')
+    elif "application/pdf" in file.content_type:
+        reader = PdfReader(file)
+        content = '\n'.join([p.extract_text() for p in reader.pages])
+        content = "ZACASNO UPORABLJEN DRUGI BRALEC KOT TIKA, TA BO SE DODANA KASNEJE...\n\n" + content
+    elif "text/xml" in file.content_type:
+        root = ET.parse(file).getroot()
+        plainText = root.findall('PlainText')
+        if len(plainText) == 0:
+            return "Didn't find anything in PlainText", 400
+        content = '\n'.join([pt.text for pt in plainText])
+    else:
+        return "Currently supporing only docx, txt/plain, pdf, xml*... more will be added later", 501
+
+    return content, 200
 
 
 def get_text_ocr(file=None):  # noqa: E501
@@ -27,4 +53,4 @@ def get_text_ocr(file=None):  # noqa: E501
 
     :rtype: str
     """
-    return 'do some magic!'
+    return 'Not yet implemented'

swagger_server/controllers/extract_controller.py

@@ -18,4 +18,4 @@ def get_candidates(body):  # noqa: E501
     """
     if connexion.request.is_json:
         body = IzlusciBody.from_dict(connexion.request.get_json())  # noqa: E501
-    return 'do some magic!'
+    return 'do some magic!3'

swagger_server/controllers/oss_controller.py

@@ -8,22 +8,6 @@ from flask import send_file
 from swagger_server.db_utils import Ngrams_Manager
 
 
-def get_conllu(file_id):  # noqa: E501
-    """Vrne CoNNL-U po id-ju datoteke
-
-     # noqa: E501
-
-    :param file_id:
-    :type file_id: int
-
-    :rtype: str
-    """
-    try:
-        return send_file(util.get_conllu_path_by_id(file_id), attachment_filename=f'{file_id}.conllu')
-    except FileNotFoundError as e:
-        return "The conllu with this ID doesn't exist.", 404
-
-
 def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
     """Vrne seznam CoNNL-U-jev glede na iskalne pogoje
 
@@ -40,7 +24,12 @@ def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
 
     :rtype: List[str]
     """
-    return 'do some magic!'
+    if not kljucnebesede:
+        return "Manjkajo kljucne besede", 400
+    files = util.get_files_by_keywords(kljucnebesede)
+    if not files:
+        return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
+    return ' '.join(files), 200
 
 
 def get_extracted_words(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
@@ -59,20 +48,7 @@ def get_extracted_words(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E50
 
     :rtype: List[TerminoloskiKandidat]
     """
-    return 'do some magic!'
+    return 'do some magic!5'
-
-
-def get_file(file_id):  # noqa: E501
-    """Vrne binarni zapis v originalnem formatu po id-ju datoteke
-
-     # noqa: E501
-
-    :param file_id:
-    :type file_id: int
-
-    :rtype: List[bytearray]
-    """
-    return 'do some magic!'
 
 
 def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
@@ -91,7 +67,12 @@ def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
 
     :rtype: List[List[bytearray]]
     """
-    return 'do some magic!'
+    if not kljucnebesede:
+        return "Manjkajo kljucne besede", 400
+    files = util.get_files_by_keywords(kljucnebesede)
+    if not files:
+        return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
+    return ' '.join(files), 200
 
 
 def get_number_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
@@ -110,7 +91,10 @@ def get_number_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
 
     :rtype: int
     """
-    return 'do some magic!'
+    if not kljucnebesede:
+        return "Manjkajo kljucne besede", 400
+    files = util.get_files_by_keywords(kljucnebesede)
+    return len(files), 200
 
 
 def get_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
@@ -129,7 +113,44 @@ def get_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
 
     :rtype: List[str]
     """
-    return 'do some magic!'
+    if not kljucnebesede:
+        return "Manjkajo kljucne besede", 400
+    files = util.get_files_by_keywords(kljucnebesede)
+    if not files:
+        return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
+    return ' '.join(files), 200
+
+
+def get_conllu(file_id):  # noqa: E501
+    """Vrne CoNNL-U po id-ju datoteke
+
+     # noqa: E501
+
+    :param file_id:
+    :type file_id: int
+
+    :rtype: str
+    """
+    try:
+        return send_file(util.get_conllu_file_path_by_id(file_id), attachment_filename=f'{file_id}.conllu')
+    except FileNotFoundError as e:
+        return "The conllu with this ID doesn't exist.", 404
+
+
+def get_file(file_id):  # noqa: E501
+    """Vrne binarni zapis v originalnem formatu po id-ju datoteke
+
+     # noqa: E501
+
+    :param file_id:
+    :type file_id: int
+
+    :rtype: List[bytearray]
+    """
+    try:
+        return send_file(util.get_original_file_path_by_id(file_id), attachment_filename=f'{file_id}.xml')
+    except FileNotFoundError as e:
+        return "The file with this ID doesn't exist.", 404
 
 
 def oss_besedilo_po_id_get(file_id):  # noqa: E501
@@ -143,6 +164,6 @@ def oss_besedilo_po_id_get(file_id):  # noqa: E501
     :rtype: str
     """
     try:
-        return send_file(util.get_original_file_by_id(file_id), attachment_filename=f'{file_id}.conllu')
+        return send_file(util.get_original_file_path_by_id(file_id), attachment_filename=f'{file_id}.xml')
     except FileNotFoundError as e:
-        return "The conllu with this ID doesn't exist.", 404
+        return "The file with this ID doesn't exist.", 404

swagger_server/util.py

@@ -1,8 +1,10 @@
 import datetime
+import os.path
 
 import six
 import typing
 from swagger_server import type_util
+import pandas as pd
 
 
 def _deserialize(data, klass):
@@ -142,9 +144,43 @@ def _deserialize_dict(data, boxed_type):
             for k, v in six.iteritems(data)}
 
 
-def get_conllu_path_by_id(file_id):
+def is_docker() -> bool:
-    return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.plainText.conllu'
+    # todo: better way of checking if we're on docker or if we're in the develoment enviroment
+    return not os.path.exists('.env')
 
 
-def get_original_file_by_id(file_id):
+def get_conllu_file_path_by_id(file_id):
-    return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.xml'
+    r = f'classla_OS2022/conll/rsdo_doc-{file_id}.plainText.conllu'
+    if is_docker():
+        return r
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_original_file_path_by_id(file_id):
+    r = f'classla_OS2022/besedila/rsdo_doc-{file_id}.xml'
+    if is_docker():
+        return r
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_tei_file_path_by_id(file_id):
+    r = f'classla_OS2022/tei/rsdo_doc-{file_id}.plainText.tei.xml'
+    if is_docker():
+        return r
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_files_by_keywords(kljucnebesede):
+    ret = []
+    kljucnebesede = [k.lower() for k in kljucnebesede]
+    # Temporary solution until connection with mariadb is fixed
+    ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/"
+    for path, dirs, files, in os.walk(ngrams_path):
+        for _file in files:
+            file = f'{ngrams_path}{_file}'
+            data = pd.read_csv(file, sep='\t')
+            amount = len(data[data['ngram_len'] == 1 & data['gram_text'].str.lower().isin(kljucnebesede)])
+            # this should be 1
+            if amount >= 1:
+                ret.append(_file[9:][:-12])
+    return ret