diff --git a/requirements.txt b/requirements.txt index cbf51e0..c5fe82a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,8 @@ swagger-ui-bundle >= 0.0.2 peewee==3.15.0 mariadb==1.0.11 classla==1.1.0 -python-decouple==3.6 \ No newline at end of file +python-decouple==3.6 +pandas==1.3.3 +python-docx==0.8.11 +lxml==4.8.0 +PyPDF2==2.10.4 \ No newline at end of file diff --git a/swagger_server/__main__.py b/swagger_server/__main__.py index 6404cae..a00597f 100644 --- a/swagger_server/__main__.py +++ b/swagger_server/__main__.py @@ -8,7 +8,6 @@ def main(): app = connexion.App(__name__, specification_dir='./swagger/') app.app.json_encoder = encoder.JSONEncoder app.add_api('swagger.yaml', arguments={'title': 'OpenAPI definition'}, pythonic_params=True) - print("Starting app4") app.run(port=8080) diff --git a/swagger_server/controllers/doc2text_controller.py b/swagger_server/controllers/doc2text_controller.py index 141d21f..920846a 100644 --- a/swagger_server/controllers/doc2text_controller.py +++ b/swagger_server/controllers/doc2text_controller.py @@ -1,7 +1,13 @@ import connexion import six +# import tika from swagger_server import util +import pandas as pd +import docx +import codecs +import xml.etree.ElementTree as ET +from PyPDF2 import PdfReader def datoteka_v_besedilo_post(file=None): # noqa: E501 @@ -14,7 +20,27 @@ def datoteka_v_besedilo_post(file=None): # noqa: E501 :rtype: str """ - return 'do some magic!' + + if file is None: + return "No file provided", 400 + elif "openxmlformats-officedocument.wordprocessingml.document" in file.content_type: + content = [p.text for p in docx.Document(file).paragraphs] + elif "text/plain" in file.content_type: + content = file.read().decode('utf-8') + elif "application/pdf" in file.content_type: + reader = PdfReader(file) + content = '\n'.join([p.extract_text() for p in reader.pages]) + content = "ZACASNO UPORABLJEN DRUGI BRALEC KOT TIKA, TA BO SE DODANA KASNEJE...\n\n" + content + elif "text/xml" in file.content_type: + root = ET.parse(file).getroot() + plainText = root.findall('PlainText') + if len(plainText) == 0: + return "Didn't find anything in PlainText", 400 + content = '\n'.join([pt.text for pt in plainText]) + else: + return "Currently supporing only docx, txt/plain, pdf, xml*... more will be added later", 501 + + return content, 200 def get_text_ocr(file=None): # noqa: E501 @@ -27,4 +53,4 @@ def get_text_ocr(file=None): # noqa: E501 :rtype: str """ - return 'do some magic!' + return 'Not yet implemented' diff --git a/swagger_server/controllers/extract_controller.py b/swagger_server/controllers/extract_controller.py index be327ff..c0006c3 100644 --- a/swagger_server/controllers/extract_controller.py +++ b/swagger_server/controllers/extract_controller.py @@ -18,4 +18,4 @@ def get_candidates(body): # noqa: E501 """ if connexion.request.is_json: body = IzlusciBody.from_dict(connexion.request.get_json()) # noqa: E501 - return 'do some magic!' + return 'do some magic!3' diff --git a/swagger_server/controllers/oss_controller.py b/swagger_server/controllers/oss_controller.py index a21d0d5..defb51d 100644 --- a/swagger_server/controllers/oss_controller.py +++ b/swagger_server/controllers/oss_controller.py @@ -8,22 +8,6 @@ from flask import send_file from swagger_server.db_utils import Ngrams_Manager -def get_conllu(file_id): # noqa: E501 - """Vrne CoNNL-U po id-ju datoteke - - # noqa: E501 - - :param file_id: - :type file_id: int - - :rtype: str - """ - try: - return send_file(util.get_conllu_path_by_id(file_id), attachment_filename=f'{file_id}.conllu') - except FileNotFoundError as e: - return "The conllu with this ID doesn't exist.", 404 - - def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 """Vrne seznam CoNNL-U-jev glede na iskalne pogoje @@ -40,7 +24,12 @@ def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 :rtype: List[str] """ - return 'do some magic!' + if not kljucnebesede: + return "Manjkajo kljucne besede", 400 + files = util.get_files_by_keywords(kljucnebesede) + if not files: + return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404 + return ' '.join(files), 200 def get_extracted_words(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 @@ -59,20 +48,7 @@ def get_extracted_words(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E50 :rtype: List[TerminoloskiKandidat] """ - return 'do some magic!' - - -def get_file(file_id): # noqa: E501 - """Vrne binarni zapis v originalnem formatu po id-ju datoteke - - # noqa: E501 - - :param file_id: - :type file_id: int - - :rtype: List[bytearray] - """ - return 'do some magic!' + return 'do some magic!5' def get_files(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 @@ -91,7 +67,12 @@ def get_files(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 :rtype: List[List[bytearray]] """ - return 'do some magic!' + if not kljucnebesede: + return "Manjkajo kljucne besede", 400 + files = util.get_files_by_keywords(kljucnebesede) + if not files: + return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404 + return ' '.join(files), 200 def get_number_texts(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 @@ -110,7 +91,10 @@ def get_number_texts(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 :rtype: int """ - return 'do some magic!' + if not kljucnebesede: + return "Manjkajo kljucne besede", 400 + files = util.get_files_by_keywords(kljucnebesede) + return len(files), 200 def get_texts(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 @@ -129,7 +113,44 @@ def get_texts(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 :rtype: List[str] """ - return 'do some magic!' + if not kljucnebesede: + return "Manjkajo kljucne besede", 400 + files = util.get_files_by_keywords(kljucnebesede) + if not files: + return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404 + return ' '.join(files), 200 + + +def get_conllu(file_id): # noqa: E501 + """Vrne CoNNL-U po id-ju datoteke + + # noqa: E501 + + :param file_id: + :type file_id: int + + :rtype: str + """ + try: + return send_file(util.get_conllu_file_path_by_id(file_id), attachment_filename=f'{file_id}.conllu') + except FileNotFoundError as e: + return "The conllu with this ID doesn't exist.", 404 + + +def get_file(file_id): # noqa: E501 + """Vrne binarni zapis v originalnem formatu po id-ju datoteke + + # noqa: E501 + + :param file_id: + :type file_id: int + + :rtype: List[bytearray] + """ + try: + return send_file(util.get_original_file_path_by_id(file_id), attachment_filename=f'{file_id}.xml') + except FileNotFoundError as e: + return "The file with this ID doesn't exist.", 404 def oss_besedilo_po_id_get(file_id): # noqa: E501 @@ -143,6 +164,6 @@ def oss_besedilo_po_id_get(file_id): # noqa: E501 :rtype: str """ try: - return send_file(util.get_original_file_by_id(file_id), attachment_filename=f'{file_id}.conllu') + return send_file(util.get_original_file_path_by_id(file_id), attachment_filename=f'{file_id}.xml') except FileNotFoundError as e: - return "The conllu with this ID doesn't exist.", 404 + return "The file with this ID doesn't exist.", 404 diff --git a/swagger_server/util.py b/swagger_server/util.py index 53e5aac..989375d 100644 --- a/swagger_server/util.py +++ b/swagger_server/util.py @@ -1,8 +1,10 @@ import datetime +import os.path import six import typing from swagger_server import type_util +import pandas as pd def _deserialize(data, klass): @@ -142,9 +144,43 @@ def _deserialize_dict(data, boxed_type): for k, v in six.iteritems(data)} -def get_conllu_path_by_id(file_id): - return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.plainText.conllu' +def is_docker() -> bool: + # todo: better way of checking if we're on docker or if we're in the develoment enviroment + return not os.path.exists('.env') -def get_original_file_by_id(file_id): - return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.xml' +def get_conllu_file_path_by_id(file_id): + r = f'classla_OS2022/conll/rsdo_doc-{file_id}.plainText.conllu' + if is_docker(): + return r + return f'../mnt/ssd/ds_ftp/{r}' + + +def get_original_file_path_by_id(file_id): + r = f'classla_OS2022/besedila/rsdo_doc-{file_id}.xml' + if is_docker(): + return r + return f'../mnt/ssd/ds_ftp/{r}' + + +def get_tei_file_path_by_id(file_id): + r = f'classla_OS2022/tei/rsdo_doc-{file_id}.plainText.tei.xml' + if is_docker(): + return r + return f'../mnt/ssd/ds_ftp/{r}' + + +def get_files_by_keywords(kljucnebesede): + ret = [] + kljucnebesede = [k.lower() for k in kljucnebesede] + # Temporary solution until connection with mariadb is fixed + ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/" + for path, dirs, files, in os.walk(ngrams_path): + for _file in files: + file = f'{ngrams_path}{_file}' + data = pd.read_csv(file, sep='\t') + amount = len(data[data['ngram_len'] == 1 & data['gram_text'].str.lower().isin(kljucnebesede)]) + # this should be 1 + if amount >= 1: + ret.append(_file[9:][:-12]) + return ret