diff --git a/ATEapi/Dockerfile b/ATEapi/Dockerfile index d151b59..45fee83 100644 --- a/ATEapi/Dockerfile +++ b/ATEapi/Dockerfile @@ -28,4 +28,4 @@ RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser / USER appuser # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug -CMD ["gunicorn", "--bind", "0.0.0.0:5000", "main:app"] +CMD ["gunicorn", "-t 0", "--bind", "0.0.0.0:5000", "main:app"] diff --git a/docker-compose.yml b/docker-compose.yml index ab8e9d2..f84d117 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - ../classla/classla_resources:/root/classla_resources environment: - PYTHONUNBUFFERED=1 - - MDB_DATABASE=corpus_120k + - MDB_DATABASE=conllus_150k - MDB_HOST=164.8.252.72 - MDB_PORT=3306 - MDB_USER=rsdo5 diff --git a/swagger_server/controllers/oss_controller.py b/swagger_server/controllers/oss_controller.py index 35043ca..d2d56f5 100644 --- a/swagger_server/controllers/oss_controller.py +++ b/swagger_server/controllers/oss_controller.py @@ -43,7 +43,9 @@ def get_extracted_words(leta=None, vrste=None, kljucnebesede=None, udk=None): # :rtype: List[TerminoloskiKandidat] """ - return 'do some magic!5' + files = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk) + return files, 200 + def get_files(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 diff --git a/swagger_server/swagger/swagger.yaml b/swagger_server/swagger/swagger.yaml index 4ef9144..6966b1b 100644 --- a/swagger_server/swagger/swagger.yaml +++ b/swagger_server/swagger/swagger.yaml @@ -348,7 +348,7 @@ paths: parameters: - name: leta in: query - required: true + required: false style: form explode: true schema: @@ -358,7 +358,7 @@ paths: format: int64 - name: vrste in: query - required: true + required: false style: form explode: true schema: @@ -368,7 +368,7 @@ paths: format: int64 - name: kljucnebesede in: query - required: true + required: false style: form explode: true schema: @@ -377,7 +377,7 @@ paths: type: string - name: udk in: query - required: true + required: false style: form explode: true schema: diff --git a/swagger_server/utils/db_utils.py b/swagger_server/utils/db_utils.py index b34be74..e85d0fc 100644 --- a/swagger_server/utils/db_utils.py +++ b/swagger_server/utils/db_utils.py @@ -78,6 +78,8 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk): print(sql) print(params) + + cur.execute(sql,params) @@ -89,6 +91,81 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk): return ret +def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk): + ret = [] + + try: + print(database_info) + conn = mariadb.connect(**database_info) + cur = conn.cursor() + + + + sql = "select distinct document_id from metadata" + where="" + params=[] + if (udk): + where_in_udk = ','.join(['%s'] * len(udk)) + where=" udk IN (%s) " % (where_in_udk) + params=udk + + if (leta): + + where_in_leta = ','.join(['%s'] * len(leta)) + if (where): + where=where+ " AND " + where=where + " leto IN (%s) " % (where_in_leta) + params=params+leta + + if (vrste): + if (where): + where=where+ " AND " + where_in_vrste = ','.join(['%s'] * len(vrste)) + where=where + " tipologija IN (%s) " % (where_in_vrste) + params=params+vrste + + if (kljucnebesede): + if (where): + where=where+ " AND " + where_in_kb = ','.join(['%s'] * len(kljucnebesede)) + where=where + " kljucnabeseda IN (%s) " % (where_in_kb) + params=params+kljucnebesede + + if(where): + sql=sql+" where " + where + + print(sql) + print(params) + + sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf from ( + SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf + FROM ngrams_upos_tf tf, documents d, + ( + Select ngram, upos, count(*) as df from ngrams_upos_tf TF + where document_id in + ({sql}) + group by TF.ngram, TF.upos + ) dff, ngrams_upos_df df + where + tf.document_id=d.document_id and + df.ngram=tf.ngram AND df.upos=tf.upos and + dff.ngram=tf.ngram AND dff.upos=tf.upos + ) X + group by ngram,upos + order by tfidf desc + limit 100""" + + print (sqltk) + + cur.execute(sqltk,params) + + ret = list(cur) + except mariadb.Error as e: + print(f"Error connecting to MariaDB Platform: {e}") + + + return ret + # class BaseModel(Model): # class Meta: # database = db diff --git a/swagger_server/utils/txt_utils.py b/swagger_server/utils/txt_utils.py index 3df799a..ed4da5b 100644 --- a/swagger_server/utils/txt_utils.py +++ b/swagger_server/utils/txt_utils.py @@ -9,7 +9,7 @@ from swagger_server.utils import cl_utils import cv2 import numpy as np import magic - +import re tika_server = "http://tika2:9999/tika" # endpoint below to be used only for development purposes (don't need to run docker) @@ -26,28 +26,54 @@ def extract_text_prepResp(file, content_type=""): try: response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"}) content = response.text + #preveri če je pretvorba uspešna + + # original string + res = re.findall(r'\w+', content) + + #preveri, če imamo vsaj 10 besed in če je povprečna dolžina >3 in < 12 + #če to drži, idi v ocr + reslen=map(lambda n:len(n),res) + print(f"Število besed je {len(res)}") + + if len(res)>0 : + avglen=sum(reslen)/len(res) + else: + avglen=0 + + print(f"Povprečna dolžina besede je {avglen}") + + if(len(res)<10 or avglen<4 or avglen>11): + print("Besedilo je sumljivo, gremo v OCR in damo file na začetek!") + file.seek(0) + response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng", + "Accept": "text/plain; charset=UTF-8"}) + content = response.text + + #odstranim še vse prelome vrstic, ker imamo s tem probleme + content=' '.join(content.splitlines()) except: content = "ERROR - something went wrong when reading file with tika" - if content == "": - if "openxmlformats-officedocument.wordprocessingml.document" in content_type: - content = '\n'.join([p.text for p in docx.Document(file).paragraphs]) - elif "application/pdf" in content_type: - reader = PdfReader(file) - content = '\n'.join([p.extract_text() for p in reader.pages]) - content = content - elif "text/xml" in content_type: - root = ET.parse(file).getroot() - plainText = root.findall('PlainText') - if len(plainText) == 0: - return "Didn't find anything in PlainText", 400 - content = '\n'.join([pt.text for pt in plainText]) - # elif "text/plain" in file.content_type: - else: - try: - content = file.read().decode('utf-8') - except: - content = "ERROR - something went wrong when reading file with not-tika method!" + #if content == "": + # if "openxmlformats-officedocument.wordprocessingml.document" in content_type: + # content = '\n'.join([p.text for p in docx.Document(file).paragraphs]) + # elif "application/pdf" in content_type: + # reader = PdfReader(file) + # content = '\n'.join([p.extract_text() for p in reader.pages]) + # content = content + # elif "text/xml" in content_type: + # root = ET.parse(file).getroot() + # plainText = root.findall('PlainText') + # if len(plainText) == 0: + # return "Didn't find anything in PlainText", 400 + # content = '\n'.join([pt.text for pt in plainText]) + # # elif "text/plain" in file.content_type: + # else: + # try: + # content = file.read().decode('utf-8') + # except: + # content = "ERROR - something went wrong when reading file with not-tika method!" return content, 200