webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dodani popravki za iskanje kandidatov in števila besedil. Dodana logika za klicanje ocrja. Popravljen bug za timeout v ateapi-ju. Popravljen swager.

Browse Source
This commit is contained in:
Marko Ferme
2022-10-19 04:07:15 +02:00
parent d44a268b24
commit 107e5fd4d3
6 changed files with 132 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ATEapi/Dockerfile
+1 -1
View File
@@ -28,4 +28,4 @@ RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /
USER appuser
# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "main:app"]
CMD ["gunicorn", "-t 0", "--bind", "0.0.0.0:5000", "main:app"]
docker-compose.yml
+1 -1
View File
@@ -11,7 +11,7 @@ services:
- ../classla/classla_resources:/root/classla_resources
environment:
- PYTHONUNBUFFERED=1
- MDB_DATABASE=corpus_120k
- MDB_DATABASE=conllus_150k
- MDB_HOST=164.8.252.72
- MDB_PORT=3306
- MDB_USER=rsdo5
swagger_server/controllers/oss_controller.py
+3 -1
View File
@@ -43,7 +43,9 @@ def get_extracted_words(leta=None, vrste=None, kljucnebesede=None, udk=None): #
:rtype: List[TerminoloskiKandidat]
"""
return 'do some magic!5'
files = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk)
return files, 200
def get_files(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501
swagger_server/swagger/swagger.yaml
+4 -4
View File
@@ -348,7 +348,7 @@ paths:
parameters:
- name: leta
in: query
required: true
required: false
style: form
explode: true
schema:
@@ -358,7 +358,7 @@ paths:
format: int64
- name: vrste
in: query
required: true
required: false
style: form
explode: true
schema:
@@ -368,7 +368,7 @@ paths:
format: int64
- name: kljucnebesede
in: query
required: true
required: false
style: form
explode: true
schema:
@@ -377,7 +377,7 @@ paths:
type: string
- name: udk
in: query
required: true
required: false
style: form
explode: true
schema:
swagger_server/utils/db_utils.py
+77
View File
@@ -79,6 +79,8 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
print(sql)
print(params)
cur.execute(sql,params)
ret = list(cur)
@@ -89,6 +91,81 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
return ret
def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
ret = []
try:
print(database_info)
conn = mariadb.connect(**database_info)
cur = conn.cursor()
sql = "select distinct document_id from metadata"
where=""
params=[]
if (udk):
where_in_udk = ','.join(['%s'] * len(udk))
where=" udk IN (%s) " % (where_in_udk)
params=udk
if (leta):
where_in_leta = ','.join(['%s'] * len(leta))
if (where):
where=where+ " AND "
where=where + " leto IN (%s) " % (where_in_leta)
params=params+leta
if (vrste):
if (where):
where=where+ " AND "
where_in_vrste = ','.join(['%s'] * len(vrste))
where=where + " tipologija IN (%s) " % (where_in_vrste)
params=params+vrste
if (kljucnebesede):
if (where):
where=where+ " AND "
where_in_kb = ','.join(['%s'] * len(kljucnebesede))
where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
params=params+kljucnebesede
if(where):
sql=sql+" where " + where
print(sql)
print(params)
sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf from (
SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf
FROM ngrams_upos_tf tf, documents d,
(
Select ngram, upos, count(*) as df from ngrams_upos_tf TF
where document_id in
({sql})
group by TF.ngram, TF.upos
) dff, ngrams_upos_df df
where
tf.document_id=d.document_id and
df.ngram=tf.ngram AND df.upos=tf.upos and
dff.ngram=tf.ngram AND dff.upos=tf.upos
) X
group by ngram,upos
order by tfidf desc
limit 100"""
print (sqltk)
cur.execute(sqltk,params)
ret = list(cur)
except mariadb.Error as e:
print(f"Error connecting to MariaDB Platform: {e}")
return ret
# class BaseModel(Model):
# class Meta:
# database = db
swagger_server/utils/txt_utils.py
+46 -20
View File
@@ -9,7 +9,7 @@ from swagger_server.utils import cl_utils
import cv2
import numpy as np
import magic
import re
tika_server = "http://tika2:9999/tika"
# endpoint below to be used only for development purposes (don't need to run docker)
@@ -26,28 +26,54 @@ def extract_text_prepResp(file, content_type=""):
try:
response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
content = response.text
#preveri če je pretvorba uspešna
# original string
res = re.findall(r'\w+', content)
#preveri, če imamo vsaj 10 besed in če je povprečna dolžina >3 in < 12
#če to drži, idi v ocr
reslen=map(lambda n:len(n),res)
print(f"Število besed je {len(res)}")
if len(res)>0 :
avglen=sum(reslen)/len(res)
else:
avglen=0
print(f"Povprečna dolžina besede je {avglen}")
if(len(res)<10 or avglen<4 or avglen>11):
print("Besedilo je sumljivo, gremo v OCR in damo file na začetek!")
file.seek(0)
response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
"Accept": "text/plain; charset=UTF-8"})
content = response.text
#odstranim še vse prelome vrstic, ker imamo s tem probleme
content=' '.join(content.splitlines())
except:
content = "ERROR - something went wrong when reading file with tika"
if content == "":
if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
elif "application/pdf" in content_type:
reader = PdfReader(file)
content = '\n'.join([p.extract_text() for p in reader.pages])
content = content
elif "text/xml" in content_type:
root = ET.parse(file).getroot()
plainText = root.findall('PlainText')
if len(plainText) == 0:
return "Didn't find anything in PlainText", 400
content = '\n'.join([pt.text for pt in plainText])
# elif "text/plain" in file.content_type:
else:
try:
content = file.read().decode('utf-8')
except:
content = "ERROR - something went wrong when reading file with not-tika method!"
#if content == "":
# if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
# content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
# elif "application/pdf" in content_type:
# reader = PdfReader(file)
# content = '\n'.join([p.extract_text() for p in reader.pages])
# content = content
# elif "text/xml" in content_type:
# root = ET.parse(file).getroot()
# plainText = root.findall('PlainText')
# if len(plainText) == 0:
# return "Didn't find anything in PlainText", 400
# content = '\n'.join([pt.text for pt in plainText])
# # elif "text/plain" in file.content_type:
# else:
# try:
# content = file.read().decode('utf-8')
# except:
# content = "ERROR - something went wrong when reading file with not-tika method!"
return content, 200