Dodani popravki za iskanje kandidatov in števila besedil. Dodana logika za klicanje ocrja. Popravljen bug za timeout v ateapi-ju. Popravljen swager.
This commit is contained in:
Marko Ferme
@@ -78,6 +78,8 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
|
||||
|
||||
print(sql)
|
||||
print(params)
|
||||
|
||||
|
||||
|
||||
cur.execute(sql,params)
|
||||
|
||||
@@ -89,6 +91,81 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
|
||||
return ret
|
||||
|
||||
|
||||
def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
|
||||
ret = []
|
||||
|
||||
try:
|
||||
print(database_info)
|
||||
conn = mariadb.connect(**database_info)
|
||||
cur = conn.cursor()
|
||||
|
||||
|
||||
|
||||
sql = "select distinct document_id from metadata"
|
||||
where=""
|
||||
params=[]
|
||||
if (udk):
|
||||
where_in_udk = ','.join(['%s'] * len(udk))
|
||||
where=" udk IN (%s) " % (where_in_udk)
|
||||
params=udk
|
||||
|
||||
if (leta):
|
||||
|
||||
where_in_leta = ','.join(['%s'] * len(leta))
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where=where + " leto IN (%s) " % (where_in_leta)
|
||||
params=params+leta
|
||||
|
||||
if (vrste):
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where_in_vrste = ','.join(['%s'] * len(vrste))
|
||||
where=where + " tipologija IN (%s) " % (where_in_vrste)
|
||||
params=params+vrste
|
||||
|
||||
if (kljucnebesede):
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where_in_kb = ','.join(['%s'] * len(kljucnebesede))
|
||||
where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
|
||||
params=params+kljucnebesede
|
||||
|
||||
if(where):
|
||||
sql=sql+" where " + where
|
||||
|
||||
print(sql)
|
||||
print(params)
|
||||
|
||||
sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf from (
|
||||
SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf
|
||||
FROM ngrams_upos_tf tf, documents d,
|
||||
(
|
||||
Select ngram, upos, count(*) as df from ngrams_upos_tf TF
|
||||
where document_id in
|
||||
({sql})
|
||||
group by TF.ngram, TF.upos
|
||||
) dff, ngrams_upos_df df
|
||||
where
|
||||
tf.document_id=d.document_id and
|
||||
df.ngram=tf.ngram AND df.upos=tf.upos and
|
||||
dff.ngram=tf.ngram AND dff.upos=tf.upos
|
||||
) X
|
||||
group by ngram,upos
|
||||
order by tfidf desc
|
||||
limit 100"""
|
||||
|
||||
print (sqltk)
|
||||
|
||||
cur.execute(sqltk,params)
|
||||
|
||||
ret = list(cur)
|
||||
except mariadb.Error as e:
|
||||
print(f"Error connecting to MariaDB Platform: {e}")
|
||||
|
||||
|
||||
return ret
|
||||
|
||||
# class BaseModel(Model):
|
||||
# class Meta:
|
||||
# database = db
|
||||
|
||||
@@ -9,7 +9,7 @@ from swagger_server.utils import cl_utils
|
||||
import cv2
|
||||
import numpy as np
|
||||
import magic
|
||||
|
||||
import re
|
||||
tika_server = "http://tika2:9999/tika"
|
||||
|
||||
# endpoint below to be used only for development purposes (don't need to run docker)
|
||||
@@ -26,28 +26,54 @@ def extract_text_prepResp(file, content_type=""):
|
||||
try:
|
||||
response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
|
||||
content = response.text
|
||||
#preveri če je pretvorba uspešna
|
||||
|
||||
# original string
|
||||
res = re.findall(r'\w+', content)
|
||||
|
||||
#preveri, če imamo vsaj 10 besed in če je povprečna dolžina >3 in < 12
|
||||
#če to drži, idi v ocr
|
||||
reslen=map(lambda n:len(n),res)
|
||||
print(f"Število besed je {len(res)}")
|
||||
|
||||
if len(res)>0 :
|
||||
avglen=sum(reslen)/len(res)
|
||||
else:
|
||||
avglen=0
|
||||
|
||||
print(f"Povprečna dolžina besede je {avglen}")
|
||||
|
||||
if(len(res)<10 or avglen<4 or avglen>11):
|
||||
print("Besedilo je sumljivo, gremo v OCR in damo file na začetek!")
|
||||
file.seek(0)
|
||||
response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
|
||||
"Accept": "text/plain; charset=UTF-8"})
|
||||
content = response.text
|
||||
|
||||
#odstranim še vse prelome vrstic, ker imamo s tem probleme
|
||||
content=' '.join(content.splitlines())
|
||||
except:
|
||||
content = "ERROR - something went wrong when reading file with tika"
|
||||
|
||||
if content == "":
|
||||
if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
|
||||
content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
|
||||
elif "application/pdf" in content_type:
|
||||
reader = PdfReader(file)
|
||||
content = '\n'.join([p.extract_text() for p in reader.pages])
|
||||
content = content
|
||||
elif "text/xml" in content_type:
|
||||
root = ET.parse(file).getroot()
|
||||
plainText = root.findall('PlainText')
|
||||
if len(plainText) == 0:
|
||||
return "Didn't find anything in PlainText", 400
|
||||
content = '\n'.join([pt.text for pt in plainText])
|
||||
# elif "text/plain" in file.content_type:
|
||||
else:
|
||||
try:
|
||||
content = file.read().decode('utf-8')
|
||||
except:
|
||||
content = "ERROR - something went wrong when reading file with not-tika method!"
|
||||
#if content == "":
|
||||
# if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
|
||||
# content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
|
||||
# elif "application/pdf" in content_type:
|
||||
# reader = PdfReader(file)
|
||||
# content = '\n'.join([p.extract_text() for p in reader.pages])
|
||||
# content = content
|
||||
# elif "text/xml" in content_type:
|
||||
# root = ET.parse(file).getroot()
|
||||
# plainText = root.findall('PlainText')
|
||||
# if len(plainText) == 0:
|
||||
# return "Didn't find anything in PlainText", 400
|
||||
# content = '\n'.join([pt.text for pt in plainText])
|
||||
# # elif "text/plain" in file.content_type:
|
||||
# else:
|
||||
# try:
|
||||
# content = file.read().decode('utf-8')
|
||||
# except:
|
||||
# content = "ERROR - something went wrong when reading file with not-tika method!"
|
||||
|
||||
return content, 200
|
||||
|
||||
|
||||
Reference in New Issue
Block a user