From 62a6539fa4f3c0e6e276ade8a6092cf59439e81e Mon Sep 17 00:00:00 2001 From: "marko.ferme" Date: Thu, 12 Jan 2023 14:24:29 +0100 Subject: [PATCH] db fix --- swagger_server/swagger/swagger.yaml | 86 ++++++++++++-- swagger_server/utils/db_utils.py | 174 ++++++++++++++++++++++++++-- 2 files changed, 246 insertions(+), 14 deletions(-) diff --git a/swagger_server/swagger/swagger.yaml b/swagger_server/swagger/swagger.yaml index af35945..92b92cf 100644 --- a/swagger_server/swagger/swagger.yaml +++ b/swagger_server/swagger/swagger.yaml @@ -311,7 +311,7 @@ paths: items: type: integer format: int64 - - name: kljucnebesede + - name: kljucneBesede in: query required: false style: form @@ -339,6 +339,60 @@ paths: format: int64 x-content-type: '*/*' x-openapi-router-controller: swagger_server.controllers.oss_controller + post: + tags: + - oss-post + summary: Vrne število besedil glede na iskalne pogoje + operationId: get_number_texts_post + parameters: + - name: leta + in: query + required: false + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 + - name: vrste + in: query + required: false + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 + - name: kljucneBesede + in: query + required: false + style: form + explode: true + schema: + type: array + items: + type: string + - name: udk + in: query + required: false + style: form + explode: true + schema: + type: array + items: + type: string + responses: + "200": + description: OK + content: + '*/*': + schema: + type: integer + format: int64 + x-content-type: '*/*' + x-openapi-router-controller: swagger_server.controllers.oss_post_controller /oss/izlusciPoIskanju: get: tags: @@ -366,7 +420,16 @@ paths: items: type: integer format: int64 - - name: kljucnebesede + - name: kljucneBesede + in: query + required: false + style: form + explode: true + schema: + type: array + items: + type: string + - name: prepovedaneBesede in: query required: false style: form @@ -427,7 +490,16 @@ paths: items: type: integer format: int64 - - name: kljucnebesede + - name: kljucneBesede + in: query + required: false + style: form + explode: true + schema: + type: array + items: + type: string + - name: prepovedaneBesede in: query required: false style: form @@ -712,7 +784,7 @@ paths: type: array items: type: string - - name: kljucnebesede + - name: kljucneBesede in: query required: true style: form @@ -816,7 +888,7 @@ paths: type: array items: type: string - - name: kljucnebesede + - name: kljucneBesede in: query required: true style: form @@ -956,7 +1028,7 @@ components: type: array items: type: string - prepovedaneBesede: + prepovedanebesede: type: array items: type: string @@ -969,7 +1041,7 @@ components: type: array items: type: string - prepovedaneBesede: + prepovedanebesede: type: array items: type: string diff --git a/swagger_server/utils/db_utils.py b/swagger_server/utils/db_utils.py index 2a076bf..bd29809 100644 --- a/swagger_server/utils/db_utils.py +++ b/swagger_server/utils/db_utils.py @@ -39,7 +39,64 @@ def get_files_by_udc(udc): return ret -def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk): +def vrni_oss_dokumente(leta, vrste, kljucne_besede, udk): + ret = [] + + try: + print(database_info) + conn = mariadb.connect(**database_info) + cur = conn.cursor() + + + + + sql="" + params=[] + if (udk): + where_in_udk = ','.join(['%s'] * len(udk)) + sql=sql+ "(select distinct document_id from metadata_150k where parameter='udc' and _value IN (%s) )" % (where_in_udk) + params=udk + + if (leta): + where_in_leta = ','.join(['%s'] * len(leta)) + if (sql): + sql=sql+ " INTERSECT " + sql=sql+ "(select distinct document_id from metadata_150k where parameter='leto' and _value IN (%s) )" % (where_in_leta) + params=params+leta + + if (vrste): + where_in_vrste = ','.join(['%s'] * len(vrste)) + if (sql): + sql=sql+ " INTERSECT " + sql=sql+ "(select distinct document_id from metadata_150k where parameter='typology' and _value IN (%s) )" % (where_in_vrste) + params=params+vrste + + if (kljucne_besede): + if (sql): + sql=sql+ " INTERSECT " + where_in_kb = ','.join(['%s'] * len(kljucne_besede)) + sql=sql+ "(select distinct document_id from metadata_150k where parameter='kljucnabeseda' and _value IN (%s) )" % (where_in_kb) + params=params+kljucne_besede + + + + print(sql) + print(params) + + + if(sql): + cur.execute(sql+";",params) + ret = list(cur.fetchall()) + + except mariadb.Error as e: + print(f"Error connecting to MariaDB Platform: {e}") + finally: + cur.close(); + conn.close(); + + return ret + +def vrni_oss_dokumente_old(leta, vrste, kljucne_besede, udk): ret = [] try: @@ -58,7 +115,6 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk): params=udk if (leta): - where_in_leta = ','.join(['%s'] * len(leta)) if (where): where=where+ " AND " @@ -72,12 +128,12 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk): where=where + " tipologija IN (%s) " % (where_in_vrste) params=params+vrste - if (kljucnebesede): + if (kljucne_besede): if (where): where=where+ " AND " - where_in_kb = ','.join(['%s'] * len(kljucnebesede)) + where_in_kb = ','.join(['%s'] * len(kljucne_besede)) where=where + " kljucnabeseda IN (%s) " % (where_in_kb) - params=params+kljucnebesede + params=params+kljucne_besede if(where): sql=sql+" where " + where + ";" @@ -97,7 +153,7 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk): return ret -def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_besede, udk,definicije=False): +def vrni_oss_terminoloske_kandidate_old(leta, vrste, kljucnebesede, prepovedane_besede, udk,definicije=False): ret = [] try: @@ -116,7 +172,6 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_bese params=udk if (leta): - where_in_leta = ','.join(['%s'] * len(leta)) if (where): where=where+ " AND " @@ -216,6 +271,111 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_bese return ret +def vrni_oss_terminoloske_kandidate(leta, vrste, kljucne_besede, prepovedane_besede, udk,definicije=False): + ret = {'terminoloski_kandidati': [] } + + try: + + + dokumenti=vrni_oss_dokumente(leta,vrste,kljucne_besede,udk); + print(list(zip(*dokumenti))[0]) + dokumenti=list(zip(*dokumenti))[0] + print(database_info) + conn = mariadb.connect(**database_info) + cur = conn.cursor(dictionary=True) + + + + where_in_doc="" + params=[] + if (dokumenti): + where_in_doc = ','.join(['%s'] * len(dokumenti)) + + + + + + sqltk=f"""Select ngram,upos,convert(avg(tfidf),FLOAT) as tfidf, convert(sum(tf),INT) as tf from ( + SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf + FROM ngrams_upos_tf tf, documents d, + ( + Select ngram, upos, count(*) as df from ngrams_upos_tf TF + where document_id in + ({where_in_doc}) + group by TF.ngram, TF.upos + ) dff, ngrams_upos_df df + where + tf.document_id=d.document_id and + df.ngram=tf.ngram AND df.upos=tf.upos and + dff.ngram=tf.ngram AND dff.upos=tf.upos + ) X + group by ngram,upos + order by tfidf desc + limit 1000;""" + # + #sqltk=f"""select ngram,upos,convert(1.0,float) as tfidf,%s as tf from ngrams_upos_tf limit 10;""" + + print (sqltk) + print (dokumenti) + #še prepovedane besede ven + if(where_in_doc): + start_time = time.time() + cur.execute(sqltk,dokumenti) + terms=cur.fetchall() + print("Čas poizbedbe je %.2f sekund" % (time.time() - start_time)) + else: + return ret + + print (terms); + #ret = list(cur) + can = {'forms':[ + ngram["ngram"] + for ngram in terms + ] + } + print (can); + res = requests.post(canonapi_endpoint, json=can) + + data = res.json() + print (data); + print (data.get("canonical_forms")); + print (terms); + print(zip(data.get("canonical_forms"),terms)) + + + + + ret['terminoloski_kandidati']= [ + { + 'POSoznake': x.get("upos"), + 'kandidat': x.get("ngram"), # more to bit lemma al terms? + 'definicija': None, + 'kanonicnaoblika': d, + 'ranking': x.get('tfidf'), + 'podporneutezi': [ + 0.0, # ???????? + 0.0 # ?????? + ], + 'pogostostpojavljanja': [x.get('tf'), 0] # ??????? + } + for (d,x) in zip(data.get("canonical_forms"),terms) + ] + + #if definicije + #idi z variablo sql po id-je dokumentov, preberi conlluje iz diska + #naredi en vlki conllu + #pokliči metodo + + + except mariadb.Error as e: + print(f"Error connecting to MariaDB Platform: {e}") + finally: + cur.close(); + conn.close(); + + return ret + + # class BaseModel(Model): # class Meta: # database = db