db fix

swagger_server/utils/db_utils.py

@@ -39,7 +39,64 @@ def get_files_by_udc(udc):
 
     return ret
 
-def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
+def vrni_oss_dokumente(leta, vrste, kljucne_besede, udk):
+    ret = []
+  
+    try:
+        print(database_info)
+        conn = mariadb.connect(**database_info)
+        cur = conn.cursor()
+        
+        
+        
+        
+        sql=""
+        params=[]
+        if (udk):
+            where_in_udk = ','.join(['%s'] * len(udk))
+            sql=sql+ "(select distinct document_id from metadata_150k where parameter='udc' and _value IN (%s) )" % (where_in_udk)
+            params=udk
+        
+        if (leta):
+            where_in_leta = ','.join(['%s'] * len(leta))
+            if (sql):
+                sql=sql+ " INTERSECT "
+            sql=sql+ "(select distinct document_id from metadata_150k where parameter='leto' and _value IN (%s) )" % (where_in_leta)
+            params=params+leta
+        
+        if (vrste):
+            where_in_vrste = ','.join(['%s'] * len(vrste))
+            if (sql):
+                sql=sql+ " INTERSECT "
+            sql=sql+ "(select distinct document_id from metadata_150k where parameter='typology' and _value IN (%s) )" % (where_in_vrste)
+            params=params+vrste
+
+        if (kljucne_besede):
+            if (sql):
+                sql=sql+ " INTERSECT "
+            where_in_kb = ','.join(['%s'] * len(kljucne_besede))
+            sql=sql+ "(select distinct document_id from metadata_150k where parameter='kljucnabeseda' and _value IN (%s) )" % (where_in_kb)
+            params=params+kljucne_besede
+
+       
+        
+        print(sql)
+        print(params)
+
+
+        if(sql):
+            cur.execute(sql+";",params)
+            ret = list(cur.fetchall())
+
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB Platform: {e}")
+    finally:
+        cur.close();
+        conn.close();
+
+    return ret
+
+def vrni_oss_dokumente_old(leta, vrste, kljucne_besede, udk):
     ret = []
   
     try:
@@ -58,7 +115,6 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
             params=udk
         
         if (leta):
-
             where_in_leta = ','.join(['%s'] * len(leta))
             if (where):
                 where=where+ " AND "
@@ -72,12 +128,12 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
             where=where + " tipologija IN (%s) " % (where_in_vrste)
             params=params+vrste
 
-        if (kljucnebesede):
+        if (kljucne_besede):
             if (where):
                 where=where+ " AND "
-            where_in_kb = ','.join(['%s'] * len(kljucnebesede))
+            where_in_kb = ','.join(['%s'] * len(kljucne_besede))
             where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
-            params=params+kljucnebesede
+            params=params+kljucne_besede
 
         if(where):
             sql=sql+" where " + where + ";"
@@ -97,7 +153,7 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
     return ret
 
 
-def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_besede, udk,definicije=False):
+def vrni_oss_terminoloske_kandidate_old(leta, vrste, kljucnebesede, prepovedane_besede, udk,definicije=False):
     ret = []
   
     try:
@@ -116,7 +172,6 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_bese
             params=udk
         
         if (leta):
-
             where_in_leta = ','.join(['%s'] * len(leta))
             if (where):
                 where=where+ " AND "
@@ -216,6 +271,111 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_bese
 
     return ret
 
+def vrni_oss_terminoloske_kandidate(leta, vrste, kljucne_besede, prepovedane_besede, udk,definicije=False):
+    ret = {'terminoloski_kandidati': [] }
+  
+    try:
+
+
+        dokumenti=vrni_oss_dokumente(leta,vrste,kljucne_besede,udk);
+        print(list(zip(*dokumenti))[0])
+        dokumenti=list(zip(*dokumenti))[0]
+        print(database_info)
+        conn = mariadb.connect(**database_info)
+        cur = conn.cursor(dictionary=True)
+        
+        
+        
+        where_in_doc=""
+        params=[]
+        if (dokumenti):
+            where_in_doc = ','.join(['%s'] * len(dokumenti))
+
+
+      
+
+
+        sqltk=f"""Select ngram,upos,convert(avg(tfidf),FLOAT) as tfidf, convert(sum(tf),INT) as tf from (
+                    SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf
+                    FROM ngrams_upos_tf tf, documents d,
+                        (
+                        Select ngram, upos, count(*) as df from ngrams_upos_tf TF
+                        where document_id in
+                    ({where_in_doc})
+                    group by TF.ngram, TF.upos
+                    ) dff, ngrams_upos_df df
+                    where
+                    tf.document_id=d.document_id and
+                    df.ngram=tf.ngram AND df.upos=tf.upos and
+                    dff.ngram=tf.ngram AND dff.upos=tf.upos
+                    ) X
+                    group by ngram,upos
+                    order by tfidf desc
+                    limit 1000;"""
+                    #
+        #sqltk=f"""select ngram,upos,convert(1.0,float) as tfidf,%s as tf from ngrams_upos_tf limit 10;"""
+
+        print (sqltk)
+        print (dokumenti)
+        #še prepovedane besede ven
+        if(where_in_doc):
+            start_time = time.time()
+            cur.execute(sqltk,dokumenti)
+            terms=cur.fetchall()
+            print("Čas poizbedbe je %.2f sekund" % (time.time() - start_time))
+        else:
+            return ret
+
+        print (terms);
+        #ret = list(cur)
+        can = {'forms':[
+            ngram["ngram"]
+            for ngram in terms
+            ]
+        }
+        print (can);
+        res = requests.post(canonapi_endpoint, json=can)
+        
+        data = res.json()
+        print (data);
+        print (data.get("canonical_forms"));
+        print (terms);
+        print(zip(data.get("canonical_forms"),terms))
+
+
+
+
+        ret['terminoloski_kandidati']= [
+            {
+                'POSoznake': x.get("upos"),
+                'kandidat': x.get("ngram"),  # more to bit lemma al terms?
+                'definicija': None,
+                'kanonicnaoblika': d,
+                'ranking': x.get('tfidf'),
+                'podporneutezi': [
+                    0.0,  # ????????
+                    0.0  # ??????
+                ],
+                'pogostostpojavljanja': [x.get('tf'), 0]  # ???????
+            }
+            for (d,x) in zip(data.get("canonical_forms"),terms) 
+        ]
+    
+        #if definicije
+        #idi z variablo sql po id-je dokumentov, preberi conlluje iz diska
+        #naredi en vlki conllu
+        #pokliči metodo
+
+
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB Platform: {e}")
+    finally:
+        cur.close();
+        conn.close();
+
+    return ret
+
+
 # class BaseModel(Model):
 #     class Meta:
 #         database = db