added adjective handling (appending _ to headwords)

corpusparser/__init__.py

@@ -1 +1,2 @@
-from corpusparser.Parser import Parser
+from corpusparser.Parser import Parser
+from corpusparser.main import enriched_lemma

corpusparser/main.py

@@ -15,6 +15,10 @@ logger = logging.getLogger(__name__)
 n_kres_files = -1  # for logging
 
 
+def enriched_lemma(token):
+    return (token["lemma"] if token["msd"][0] == "G" else token["lemma"] + "_")
+
+
 def _helper_tid_to_token(tid, tokens):
     for t in tokens:
         if t["tid"] == tid:
@@ -29,7 +33,7 @@ def _db_preprocess(e):
     else:
         hw_tids = list(set([x["from"] for x in e["srl_links"]]))
         hw_tokens = [_helper_tid_to_token(tid, e["tokens"]) for tid in hw_tids]
-        headwords = [(t["lemma"] if t["msd"][0] == "G" else t["lemma"] + "_") for t in hw_tokens]
+        headwords = [enriched_lemma(t) for t in hw_tokens]
         e["headwords"] = headwords
 
         functors = list(set([x["afun"] for x in e["srl_links"]]))
@@ -67,9 +71,15 @@ def _handle_kres_file_tpl(kres_file_tpl):
             kres_col.update({"sid": sentence["sid"]}, sentence, upsert=True)
         """
 
+        # skip if one of the sentences is already in DB
+        if kres_col.find({"sid": kres_data[0]["sid"]}).count() > 0:
+            logging.info("File {} already in DB ({}/{})".format(
+                kres_file, kres_file_idx, n_kres_files))
+            return
+
         kres_data_1 = [_db_preprocess(x) for x in kres_data]
         kres_col.insert_many(kres_data_1)  # much much better (just make sure sid has a unique index)
-        logging.info("Handled {} ({}/{}) in {:.2f} s".format(
+        logging.info("Inserted data from {} ({}/{}) in {:.2f} s".format(
             kres_file, kres_file_idx, n_kres_files, time.time() - tstart))
 
 def _get_dbclient(args):