diff --git a/Tree.py b/Tree.py index aefa000..b6ea997 100644 --- a/Tree.py +++ b/Tree.py @@ -8,7 +8,7 @@ from generic import generate_key class Tree(object): - def __init__(self, index, form, lemma, upos, xpos, deprel, feats, feats_detailed, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, feats_dict, feats_detailed_dict, head): + def __init__(self, index, form, lemma, upos, xpos, deprel, feats_detailed, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, feats_dict, feats_detailed_dict, head): if not hasattr(self, 'feats'): self.feats_detailed = {} @@ -27,9 +27,6 @@ class Tree(object): if deprel not in deprel_dict: deprel_dict[deprel] = Value(deprel) self.deprel = deprel_dict[deprel] - if feats not in feats_dict: - feats_dict[feats] = Value(feats) - self.feats = feats_dict[feats] for feat in feats_detailed.keys(): if feat not in feats_detailed_dict: feats_detailed_dict[feat] = {} @@ -81,7 +78,6 @@ class Tree(object): filter_passed = filter_passed and \ ('deprel' not in option or option['deprel'] == self.deprel.get_value()) and \ - ('feats' not in option or option['feats'] == self.feats.get_value()) and \ ('form' not in option or option['form'] == self.form.get_value()) and \ ('lemma' not in option or option['lemma'] == self.lemma.get_value()) and \ ('upos' not in option or option['upos'] == self.upos.get_value()) @@ -100,7 +96,6 @@ class Tree(object): ('upos' not in query_tree or query_tree['upos'] == self.upos.get_value()) and \ ('xpos' not in query_tree or query_tree['xpos'] == self.xpos.get_value()) and \ ('deprel' not in query_tree or query_tree['deprel'] == self.deprel.get_value()) and \ - ('feats' not in query_tree or query_tree['feats'] == self.feats.get_value()) and \ (not filters['complete_tree_type'] or (len(self.children) == 0 and 'children' not in query_tree) or ('children' in query_tree and len(self.children) == len(query_tree['children']))) and \ self.fits_static_requirements_feats(query_tree) diff --git a/generic.py b/generic.py index e9e47c4..44da53d 100644 --- a/generic.py +++ b/generic.py @@ -23,7 +23,7 @@ def create_output_string_deprel(tree): return tree.deprel.get_value() def create_output_string_lemma(tree): - return tree.lemma.get_value() + return tree.lemma.get_value() if tree.lemma.get_value() is not None else '_' def create_output_string_upos(tree): return tree.upos.get_value() diff --git a/requirements.txt b/requirements.txt index 25db5b5..4d03919 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pyconll==2.1.1 +pyconll==3.1.0 diff --git a/run-multiple-depparse.py b/run-multiple-starks.py similarity index 80% rename from run-multiple-depparse.py rename to run-multiple-starks.py index 3ea8dc0..38cd460 100644 --- a/run-multiple-depparse.py +++ b/run-multiple-starks.py @@ -14,4 +14,4 @@ for path in sorted(os.listdir(input_path)): if not os.path.exists(folder_name): os.makedirs(folder_name) if not os.path.exists(file_name): - os.system("python /home/luka/Development/STARK/dependency-parsetree.py --config_file " + config_path + " --input " + str(path) + " --output " + file_name) + os.system("python /home/luka/Development/STARK/stark.py --config_file " + config_path + " --input " + str(path) + " --output " + file_name) diff --git a/dependency-parsetree.py b/stark.py similarity index 96% rename from dependency-parsetree.py rename to stark.py index 8ede84c..bd7128d 100644 --- a/dependency-parsetree.py +++ b/stark.py @@ -134,9 +134,8 @@ def create_trees(input_path, internal_saves, feats_detailed_dict={}, save=True): continue # TODO check if 5th place is always there for feats - feats = token._fields[5] token_form = token.form if token.form is not None else '_' - node = Tree(int(token.id), token_form, token.lemma, token.upos, token.xpos, token.deprel, feats, token.feats, form_dict, + node = Tree(int(token.id), token_form, token.lemma, token.upos, token.xpos, token.deprel, token.feats, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, feats_dict, feats_detailed_dict, token.head) token_nodes.append(node) if token.deprel == 'root': @@ -351,7 +350,7 @@ def read_filters(config, args, feats_detailed_list): tree_size_range = tree_size.split('-') tree_size_range = [int(r) for r in tree_size_range] - if tree_size_range[0] > 1: + if tree_size_range[0] > 0: if len(tree_size_range) == 1: query_tree = create_ngrams_query_trees(tree_size_range[0], [{}]) elif len(tree_size_range) == 2: @@ -505,8 +504,12 @@ def get_keyness(abs_freq_A, abs_freq_B, count_A, count_B): E2 = count_B * (abs_freq_A + abs_freq_B) / (count_A + count_B) LL = 2 * ((abs_freq_A * math.log(abs_freq_A / E1)) + (abs_freq_B * math.log(abs_freq_B / E2))) if abs_freq_B > 0 else 'NaN' + BIC = LL - math.log(count_A + count_B) if abs_freq_B > 0 else 'NaN' + log_ratio = math.log(((abs_freq_A/count_A)/(abs_freq_B/count_B)), 2) if abs_freq_B > 0 else 'NaN' + OR = (abs_freq_A/(count_A-abs_freq_A)) / (abs_freq_B/(count_B-abs_freq_B)) if abs_freq_B > 0 else 'NaN' + diff = (((abs_freq_A/count_A)*1000000 - (abs_freq_B/count_B)*1000000)*100) / ((abs_freq_B/count_B)*1000000) if abs_freq_B > 0 else 'NaN' - return [LL] + return [abs_freq_B, abs_freq_B/count_B, LL, BIC, log_ratio, OR, diff] def main(): @@ -519,7 +522,7 @@ def main(): parser.add_argument("--internal_saves", default=None, type=str, help="Location for internal_saves.") parser.add_argument("--cpu_cores", default=None, type=int, help="Number of cores used.") - parser.add_argument("--tree_size", default=None, type=int, help="Size of trees.") + parser.add_argument("--tree_size", default=None, type=str, help="Size of trees.") parser.add_argument("--tree_type", default=None, type=str, help="Tree type.") parser.add_argument("--dependency_type", default=None, type=str, help="Dependency type.") parser.add_argument("--node_order", default=None, type=str, help="Order of node.") @@ -574,7 +577,7 @@ def main(): if filters['association_measures']: header += ['MI', 'MI3', 'Dice', 'logDice', 't-score', 'simple-LL'] if args.compare: - header += ['LL'] + header += ['Other absolute frequency', 'Other relative frequency', 'LL', 'BIC', 'Log ratio', 'OR', '%DIFF'] writer.writerow(header) if filters['lines_threshold']: @@ -587,7 +590,8 @@ def main(): if filters['frequency_threshold'] and filters['frequency_threshold'] > v['number']: break words_only = [word_att for word in v['object'].array for word_att in word] + ['' for i in range((tree_size_range[-1] - len(v['object'].array)) * len(v['object'].array[0]))] - row = [v['object'].get_key()[1:-1]] + words_only + [str(v['number'])] + key = [v['object'].get_key()[1:-1]] if v['object'].get_key()[0] == '(' and v['object'].get_key()[-1] == ')' else [v['object'].get_key()] + row = key + words_only + [str(v['number'])] row += ['%.4f' % relative_frequency] if filters['node_order']: row += [v['object'].order]