diff --git a/dependency-parsetree.py b/dependency-parsetree.py index 1de6d18..37c99da 100644 --- a/dependency-parsetree.py +++ b/dependency-parsetree.py @@ -375,7 +375,7 @@ def main(): # query_tree = [{"children": [{}, {}, {}, {}]}, {"children": [{"children": [{}]}, {}, {}]}, {"children": [{"children": [{}, {}]}, {}]}, {"children": [{"children": [{}]}, {"children": [{}]}]}, # {"children": [{"children": [{"children": [{}]}]}, {}]}, {"children": [{"children": [{"children": [{}]}, {}]}]}, {"children": [{"children": [{"children": [{}, {}]}]}]}, # {"children": [{"children": [{"children": [{"children": [{}]}]}]}]}, {'children': [{'children': [{}, {}, {}]}]}] - tree_size_range = config.get('settings', 'tree_size').split('-') + tree_size_range = config.get('settings', 'tree_size', fallback='0').split('-') tree_size_range = [int(r) for r in tree_size_range] if tree_size_range[0] > 1: @@ -440,8 +440,8 @@ def main(): filters['complete_tree_type'] = config.get('settings', 'tree_type') == 'complete' filters['association_measures'] = config.getboolean('settings', 'association_measures') filters['nodes_number'] = config.getboolean('settings', 'nodes_number') - filters['frequency_threshold'] = config.getfloat('settings', 'frequency_threshold') - filters['lines_threshold'] = config.getint('settings', 'lines_threshold') + filters['frequency_threshold'] = config.getfloat('settings', 'frequency_threshold', fallback=0) + filters['lines_threshold'] = config.getint('settings', 'lines_threshold', fallback=0) filters['print_root'] = config.getboolean('settings', 'print_root') @@ -571,7 +571,7 @@ def main(): if filters['print_root']: header += ['Root node'] if filters['association_measures']: - header += ['MI', 'MI3', 'Dice', 't-score', 'simple-LL'] + header += ['MI', 'MI3', 'Dice', 'logDice', 't-score', 'simple-LL'] # header = [" ".join(words[i:i + span]) for i in range(0, len(words), span)] + ['Absolute frequency'] writer.writerow(header) @@ -581,13 +581,13 @@ def main(): # body for k, v in sorted_list: v['object'].get_array() - absolute_frequency = v['number'] * 1000000.0 / corpus_size - if filters['frequency_threshold'] and filters['frequency_threshold'] > absolute_frequency: + relative_frequency = v['number'] * 1000000.0 / corpus_size + if filters['frequency_threshold'] and filters['frequency_threshold'] > v['number']: break words_only = [word_att for word in v['object'].array for word_att in word] + ['' for i in range((tree_size_range[-1] - len(v['object'].array)) * len(v['object'].array[0]))] # words_only = printable_answers(k) row = [v['object'].get_key()[1:-1]] + words_only + [str(v['number'])] - row += ['%.4f' % absolute_frequency] + row += ['%.4f' % relative_frequency] if filters['node_order']: row += [v['object'].order] row += [v['object'].get_key_sorted()[1:-1]] diff --git a/generic.py b/generic.py index 6983d13..735269f 100644 --- a/generic.py +++ b/generic.py @@ -76,12 +76,11 @@ def get_collocabilities(ngram, unigrams_dict, corpus_size): O = ngram['number'] E = mul_fwi / pow(N, n-1) - # ['MI', 'MI3', 'Dice', 't-score', 'simple-LL'] - # mi = Math.log(O / E) / Math.log(2); + # ['MI', 'MI3', 'Dice', 'logDice', 't-score', 'simple-LL'] mi = math.log(O / E, 2) - # Math.log(Math.pow(O, 3.0) / E) / Math.log(2); mi3 = math.log(pow(O, 3) / E, 2) dice = n * O / sum_fwi + logdice = 14 + math.log(dice, 2) tscore = (O - E) / math.sqrt(O) simplell = 2 * (O * math.log10(O / E) - (O - E)) - return ['%.4f' % mi, '%.4f' % mi3, '%.4f' % dice, '%.4f' % tscore, '%.4f' % simplell] + return [('%.4f; N=%.4f, n=%.4f; O=%.4f, E=%.4f, sum_fwi=%.4f, mul_fwi=%.4f' % (mi, N, n, O, E, sum_fwi, mul_fwi)), '%.4f' % mi3, '%.4f' % dice, '%.4f' % logdice, '%.4f' % tscore, '%.4f' % simplell]