diff --git a/.gitignore b/.gitignore index 429f144..16edd0d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ data/ __pycache__/ venv/ +src/__pycache__/ diff --git a/src/__pycache__/__init__.cpython-37.pyc b/src/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index e666e67..0000000 Binary files a/src/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/src/__pycache__/create_tei.cpython-37.pyc b/src/__pycache__/create_tei.cpython-37.pyc deleted file mode 100644 index 2906192..0000000 Binary files a/src/__pycache__/create_tei.cpython-37.pyc and /dev/null differ diff --git a/src/create_tei.py b/src/create_tei.py index f32fe3e..f0cc260 100644 --- a/src/create_tei.py +++ b/src/create_tei.py @@ -12,8 +12,8 @@ class Sentence: self.links = [] self.no_ud = no_ud - def add_item(self, token, lemma, upos, upos_other, xpos, misc): - self.items.append([token, lemma, upos, upos_other, xpos, "SpaceAfter=No" in misc.split('|')]) + def add_item(self, word_id, token, lemma, upos, upos_other, xpos, misc): + self.items.append([word_id, token, lemma, upos, upos_other, xpos, "SpaceAfter=No" in misc.split('|')]) def add_link(self, link_ref, link_type): self.links.append([link_ref, link_type]) @@ -25,10 +25,9 @@ class Sentence: xml_id = self._id base = etree.Element('s') set_xml_attr(base, 'id', xml_id) - id_counter = 1 for item in self.items: - token, lemma, upos, upos_other, xpos, no_space_after = item + word_id, token, lemma, upos, upos_other, xpos, no_space_after = item if xpos in {'U', 'Z'}: # hmm, safe only as long as U is unused in English tagset and Z in Slovenian one to_add = etree.Element('pc') @@ -43,11 +42,9 @@ class Sentence: else: to_add.set('msd', f'UposTag={upos}') - set_xml_attr(to_add, 'id', "{}.{}".format(xml_id, id_counter)) + set_xml_attr(to_add, 'id', word_id) to_add.text = token - id_counter += 1 - if no_space_after: to_add.set('join', 'right') @@ -57,8 +54,9 @@ class Sentence: class Paragraph: - def __init__(self, _id): + def __init__(self, _id, _doc_id): self._id = _id if _id is not None else 'no-id' + self._doc_id = _doc_id if _doc_id is not None else '' self.sentences = [] def add_sentence(self, sentence): @@ -68,7 +66,10 @@ class Paragraph: if id_prefix: xml_id = id_prefix + '.' + self._id else: - xml_id = self._id + if self._doc_id: + xml_id = self._doc_id + '.' + self._id + else: + xml_id = self._id p = etree.Element('p') set_xml_attr(p, 'id', xml_id) @@ -97,7 +98,7 @@ class TeiDocument: text = etree.SubElement(root, 'text') body = etree.SubElement(text, 'body') for para in self.paragraphs: - body.append(para.as_xml(id_prefix=xml_id)) + body.append(para.as_xml()) encoding_desc = etree.SubElement(tei_header, 'encodingDesc') tags_decl = etree.SubElement(encoding_desc, 'tagsDecl') @@ -121,6 +122,36 @@ def build_tei_etrees(documents): return elements +def build_links(all_edges): + root = etree.Element('TEI') + root.set('xmlns', 'http://www.tei-c.org/ns/1.0') + set_xml_attr(root, 'lang', 'sl') + + # elements = [] + for document_edges in all_edges: + d = etree.Element('linkGrp') + for paragraph_edges in document_edges: + p = etree.Element('linkGrp') + for sentence_edges in paragraph_edges: + s = etree.Element('linkGrp') + random_id = '' + for token_edges in sentence_edges: + link = etree.Element('link') + link.set('labels', ' '.join(token_edges['labels'])) + link.set('sources', ' '.join(['#' + source for source in token_edges['source_ids']])) + link.set('targets', ' '.join(['#' + source for source in token_edges['target_ids']])) + if not random_id: + random_id = token_edges['source_ids'][0] if len(token_edges['source_ids']) > 0 else token_edges['target_ids'][0] + s.append(link) + set_xml_attr(s, 'sentence_id', '.'.join(random_id.split('.')[:3])) + p.append(s) + set_xml_attr(p, 'paragraph_id', '.'.join(random_id.split('.')[:2])) + d.append(p) + set_xml_attr(d, 'document_id', random_id.split('.')[0]) + root.append(d) + return root + + def set_xml_attr(node, attribute, value): node.attrib['{http://www.w3.org/XML/1998/namespace}' + attribute] = value @@ -173,7 +204,7 @@ def construct_tei_documents_from_list(object_list): # para_buffer.append(line) if len(object_list) > 0: - document_paragraphs.append(construct_paragraph(para_id, object_list)) + document_paragraphs.append(construct_paragraph(doc_id, para_id, object_list)) if len(document_paragraphs) > 0: documents.append( @@ -196,7 +227,7 @@ def construct_tei_documents(conllu_lines): key, val = parse_metaline(line) if key == 'newdoc id': if len(para_buffer) > 0: - document_paragraphs.append(construct_paragraph(para_id, para_buffer)) + document_paragraphs.append(construct_paragraph(doc_id, para_id, para_buffer)) if len(document_paragraphs) > 0: documents.append( TeiDocument(doc_id, document_paragraphs)) @@ -204,7 +235,7 @@ def construct_tei_documents(conllu_lines): doc_id = val elif key == 'newpar id': if len(para_buffer) > 0: - document_paragraphs.append(construct_paragraph(para_id, para_buffer)) + document_paragraphs.append(construct_paragraph(doc_id, para_id, para_buffer)) para_buffer = [] para_id = val elif key == 'sent_id': @@ -214,7 +245,7 @@ def construct_tei_documents(conllu_lines): para_buffer.append(line) if len(para_buffer) > 0: - document_paragraphs.append(construct_paragraph(para_id, para_buffer)) + document_paragraphs.append(construct_paragraph(doc_id, para_id, para_buffer)) if len(document_paragraphs) > 0: documents.append( @@ -223,8 +254,8 @@ def construct_tei_documents(conllu_lines): return documents -def construct_paragraph_from_list(para_id, etree_source_sentences): - para = Paragraph(para_id) +def construct_paragraph_from_list(doc_id, para_id, etree_source_sentences): + para = Paragraph(para_id, doc_id) for sentence in etree_source_sentences: para.add_sentence(sentence) @@ -232,8 +263,8 @@ def construct_paragraph_from_list(para_id, etree_source_sentences): return para -def construct_paragraph(para_id, conllu_lines): - para = Paragraph(para_id) +def construct_paragraph(doc_id, para_id, conllu_lines): + para = Paragraph(para_id, doc_id) sent_id = None sent_buffer = [] @@ -267,6 +298,7 @@ def construct_sentence_from_list(sent_id, object_list): misc = '_' if tokens['space_after'] else 'SpaceAfter=No' sentence.add_item( + word_id, token, lemma, upos, diff --git a/svala2tei.py b/svala2tei.py index 7af0eb9..ca5c556 100644 --- a/svala2tei.py +++ b/svala2tei.py @@ -9,17 +9,17 @@ from xml.etree import ElementTree from lxml import etree from src.create_tei import construct_tei_etrees, construct_tei_documents_from_list, construct_sentence_from_list, \ - construct_paragraph_from_list, TeiDocument, build_tei_etrees + construct_paragraph_from_list, TeiDocument, build_tei_etrees, build_links logging.basicConfig(level=logging.INFO) -def add_token(svala_i, source_i, target_i, el, source, target, edges, svala_data, sentence_string_source_id, sentence_string_target_id): +def add_token(svala_i, source_i, target_i, el, source, target, edges, svala_data, sentence_string_id): source_id = "s" + svala_i target_id = "t" + svala_i edge_id = "e-" + source_id + "-" + target_id - source_token_id = sentence_string_source_id + f'.{source_i}' - target_token_id = sentence_string_target_id + f'.{target_i}' + source_token_id = sentence_string_id + f'.s{source_i}' + target_token_id = sentence_string_id + f'.t{target_i}' token_tag = 'w' if el.tag.startswith('w') else 'pc' lemma = el.attrib['lemma'] if token_tag == 'w' else el.text source.append({'token': el.text, 'tag': token_tag, 'ana': el.attrib['ana'], 'lemma': lemma, 'id': source_token_id, 'space_after': False}) @@ -27,15 +27,15 @@ def add_token(svala_i, source_i, target_i, el, source, target, edges, svala_data edges.append({'source_ids': [source_token_id], 'target_ids': [target_token_id], 'labels': svala_data['edges'][edge_id]['labels']}) -def add_error_token(el, out_list, sentence_string_id, out_list_i, out_list_ids): - source_token_id = sentence_string_id + f'.{out_list_i}' +def add_error_token(el, out_list, sentence_string_id, out_list_i, out_list_ids, is_source): + source_token_id = sentence_string_id + f'.s{out_list_i}' if is_source else sentence_string_id + f'.t{out_list_i}' token_tag = 'w' if el.tag.startswith('w') else 'pc' lemma = el.attrib['lemma'] if token_tag == 'w' else el.text out_list.append({'token': el.text, 'tag': token_tag, 'ana': el.attrib['ana'], 'lemma': lemma, 'id': source_token_id, 'space_after': False}) out_list_ids.append(source_token_id) -def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_data, sentence_string_source_id, sentence_string_target_id): +def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_data, sentence_string_id): source_edge_ids = [] target_edge_ids = [] source_ids = [] @@ -49,7 +49,7 @@ def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_ source_id = "s" + ind source_edge_ids.append(source_id) - add_error_token(el, source, sentence_string_source_id, source_i, source_ids) + add_error_token(el, source, sentence_string_id, source_i, source_ids, True) source_i += 1 svala_i += 1 @@ -65,7 +65,7 @@ def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_ target_id = "t" + ind target_edge_ids.append(target_id) - add_error_token(p_el, target, sentence_string_target_id, target_i, target_ids) + add_error_token(p_el, target, sentence_string_id, target_i, target_ids, False) target_i += 1 svala_i += 1 @@ -81,7 +81,7 @@ def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_ source_id = "s" + ind source_edge_ids.append(source_id) - add_error_token(el_l2, source, sentence_string_source_id, source_i, source_ids) + add_error_token(el_l2, source, sentence_string_id, source_i, source_ids, True) source_i += 1 svala_i += 1 @@ -97,7 +97,7 @@ def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_ source_id = "s" + ind source_edge_ids.append(source_id) - add_error_token(el_l3, source, sentence_string_source_id, source_i, source_ids) + add_error_token(el_l3, source, sentence_string_id, source_i, source_ids, True) source_i += 1 svala_i += 1 @@ -113,7 +113,7 @@ def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_ source_id = "s" + ind source_edge_ids.append(source_id) - add_error_token(el_l4, source, sentence_string_source_id, source_i, source_ids) + add_error_token(el_l4, source, sentence_string_id, source_i, source_ids, True) source_i += 1 svala_i += 1 @@ -128,7 +128,7 @@ def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_ source_id = "s" + ind source_edge_ids.append(source_id) - add_error_token(el_l5, source, sentence_string_source_id, source_i, source_ids) + add_error_token(el_l5, source, sentence_string_id, source_i, source_ids, True) source_i += 1 svala_i += 1 @@ -142,14 +142,13 @@ def add_errors(svala_i, source_i, target_i, error, source, target, edges, svala_ target_id = "t" + ind target_edge_ids.append(target_id) - add_error_token(p_el, target, sentence_string_target_id, target_i, target_ids) + add_error_token(p_el, target, sentence_string_id, target_i, target_ids, False) target_i += 1 svala_i += 1 elif p_el.tag.startswith('c'): target[-1]['space_after'] = True - edge_ids = sorted(source_edge_ids) + sorted(target_edge_ids) edge_id = "e-" + "-".join(edge_ids) edges.append({'source_ids': source_ids, 'target_ids': target_ids, 'labels': svala_data['edges'][edge_id]['labels']}) @@ -161,6 +160,12 @@ def process_file(et, args): if os.path.exists(args.results_folder): shutil.rmtree(args.results_folder) os.mkdir(args.results_folder) + etree_source_documents = [] + etree_target_documents = [] + etree_source_paragraphs = [] + etree_target_paragraphs = [] + + document_edges = [] for div in et.iter('div'): bibl = div.find('bibl') file_name = bibl.get('n') @@ -174,13 +179,13 @@ def process_file(et, args): svala_list = [[fname[:-13], fname] if 'problem' in fname else [fname[:-5], fname] for fname in os.listdir(svala_path)] svala_dict = {e[0]: e[1] for e in svala_list} + paragraph_edges = [] + paragraphs = div.findall('p') for paragraph in paragraphs: sentences = paragraph.findall('s') svala_i = 1 - - # read json svala_file = os.path.join(svala_path, svala_dict[paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id']]) jf = open(svala_file) @@ -189,54 +194,68 @@ def process_file(et, args): etree_source_sentences = [] etree_target_sentences = [] - edges = [] + + sentence_edges = [] + for sentence_id, sentence in enumerate(sentences): source = [] target = [] + edges = [] sentence_id += 1 source_i = 1 target_i = 1 - sentence_string_source_id = paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'] + f's.{sentence_id}' - sentence_string_target_id = paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'] + f't.{sentence_id}' + sentence_string_id = paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'] + f'.{sentence_id}' for el in sentence: if el.tag.startswith('w'): - add_token(str(svala_i), source_i, target_i, el, source, target, edges, svala_data, sentence_string_source_id, sentence_string_target_id) + add_token(str(svala_i), source_i, target_i, el, source, target, edges, svala_data, sentence_string_id) svala_i += 1 source_i += 1 target_i += 1 elif el.tag.startswith('pc'): - add_token(str(svala_i), source_i, target_i, el, source, target, edges, svala_data, sentence_string_source_id, sentence_string_target_id) + add_token(str(svala_i), source_i, target_i, el, source, target, edges, svala_data, sentence_string_id) svala_i += 1 source_i += 1 target_i += 1 elif el.tag.startswith('u'): - svala_i, source_i, target_i = add_errors(svala_i, source_i, target_i, el, source, target, edges, svala_data, sentence_string_source_id, sentence_string_target_id) + svala_i, source_i, target_i = add_errors(svala_i, source_i, target_i, el, source, target, edges, svala_data, sentence_string_id) elif el.tag.startswith('c'): - source[-1]['space_after'] = True - target[-1]['space_after'] = True + if len(source) > 0: + source[-1]['space_after'] = True + if len(target) > 0: + target[-1]['space_after'] = True + + sentence_edges.append(edges) etree_source_sentences.append(construct_sentence_from_list(str(sentence_id), source)) etree_target_sentences.append(construct_sentence_from_list(str(sentence_id), target)) - etree_source_paragraph = construct_paragraph_from_list(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[1] + 's', etree_source_sentences) - etree_source_document = TeiDocument(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[0], [etree_source_paragraph]) - etree_source = build_tei_etrees([etree_source_document]) + etree_source_paragraphs.append(construct_paragraph_from_list(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[0], paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[1], etree_source_sentences)) + etree_target_paragraphs.append(construct_paragraph_from_list(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[0], paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[1], etree_target_sentences)) + paragraph_edges.append(sentence_edges) + + document_edges.append(paragraph_edges) + + etree_source_documents.append(TeiDocument(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[0], etree_source_paragraphs)) + etree_target_documents.append(TeiDocument(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[0], etree_target_paragraphs)) + + etree_source = build_tei_etrees(etree_source_documents) + etree_target = build_tei_etrees(etree_target_documents) - etree_target_paragraph = construct_paragraph_from_list(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[1] + 't', etree_target_sentences) - etree_target_document = TeiDocument(paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id'].split('.')[0], [etree_target_paragraph]) - etree_target = build_tei_etrees([etree_target_document]) + # TODO FIX THIS + etree_links = build_links(document_edges) - with open(os.path.join(args.results_folder, f"{paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id']}_source"), 'w') as sf: - sf.write(etree.tostring(etree_source[0], pretty_print=True, encoding='utf-8').decode()) + with open(os.path.join(args.results_folder, f"source.xml"), 'w') as sf: + sf.write(etree.tostring(etree_source[0], pretty_print=True, encoding='utf-8').decode()) - with open(os.path.join(args.results_folder, f"{paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id']}_target"), 'w') as tf: - tf.write(etree.tostring(etree_target[0], pretty_print=True, encoding='utf-8').decode()) + with open(os.path.join(args.results_folder, f"target.xml"), 'w') as tf: + tf.write(etree.tostring(etree_target[0], pretty_print=True, encoding='utf-8').decode()) - with open(os.path.join(args.results_folder, f"{paragraph.attrib['{http://www.w3.org/XML/1998/namespace}id']}_errors"), 'w') as jf: - json.dump(edges, jf, ensure_ascii=False, indent=" ") + with open(os.path.join(args.results_folder, f"links.xml"), 'w') as tf: + tf.write(etree.tostring(etree_links, pretty_print=True, encoding='utf-8').decode()) - break + with open(os.path.join(args.results_folder, f"links.json"), 'w') as jf: + json.dump(document_edges, jf, ensure_ascii=False, indent=" ") def main(args):