From 6c78507e9db0042dcf8dc0a5b1ab59e8cfeb7ef8 Mon Sep 17 00:00:00 2001 From: Cyprian Laskowski Date: Wed, 4 Nov 2020 18:05:12 +0100 Subject: [PATCH] IssueID #1487: added schema validation for structures and dictionary --- .gitignore | 1 + resources/.gitignore | 3 +++ scripts/pipeline.py | 17 +++++++++++++++++ scripts/setup.sh | 8 ++++++-- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 754b9ab..492e6ae 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /data_admin /luscenje_struktur /nova_slovnica +/xml_schemas diff --git a/resources/.gitignore b/resources/.gitignore index 6118611..b40c628 100644 --- a/resources/.gitignore +++ b/resources/.gitignore @@ -2,3 +2,6 @@ /dict.xml /obeliks.jar /structures.xml +/structures.xsd +/inventory.xsd +/monolingual_dictionaries.xsd diff --git a/scripts/pipeline.py b/scripts/pipeline.py index a9e592c..528668d 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -3,6 +3,7 @@ import os import shutil import codecs import re +import lxml.etree as lxml import classla from classla import Document @@ -24,6 +25,8 @@ OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar' TRANSLATION_FILE_NAME = '../resources/dict.xml' CLASSLA_MODELS_DIRECTORY = '../resources/classla' STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml' +STRUCTURE_SCHEMA_FILE_NAME = '../resources/structures.xsd' +DICTIONARY_SCHEMA_FILE_NAME = '../resources/monolingual_dictionaries.xsd' STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt' OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu' @@ -57,9 +60,11 @@ def run_pipeline(input_file_name, output_file_name): run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME) run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME) run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME, STRUCTURE_NEW_FILE_NAME) + validate_structures(STRUCTURE_NEW_FILE_NAME) run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME) run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_2_FILE_NAME) run_dictionary_conversion(TEI_STRUCTURE_2_FILE_NAME, DICTIONARY_XML_FILE_NAME) + validate_dictionary(DICTIONARY_XML_FILE_NAME) shutil.copyfile(DICTIONARY_XML_FILE_NAME, output_file_name) def run_obeliks4J(obeliks_file_name, classla_file_name): @@ -100,6 +105,12 @@ def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name): print(extraction_command) os.system(extraction_command) +def validate_structures(structure_file_name): + print('Validating updated structure specifications ...') + xml_schema = lxml.XMLSchema(lxml.parse(STRUCTURE_SCHEMA_FILE_NAME)) + xml_tree = lxml.parse(structure_file_name) + xml_schema.assertValid(xml_tree) + def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name): print('Assigning structure ids ...') assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name]) @@ -115,4 +126,10 @@ def run_dictionary_conversion(tei_file_name, xml_file_name): convert_command = ' '.join(['python', TEI_DICTIONARY_SCRIPT_NAME, '-infile', tei_file_name, '-outfile', xml_file_name]) os.system(convert_command) +def validate_dictionary(dictionary_file_name): + print('Validating output dictionary file ...') + xml_schema = lxml.XMLSchema(lxml.parse(DICTIONARY_SCHEMA_FILE_NAME)) + xml_tree = lxml.parse(dictionary_file_name) + xml_schema.assertValid(xml_tree) + run_pipeline(input_file_name, output_file_name) diff --git a/scripts/setup.sh b/scripts/setup.sh index a6b5203..68d6900 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -14,7 +14,8 @@ deactivate git clone https://github.com/clarinsi/Obeliks4J git clone git@gitea.cjvt.si:redmine_projects/nova_slovnica.git git clone git@gitea.cjvt.si:ozbolt/luscenje_struktur.git -git clone git@gitea.cjvt.si:generic/data_admin.git # this doesn't include structures.xml yet +git clone git@gitea.cjvt.si:generic/data_admin.git +git clone git@gitea.cjvt.si:generic/xml_schemas.git ## set up obeliks cd Obeliks4J @@ -29,7 +30,7 @@ ln -s ../nova_slovnica/python/scripts/conllu_to_xml.py . ln -s ../nova_slovnica/python/scripts/assign_structures.py . ln -s ../nova_slovnica/python/scripts/create_structures.py . ln -s ../nova_slovnica/python/scripts/tei_to_dictionary.py . -ln -s ../luscenje_struktur/luscenje_struktur/wani.py . +ln -s ../luscenje_struktur/wani.py . cd .. ## put needed resources in place @@ -37,4 +38,7 @@ cd resources mv ../Obeliks4J/obeliks.jar . ln -s ../nova_slovnica/resources/dict.xml . ln -s ../data_admin/resources/structures.xml . +ln -s ../data_admin/resources/structures.xsd . +ln -s ../xml_schemas/resources/schema/inventory.xsd . +ln -s ../xml_schemas/resources/schema/monolingual_dictionaries.xsd . cd ..