generic/structure_assignment
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

IssueID #1487: added schema validation for structures and dictionary

Browse Source
This commit is contained in:
Cyprian Laskowski 2020-11-04 18:05:12 +01:00
parent bc2ad4d45f
commit 6c78507e9d
4 changed files with 27 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,3 +3,4 @@
/data_admin
/luscenje_struktur
/nova_slovnica
/xml_schemas

3
resources/.gitignore vendored
View File

@ -2,3 +2,6 @@
/dict.xml
/obeliks.jar
/structures.xml
/structures.xsd
/inventory.xsd
/monolingual_dictionaries.xsd

17
scripts/pipeline.py
View File

@ -3,6 +3,7 @@ import os
import shutil
import codecs
import re
import lxml.etree as lxml
import classla
from classla import Document
@ -24,6 +25,8 @@ OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar'
TRANSLATION_FILE_NAME = '../resources/dict.xml'
CLASSLA_MODELS_DIRECTORY = '../resources/classla'
STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml'
STRUCTURE_SCHEMA_FILE_NAME = '../resources/structures.xsd'
DICTIONARY_SCHEMA_FILE_NAME = '../resources/monolingual_dictionaries.xsd'
STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt'
OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu'
@ -57,9 +60,11 @@ def run_pipeline(input_file_name, output_file_name):
run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME)
run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME)
run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME, STRUCTURE_NEW_FILE_NAME)
validate_structures(STRUCTURE_NEW_FILE_NAME)
run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME)
run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_2_FILE_NAME)
run_dictionary_conversion(TEI_STRUCTURE_2_FILE_NAME, DICTIONARY_XML_FILE_NAME)
validate_dictionary(DICTIONARY_XML_FILE_NAME)
shutil.copyfile(DICTIONARY_XML_FILE_NAME, output_file_name)
def run_obeliks4J(obeliks_file_name, classla_file_name):
@ -100,6 +105,12 @@ def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name):
print(extraction_command)
os.system(extraction_command)
def validate_structures(structure_file_name):
print('Validating updated structure specifications ...')
xml_schema = lxml.XMLSchema(lxml.parse(STRUCTURE_SCHEMA_FILE_NAME))
xml_tree = lxml.parse(structure_file_name)
xml_schema.assertValid(xml_tree)
def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name):
print('Assigning structure ids ...')
assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name])
@ -115,4 +126,10 @@ def run_dictionary_conversion(tei_file_name, xml_file_name):
convert_command = ' '.join(['python', TEI_DICTIONARY_SCRIPT_NAME, '-infile', tei_file_name, '-outfile', xml_file_name])
os.system(convert_command)
def validate_dictionary(dictionary_file_name):
print('Validating output dictionary file ...')
xml_schema = lxml.XMLSchema(lxml.parse(DICTIONARY_SCHEMA_FILE_NAME))
xml_tree = lxml.parse(dictionary_file_name)
xml_schema.assertValid(xml_tree)
run_pipeline(input_file_name, output_file_name)

8
scripts/setup.sh
View File

@ -14,7 +14,8 @@ deactivate
git clone https://github.com/clarinsi/Obeliks4J
git clone git@gitea.cjvt.si:redmine_projects/nova_slovnica.git
git clone git@gitea.cjvt.si:ozbolt/luscenje_struktur.git
git clone git@gitea.cjvt.si:generic/data_admin.git # this doesn't include structures.xml yet
git clone git@gitea.cjvt.si:generic/data_admin.git
git clone git@gitea.cjvt.si:generic/xml_schemas.git
## set up obeliks
cd Obeliks4J
@ -29,7 +30,7 @@ ln -s ../nova_slovnica/python/scripts/conllu_to_xml.py .
ln -s ../nova_slovnica/python/scripts/assign_structures.py .
ln -s ../nova_slovnica/python/scripts/create_structures.py .
ln -s ../nova_slovnica/python/scripts/tei_to_dictionary.py .
ln -s ../luscenje_struktur/luscenje_struktur/wani.py .
ln -s ../luscenje_struktur/wani.py .
cd ..
## put needed resources in place
@ -37,4 +38,7 @@ cd resources
mv ../Obeliks4J/obeliks.jar .
ln -s ../nova_slovnica/resources/dict.xml .
ln -s ../data_admin/resources/structures.xml .
ln -s ../data_admin/resources/structures.xsd .
ln -s ../xml_schemas/resources/schema/inventory.xsd .
ln -s ../xml_schemas/resources/schema/monolingual_dictionaries.xsd .
cd ..