kristjan/cjvt-valency
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

a

Browse Source
This commit is contained in:
voje 2019-04-21 19:18:40 +02:00
parent 617babc98d
commit ae9aca0b1e
5 changed files with 33 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -6,3 +6,4 @@ src/frontend_vue/dist/
dockerfiles/database/create.js dockerfiles/database/create.js
*__pycache__/ *__pycache__/
env.local env.local
logs/

24
Makefile
View File

@ -8,12 +8,12 @@ MAKE_ROOT = $(shell pwd)
# kres is composed of many .xml files # kres is composed of many .xml files
# I generated srl tags for kres in separate .json files # I generated srl tags for kres in separate .json files
# (for each kres.xml file there is a kres.json file with srl tags) # (for each kres.xml file there is a kres.json file with srl tags)
# SSJ_FILE = "$(MAKE_ROOT)/data/samples/ssj_xml/ssj500k-sl.body.sample.xml" SSJ_FILE = "$(MAKE_ROOT)/data/samples/ssj_xml/ssj500k-sl.body.sample.xml"
SSJ_FILE = "$(MAKE_ROOT)/data/ssj_file_link" # SSJ_FILE = "$(MAKE_ROOT)/data/ssj_file_link"
# KRES_FOLDER = "$(MAKE_ROOT)/data/samples/kres_xml" KRES_FOLDER = "$(MAKE_ROOT)/data/samples/kres_xml"
KRES_FOLDER = "$(MAKE_ROOT)/data/kres_xml_folder_link # KRES_FOLDER = "$(MAKE_ROOT)/data/kres_xml_folder_link"
# KRES_SRL_FOLDER = "$(MAKE_ROOT)/data/samples/kres_srl_json" KRES_SRL_FOLDER = "$(MAKE_ROOT)/data/samples/kres_srl_json"
KRES_SRL_FOLDER = "$(MAKE_ROOT)/data/kres_json_folder_link" # KRES_SRL_FOLDER = "$(MAKE_ROOT)/data/kres_json_folder_link"
OUTPUT = "db" OUTPUT = "db"
# OUTPUT = "file" # OUTPUT = "file"
@ -62,13 +62,23 @@ data/samples:
# from inside python-env container: # from inside python-env container:
# you can set OUTPUT = "file" and a valid OUTDIR to test writing to json files instead of DB # you can set OUTPUT = "file" and a valid OUTDIR to test writing to json files instead of DB
fill-database: data/samples fill-database-ssj: data/samples
python3 src/pkg/cjvt-corpusparser/corpusparser/main.py --kres-folder $(KRES_FOLDER) \ python3 src/pkg/cjvt-corpusparser/corpusparser/main.py --kres-folder $(KRES_FOLDER) \
--corpus="ssj" \
--ssj-file $(SSJ_FILE) --kres-srl-folder $(KRES_SRL_FOLDER) \ --ssj-file $(SSJ_FILE) --kres-srl-folder $(KRES_SRL_FOLDER) \
--output $(OUTPUT) --outdir $(OUTDIR) --dbaddr $(DBADDR) \ --output $(OUTPUT) --outdir $(OUTDIR) --dbaddr $(DBADDR) \
--dbuser $(DB_USR_USER) --dbpass $(DB_USR_PASS) \ --dbuser $(DB_USR_USER) --dbpass $(DB_USR_PASS) \
--cores $(N_CORES) --cores $(N_CORES)
fill-database-kres: data/samples
python3 src/pkg/cjvt-corpusparser/corpusparser/main.py --kres-folder $(KRES_FOLDER) \
--corpus="kres" \
--ssj-file $(SSJ_FILE) --kres-srl-folder $(KRES_SRL_FOLDER) \
--output $(OUTPUT) --outdir $(OUTDIR) --dbaddr $(DBADDR) \
--dbuser $(DB_USR_USER) --dbpass $(DB_USR_PASS) \
--cores $(N_CORES)
## Frontend ## Frontend

9
README.md
View File

@ -45,7 +45,14 @@ $ make python-env-install
# beforehand, set the data files in Makefile # beforehand, set the data files in Makefile
# instead of mounting directories into the container, you can # instead of mounting directories into the container, you can
# create a link inside ./data, that points to the desired location # create a link inside ./data, that points to the desired location
$ make fill-database
# I've separated the processes for better memory management
$ make fill-database-ssj
$ make fill-database-kres
# You can detach from the running process using Ctrl-p + Ctrl-q
# this is a long operation # this is a long operation
# if running on a remote server, use nohup: # if running on a remote server, use nohup:

8
dockerfiles/python-env/Makefile
View File

@ -1,4 +1,5 @@
IMAGE_NAME="cjvt-python-env" IMAGE_NAME="cjvt-python-env"
CNNAME="python-env"
all: build run all: build run
@ -6,8 +7,10 @@ build:
docker build . -t $(IMAGE_NAME) docker build . -t $(IMAGE_NAME)
run: run:
- docker kill $(CNNAME)
- docker rm $(CNNAME)
docker run \ docker run \
-it \ -it \
-v /home/${USER}:/home/${USER} \ -v /home/${USER}:/home/${USER} \
--user $(shell id -u):$(shell id -g) \ --user $(shell id -u):$(shell id -g) \
-v /etc/passwd:/etc/passwd \ -v /etc/passwd:/etc/passwd \
@ -15,5 +18,6 @@ run:
-v $(MAKE_ROOT):/project \ -v $(MAKE_ROOT):/project \
-w /project \ -w /project \
--net host \ --net host \
--name $(CNNAME) \
$(IMAGE_NAME) \ $(IMAGE_NAME) \
/bin/bash /bin/bash

2
src/pkg/cjvt-corpusparser

@ -1 +1 @@
Subproject commit 00d9192993a2ccfa45237d7a3a6a851058d4ee09 Subproject commit bcc64c767c36ed8e41fd07dc896b83a49faf785b