diff --git a/.gitignore b/.gitignore index e3ecd39..2e50ea8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,11 @@ data/appindex.json src/frontend_vue/node_modules/ src/frontend_vue/dist/ dockerfiles/database/create.js +dockerfiles/database/create_mongo.js +dockerfiles/database/create_postgres.js +dockerfiles/database/mongo_db.gz +dockerfiles/database/postgres_db.tar +dockerfiles/database/postgres_db_OLD.tar *__pycache__/ env.local logs/* @@ -15,3 +20,4 @@ venv* data/ data deploy_instructions/ +run.sh diff --git a/.gitmodules b/.gitmodules index d3c4f22..14a4f2e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "src/pkg/cjvt-corpusparser"] path = src/pkg/cjvt-corpusparser url = git@gitea.cjvt.si:kristjan/cjvt-corpusparser.git +[submodule "src/pkg/luscenje_struktur"] + path = src/pkg/luscenje_struktur + url = https://gitea.cjvt.si/ozbolt/luscenje_struktur.git diff --git a/Makefile b/Makefile index eba603f..9d4398b 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,12 @@ database-service: database-users: cd dockerfiles/database; $(MAKE) create_users +database-restore: + cd dockerfiles/database; $(MAKE) restore_db + +database-restore-postgres: + cd dockerfiles/database; $(MAKE) restore_postgres_db + # also useful, if we want to restart the db database-clean: cd dockerfiles/database; $(MAKE) clean_stack @@ -70,6 +76,7 @@ python-env-install: pip3 install -e src/pkg/cjvt-corpusparser/. pip3 install -e src/pkg/valency/. pip3 install -e src/pkg/seqparser/. + pip3 install -e src/pkg/luscenje_struktur/. # from inside python-env container: data/samples: diff --git a/dockerfiles/database/Dockerfile b/dockerfiles/database/Dockerfile index af9aaac..5b14d6f 100644 --- a/dockerfiles/database/Dockerfile +++ b/dockerfiles/database/Dockerfile @@ -1,5 +1,5 @@ FROM mongo:4.2.9 WORKDIR / -COPY init_inside_container.sh /. -COPY create.js /. \ No newline at end of file +COPY init_inside_mongo_container.sh /. +COPY create_mongo.js /. diff --git a/dockerfiles/database/Makefile b/dockerfiles/database/Makefile index 3dbf58d..4efc1a5 100644 --- a/dockerfiles/database/Makefile +++ b/dockerfiles/database/Makefile @@ -2,33 +2,62 @@ # collection names: lower case, plural # user names? -# mongo admin -u root -p password --eval "db.getSiblingDB('vlDB').addUser('vluser', 'password')" +all: build_run create_users -STACKNAME = dbstack +build_run: build_mongo run_docker_compose + +postgres_create_roles: + echo 'psql -v ON_ERROR_STOP=OFF --username $(DB_ADM_USER) <<-EOSQL' > create_postgres.js + echo "create user $(DB_USR_USER) with encrypted password '$(DB_USR_PASS)';" >> create_postgres.js + echo "create database superdb_small;" >> create_postgres.js + echo "grant all privileges on database superdb_small to $(DB_USR_USER);" >> create_postgres.js + echo "grant usage on schema public to $(DB_USR_USER);" >> create_postgres.js + echo "grant select on all tables in schema public to $(DB_USR_USER);" >> create_postgres.js + echo "EOSQL" >> create_postgres.js + chmod +x create_postgres.js + +mongo_create_roles: + echo 'db.auth("$(DB_ADM_USER)", "$(DB_ADM_PASS)")' > create_mongo.js + echo 'use valdb' >> create_mongo.js + echo 'db.createUser({user: "$(DB_USR_USER)", pwd: "$(DB_USR_PASS)", roles: ["readWrite"]})' >> create_mongo.js + echo 'db.grantRolesToUser("$(DB_USR_USER)", [{ role: "readWrite", db: "extvaldb"}])' >> create_mongo.js + +build_mongo: mongo_create_roles + docker build . -t my-mongo --no-cache -.PHONY: start_db FORCE +# build_postgres: postgres_create_roles +# docker build . -t my-mongo --no-cache -all: build_run create_users +run_docker_compose: + mkdir -p ${HOME}/valency_data/mongo_container/data/ + #docker kill $(shell ./get_mongo_container_name.sh) + #docker kill $(shell ./get_postgres_container_name.sh) + #docker-compose stop + docker-compose -f valency-stack.yml up -d --force-recreate + # docker stack deploy --compose-file mongodb-stack.yml $(STACKNAME) -build_run: build_mongo run_stack +create_users: create_mongo_users create_postgres_users -create.js: FORCE -FORCE: - echo 'db.auth("$(DB_ADM_USER)", "$(DB_ADM_PASS)")' > create.js - echo 'use valdb' >> create.js - echo 'db.createUser({user: "$(DB_USR_USER)", pwd: "$(DB_USR_PASS)", roles: ["readWrite"]})' >> create.js +create_mongo_users: mongo_create_roles + docker exec $(shell ./get_mongo_container_name.sh) /init_inside_mongo_container.sh + # rm create.js -build_mongo: create.js - docker build . -t my-mongo --no-cache +create_postgres_users: postgres_create_roles + docker exec $(shell ./get_postgres_container_name.sh) /scripts/init_inside_postgres_container.sh -clean_stack: - docker stack rm $(STACKNAME) +restore_db: restore_mongo_db restore_postgres_db -run_stack: - mkdir -p ${HOME}/mongo_container/data/ - docker stack deploy --compose-file mongodb-stack.yml $(STACKNAME) +restore_mongo_db: +ifeq (,$(wildcard ./mongo_db.gz)) + $(error "mongo_db.gz does not exists. Make sure to have dump of mongo db in 'dockerfiles/database/mongo_db.gz'") +else + docker exec $(shell ./get_mongo_container_name.sh) sh -c 'mongorestore --gzip --archive=/scripts/mongo_db.gz --db valdb --username $(DB_USR_USER) --password $(DB_USR_PASS) --authenticationDatabase valdb' +endif -create_users: create.js - docker exec $(shell ./get_container_name.sh) /init_inside_container.sh - # rm create.js +restore_postgres_db: +ifeq (,$(wildcard ./postgres_db.tar)) + $(error "postgres_db.tar does not exists. Make sure to have dump of postgres db in 'dockerfiles/database/postgres_db.tar'") +else + docker exec $(shell ./get_postgres_container_name.sh) sh -c 'pg_restore -U $(DB_ADM_USER) --dbname=superdb_small --create --verbose /scripts/postgres_db.tar' +endif diff --git a/dockerfiles/database/get_container_name.sh b/dockerfiles/database/get_mongo_container_name.sh similarity index 100% rename from dockerfiles/database/get_container_name.sh rename to dockerfiles/database/get_mongo_container_name.sh diff --git a/dockerfiles/database/get_postgres_container_name.sh b/dockerfiles/database/get_postgres_container_name.sh new file mode 100755 index 0000000..8ccda6b --- /dev/null +++ b/dockerfiles/database/get_postgres_container_name.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker ps | grep postgres | awk '{print $1}' diff --git a/dockerfiles/database/init_inside_container.sh b/dockerfiles/database/init_inside_container.sh deleted file mode 100755 index 76c3398..0000000 --- a/dockerfiles/database/init_inside_container.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -mongo admin < /create.js diff --git a/dockerfiles/database/init_inside_mongo_container.sh b/dockerfiles/database/init_inside_mongo_container.sh new file mode 100755 index 0000000..7c5e1a3 --- /dev/null +++ b/dockerfiles/database/init_inside_mongo_container.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +mongo admin < /create_mongo.js diff --git a/dockerfiles/database/init_inside_postgres_container.sh b/dockerfiles/database/init_inside_postgres_container.sh new file mode 100755 index 0000000..27ca19a --- /dev/null +++ b/dockerfiles/database/init_inside_postgres_container.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +/scripts/create_postgres.js diff --git a/dockerfiles/database/mongodb-stack.yml b/dockerfiles/database/mongodb-stack.yml deleted file mode 100644 index f3e9590..0000000 --- a/dockerfiles/database/mongodb-stack.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '3.1' - -services: - - my_mongo: - image: my-mongo - restart: always - ports: - - 27017:27017 - environment: - MONGO_INITDB_ROOT_USERNAME: ${DB_ADM_USER} - MONGO_INITDB_ROOT_PASSWORD: ${DB_ADM_PASS} - volumes: - - ${HOME}/mongo_container/data/:/data/db - - mongo_express: - image: mongo-express - restart: always - ports: - - 8087:8081 - environment: - ME_CONFIG_BASICAUTH_USERNAME: ${MONGOEXPRESS_USER} - ME_CONFIG_BASICAUTH_PASSWORD: ${MONGOEXPRESS_PASS} - ME_CONFIG_MONGODB_ADMINUSERNAME: ${DB_ADM_USER} - ME_CONFIG_MONGODB_ADMINPASSWORD: ${DB_ADM_PASS} - ME_CONFIG_MONGODB_SERVER: my_mongo diff --git a/dockerfiles/database/valency-stack.yml b/dockerfiles/database/valency-stack.yml new file mode 100644 index 0000000..bdc2ff1 --- /dev/null +++ b/dockerfiles/database/valency-stack.yml @@ -0,0 +1,27 @@ +version: '3.1' + +services: + + my_mongo: + image: my-mongo + restart: always + ports: + - 127.0.0.1:27017:27017 + environment: + MONGO_INITDB_ROOT_USERNAME: ${DB_ADM_USER} + MONGO_INITDB_ROOT_PASSWORD: ${DB_ADM_PASS} + volumes: + - ${HOME}/valency_data/mongo_container/data/:/data/db + - ./:/scripts + + my_postgres: + image: postgres + restart: always + ports: + - 127.0.0.1:5432:5432 + environment: + POSTGRES_USER: ${DB_ADM_USER} + POSTGRES_PASSWORD: ${DB_ADM_PASS} + volumes: + - ${HOME}/valency_data/postgres_container/data/:/var/lib/postgresql/data + - ./:/scripts diff --git a/dockerfiles/python-env/Dockerfile b/dockerfiles/python-env/Dockerfile index 9d82cb6..c5fa020 100644 --- a/dockerfiles/python-env/Dockerfile +++ b/dockerfiles/python-env/Dockerfile @@ -6,7 +6,8 @@ vim \ python3 \ python3-pip \ sshfs \ -curl +curl \ +locales RUN pip3 install --upgrade pip @@ -21,6 +22,16 @@ RUN pip3 install \ flask_cors \ pymongo \ flask-pymongo \ - gunicorn + gunicorn \ + SQLAlchemy \ + tqdm \ + psycopg2-binary + +# Set the locale +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 ENV PYTHONIOENCODING UTF-8 diff --git a/scripts/create_xml.py b/scripts/create_xml.py index fdf6feb..4ab2e69 100644 --- a/scripts/create_xml.py +++ b/scripts/create_xml.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- #!/usr/bin/python3 #imports from luscenje_struktur diff --git a/src/pkg/cjvt-corpusparser b/src/pkg/cjvt-corpusparser index 92b3ac4..01adf47 160000 --- a/src/pkg/cjvt-corpusparser +++ b/src/pkg/cjvt-corpusparser @@ -1 +1 @@ -Subproject commit 92b3ac4ea3a73b93c25b363b5b9cb096d4d011cd +Subproject commit 01adf47b9b63b43f86bff52429792b0de2327ddd diff --git a/src/pkg/luscenje_struktur b/src/pkg/luscenje_struktur new file mode 160000 index 0000000..8c87d07 --- /dev/null +++ b/src/pkg/luscenje_struktur @@ -0,0 +1 @@ +Subproject commit 8c87d07b8a3ca73faac2fac30c39969bc5f97d45