Merge branch 'scripts' of https://gitea.cjvt.si/lkrsnik/cjvt-valency into scripts

.gitignore

@@ -7,6 +7,11 @@ data/appindex.json
 src/frontend_vue/node_modules/
 src/frontend_vue/dist/
 dockerfiles/database/create.js
+dockerfiles/database/create_mongo.js
+dockerfiles/database/create_postgres.js
+dockerfiles/database/mongo_db.gz
+dockerfiles/database/postgres_db.tar
+dockerfiles/database/postgres_db_OLD.tar
 *__pycache__/
 env.local
 logs/*
@@ -15,3 +20,4 @@ venv*
 data/
 data
 deploy_instructions/
+run.sh

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "src/pkg/cjvt-corpusparser"]
 	path = src/pkg/cjvt-corpusparser
 	url = git@gitea.cjvt.si:kristjan/cjvt-corpusparser.git
+[submodule "src/pkg/luscenje_struktur"]
+	path = src/pkg/luscenje_struktur
+	url = https://gitea.cjvt.si/ozbolt/luscenje_struktur.git

Makefile

@@ -57,6 +57,12 @@ database-service:
 database-users:
 	cd dockerfiles/database; $(MAKE) create_users
 
+database-restore:
+	cd dockerfiles/database; $(MAKE) restore_db
+
+database-restore-postgres:
+	cd dockerfiles/database; $(MAKE) restore_postgres_db
+
 # also useful, if we want to restart the db
 database-clean:
 	cd dockerfiles/database; $(MAKE) clean_stack
@@ -70,6 +76,7 @@ python-env-install:
 	pip3 install -e src/pkg/cjvt-corpusparser/.
 	pip3 install -e src/pkg/valency/.
 	pip3 install -e src/pkg/seqparser/.
+	pip3 install -e src/pkg/luscenje_struktur/.
 
 # from inside python-env container:
 data/samples:

dockerfiles/database/Dockerfile

@@ -1,5 +1,5 @@
 FROM mongo:4.2.9
 
 WORKDIR /
-COPY init_inside_container.sh /.
+COPY init_inside_mongo_container.sh /.
-COPY create.js /.
+COPY create_mongo.js /.

dockerfiles/database/Makefile

@@ -2,33 +2,62 @@
 # collection names: lower case, plural
 # user names?
 
-# mongo admin -u root -p password --eval "db.getSiblingDB('vlDB').addUser('vluser', 'password')"
-
-STACKNAME = dbstack
-
-.PHONY: start_db FORCE
-
 all: build_run create_users
 
-build_run: build_mongo run_stack
+build_run: build_mongo run_docker_compose
 
-create.js: FORCE
+postgres_create_roles:
+	echo 'psql -v ON_ERROR_STOP=OFF --username $(DB_ADM_USER) <<-EOSQL' > create_postgres.js
+	echo "create user $(DB_USR_USER) with encrypted password '$(DB_USR_PASS)';" >> create_postgres.js
+	echo "create database superdb_small;" >> create_postgres.js
+	echo "grant all privileges on database superdb_small to $(DB_USR_USER);" >> create_postgres.js
+	echo "grant usage on schema public to $(DB_USR_USER);" >> create_postgres.js
+	echo "grant select on all tables in schema public to $(DB_USR_USER);" >> create_postgres.js
+	echo "EOSQL" >> create_postgres.js
+	chmod +x create_postgres.js
 
-FORCE:
+mongo_create_roles:
-	echo 'db.auth("$(DB_ADM_USER)", "$(DB_ADM_PASS)")' > create.js
+	echo 'db.auth("$(DB_ADM_USER)", "$(DB_ADM_PASS)")' > create_mongo.js
-	echo 'use valdb' >> create.js
+	echo 'use valdb' >> create_mongo.js
-	echo 'db.createUser({user: "$(DB_USR_USER)", pwd: "$(DB_USR_PASS)", roles: ["readWrite"]})' >> create.js
+	echo 'db.createUser({user: "$(DB_USR_USER)", pwd: "$(DB_USR_PASS)", roles: ["readWrite"]})' >> create_mongo.js
+	echo 'db.grantRolesToUser("$(DB_USR_USER)", [{ role: "readWrite", db: "extvaldb"}])' >> create_mongo.js
 
-build_mongo: create.js
+build_mongo: mongo_create_roles
 	docker build . -t my-mongo --no-cache
 
-clean_stack:
+# build_postgres: postgres_create_roles
-	docker stack rm $(STACKNAME)
+# 	docker build . -t my-mongo --no-cache
 
-run_stack:
+run_docker_compose:
-	mkdir -p ${HOME}/mongo_container/data/
+	mkdir -p ${HOME}/valency_data/mongo_container/data/
-	docker stack deploy --compose-file mongodb-stack.yml $(STACKNAME)
+	#docker kill $(shell ./get_mongo_container_name.sh)
+	#docker kill $(shell ./get_postgres_container_name.sh)
+	#docker-compose stop
+	docker-compose -f valency-stack.yml up -d --force-recreate
+	# docker stack deploy --compose-file mongodb-stack.yml $(STACKNAME)
 
-create_users: create.js
+create_users: create_mongo_users create_postgres_users
-	docker exec $(shell ./get_container_name.sh) /init_inside_container.sh
+
+
+create_mongo_users: mongo_create_roles
+	docker exec $(shell ./get_mongo_container_name.sh) /init_inside_mongo_container.sh
 	# rm create.js
+
+create_postgres_users: postgres_create_roles
+	docker exec $(shell ./get_postgres_container_name.sh) /scripts/init_inside_postgres_container.sh
+
+restore_db: restore_mongo_db restore_postgres_db
+
+restore_mongo_db:
+ifeq (,$(wildcard ./mongo_db.gz))
+	$(error "mongo_db.gz does not exists. Make sure to have dump of mongo db in 'dockerfiles/database/mongo_db.gz'")
+else
+	docker exec $(shell ./get_mongo_container_name.sh) sh -c 'mongorestore --gzip --archive=/scripts/mongo_db.gz --db valdb --username $(DB_USR_USER) --password $(DB_USR_PASS) --authenticationDatabase valdb'
+endif
+
+restore_postgres_db:
+ifeq (,$(wildcard ./postgres_db.tar))
+	$(error "postgres_db.tar does not exists. Make sure to have dump of postgres db in 'dockerfiles/database/postgres_db.tar'")
+else
+	docker exec $(shell ./get_postgres_container_name.sh) sh -c 'pg_restore -U $(DB_ADM_USER) --dbname=superdb_small --create --verbose /scripts/postgres_db.tar'
+endif

dockerfiles/database/get_postgres_container_name.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker ps | grep postgres | awk '{print $1}'

dockerfiles/database/init_inside_container.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-mongo admin < /create.js

dockerfiles/database/init_inside_mongo_container.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+mongo admin < /create_mongo.js

dockerfiles/database/init_inside_postgres_container.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+/scripts/create_postgres.js

dockerfiles/database/mongodb-stack.yml

@@ -1,26 +0,0 @@
-version: '3.1'
-
-services:
-
-  my_mongo:
-    image: my-mongo
-    restart: always
-    ports:
-      - 27017:27017
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: ${DB_ADM_USER}
-      MONGO_INITDB_ROOT_PASSWORD: ${DB_ADM_PASS}
-    volumes:
-      - ${HOME}/mongo_container/data/:/data/db
-
-  mongo_express:
-    image: mongo-express
-    restart: always
-    ports:
-      - 8087:8081
-    environment:
-      ME_CONFIG_BASICAUTH_USERNAME: ${MONGOEXPRESS_USER}
-      ME_CONFIG_BASICAUTH_PASSWORD: ${MONGOEXPRESS_PASS}
-      ME_CONFIG_MONGODB_ADMINUSERNAME: ${DB_ADM_USER}
-      ME_CONFIG_MONGODB_ADMINPASSWORD: ${DB_ADM_PASS}
-      ME_CONFIG_MONGODB_SERVER: my_mongo

dockerfiles/database/valency-stack.yml

@@ -0,0 +1,27 @@
+version: '3.1'
+
+services:
+
+  my_mongo:
+    image: my-mongo
+    restart: always
+    ports:
+      - 127.0.0.1:27017:27017
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${DB_ADM_USER}
+      MONGO_INITDB_ROOT_PASSWORD: ${DB_ADM_PASS}
+    volumes:
+      - ${HOME}/valency_data/mongo_container/data/:/data/db
+      - ./:/scripts
+
+  my_postgres:
+    image: postgres
+    restart: always
+    ports:
+      - 127.0.0.1:5432:5432
+    environment:
+      POSTGRES_USER: ${DB_ADM_USER}
+      POSTGRES_PASSWORD: ${DB_ADM_PASS}
+    volumes:
+      - ${HOME}/valency_data/postgres_container/data/:/var/lib/postgresql/data
+      - ./:/scripts

dockerfiles/python-env/Dockerfile

@@ -6,7 +6,8 @@ vim \
 python3 \
 python3-pip \
 sshfs \
-curl
+curl \
+locales
 
 RUN pip3 install --upgrade pip
 
@@ -21,6 +22,16 @@ RUN pip3 install \
 	flask_cors \
 	pymongo \
 	flask-pymongo \
-	gunicorn
+	gunicorn \
+	SQLAlchemy \
+	tqdm \
+	psycopg2-binary
+
+# Set the locale
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8  
+ENV LANGUAGE en_US:en  
+ENV LC_ALL en_US.UTF-8
 
 ENV PYTHONIOENCODING UTF-8

scripts/create_xml.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #!/usr/bin/python3
 
 #imports from luscenje_struktur

src/pkg/cjvt-corpusparser

@@ -1 +1 @@
-Subproject commit 92b3ac4ea3a73b93c25b363b5b9cb096d4d011cd
+Subproject commit 01adf47b9b63b43f86bff52429792b0de2327ddd

src/pkg/luscenje_struktur

@@ -0,0 +1 @@
+Subproject commit 8c87d07b8a3ca73faac2fac30c39969bc5f97d45