diff --git a/.gitignore b/.gitignore index b6465ec..2079f94 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ data/samples/ *.pyc src/frontend_vue/node_modules/ src/frontend_vue/dist/ +dockerfiles/database/create.js diff --git a/Makefile b/Makefile index b64a148..7d3129d 100644 --- a/Makefile +++ b/Makefile @@ -15,13 +15,13 @@ OUTPUT = "db" OUTDIR = "/home/voje/workdir/test_out" DBADDR = "0.0.0.0:27017" # don't use localhost -DB_ADM_USER = testadmin -DB_ADM_PASS = testadminpass -DB_USR_USER = testuser -DB_USR_PASS = testuserpass +DB_ADM_USER = valadmin +DB_ADM_PASS = valadminpass +DB_USR_USER = valuser +DB_USR_PASS = valuserpass export -.PHONY: python-env fill_database +.PHONY: python-env fill-database all: echo "Select an argument" @@ -54,7 +54,7 @@ data/samples: cd data; tar xzvf samples.tar.gz # from inside python-env container: -fill_database: data/samples +fill-database: data/samples python3 src/pkg/cjvt-corpusparser/corpusparser/main.py --kres-folder $(KRES_FOLDER) \ --ssj-file $(SSJ_FILE) --kres-srl-folder $(KRES_SRL_FOLDER) \ --output $(OUTPUT) --outdir $(OUTDIR) --dbaddr $(DBADDR) \ @@ -76,4 +76,6 @@ backend-env: python-env-install backend-dev: python-env-install - cd ./src/backend_flask; python3 app.py --config-file ./conf_files/dev_conf.yaml + cd ./src/backend_flask; python3 app.py \ + --config-file ./conf_files/dev_conf.yaml \ + --dbuser $(DB_USR_USER) --dbpass $(DB_USR_PASS) --dbaddr $(DBADDR) diff --git a/README.md b/README.md index c9c868f..d474132 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,15 @@ Required submodules: ```bash $ git submodule init +$ git submodule update ``` ## Components ### Database (2 containers) -Spin up the database service and create users: +Set db admin, user, pass, etc in Makefile. +Spin up the database service and create users: ```bash # $ make database-clean # opt $ make database-service @@ -33,7 +35,7 @@ $ make python-env $ make python-env-install # run the code -$ make fill_database +$ make fill-database ``` If all goes well, we should be able to inspect the database on `0.0.0.0:8087`. diff --git a/dockerfiles/database/create.js b/dockerfiles/database/create.js deleted file mode 100644 index 2b12f7d..0000000 --- a/dockerfiles/database/create.js +++ /dev/null @@ -1,3 +0,0 @@ -db.auth("testadmin", "testadminpass") -use valdb -db.createUser({user: "testuser", pwd: "testuserpass", roles: ["readWrite"]}) diff --git a/dockerfiles/database/mongo_init_db.sh b/dockerfiles/database/mongo_init_db.sh deleted file mode 100755 index 0d69be3..0000000 --- a/dockerfiles/database/mongo_init_db.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -echo "exiting - check script" -exit - -# obviously, change root and password! - -# db names: nameDB -# collection names: lower case, plural -# user names? - -# mongo admin -u root -p password --eval "db.getSiblingDB('vlDB').addUser('vluser', 'password')" - -docker exec -it mongo-container /bin/bash -mongo -u root -p example -use valDB -db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]}) diff --git a/dockerfiles/database/mongo_terminator.log b/dockerfiles/database/mongo_terminator.log deleted file mode 100644 index 798f717..0000000 --- a/dockerfiles/database/mongo_terminator.log +++ /dev/null @@ -1,316 +0,0 @@ - -> -> exit() -2019-02-01T15:44:26.624+0000 E QUERY [js] ReferenceError: exit is not defined : -@(shell):1:1 -> -> -> exit -bye -# exit -┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir] -└──╼ $ -┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir] -└──╼ $ -┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir] -└──╼ $clear - -┌─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir] -└──╼ $ -┌─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir] -└──╼ $docker exec -it mng3_mongo.1.tz20az7xgqjhu4nxzokcifeyz /bin/sh -# mongo -MongoDB shell version v4.0.5 -connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb -Implicit session: session { "id" : UUID("37ea3984-719e-4381-b5da-806ef3fac970") } -MongoDB server version: 4.0.5 -> exit -bye -# -# mongo -admin root -password example -Error parsing command line: unrecognised option '-admin' -try 'mongo --help' for more information -# -# mongo -h -MongoDB shell version v4.0.5 -usage: mongo [options] [db address] [file names (ending in .js)] -db address can be: - foo foo database on local machine - 192.168.0.5/foo foo database on 192.168.0.5 machine - 192.168.0.5:9999/foo foo database on 192.168.0.5 machine on port 9999 -Options: - --shell run the shell after executing files - --nodb don't connect to mongod on startup - no - 'db address' arg expected - --norc will not run the ".mongorc.js" file on - start up - --quiet be less chatty - --port arg port to connect to - --host arg server to connect to - --eval arg evaluate javascript - -h [ --help ] show this usage information - --version show version information - --verbose increase verbosity - --ipv6 enable IPv6 support (disabled by default) - --disableJavaScriptJIT disable the Javascript Just In Time - compiler - --enableJavaScriptJIT enable the Javascript Just In Time - compiler - --disableJavaScriptProtection allow automatic JavaScript function - marshalling - --ssl use SSL for all connections - --sslCAFile arg Certificate Authority file for SSL - --sslPEMKeyFile arg PEM certificate/key file for SSL - --sslPEMKeyPassword arg password for key in PEM file for SSL - --sslCRLFile arg Certificate Revocation List file for SSL - --sslAllowInvalidHostnames allow connections to servers with - non-matching hostnames - --sslAllowInvalidCertificates allow connections to servers with invalid - certificates - --sslFIPSMode activate FIPS 140-2 mode at startup - --sslDisabledProtocols arg Comma separated list of TLS protocols to - disable [TLS1_0,TLS1_1,TLS1_2] - --retryWrites automatically retry write operations upon - transient network errors - --disableImplicitSessions do not automatically create and use - implicit sessions - --jsHeapLimitMB arg set the js scope's heap size limit - -Authentication Options: - -u [ --username ] arg username for authentication - -p [ --password ] arg password for authentication - --authenticationDatabase arg user source (defaults to dbname) - --authenticationMechanism arg authentication mechanism - --gssapiServiceName arg (=mongodb) Service name to use when authenticating - using GSSAPI/Kerberos - --gssapiHostName arg Remote host name to use for purpose of - GSSAPI/Kerberos authentication - -file names: a list of files to run. files have to end in .js and will exit after unless --shell is specified -#^[[A^[[A^C -# mongo -u root -p example -MongoDB shell version v4.0.5 -connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb -Implicit session: session { "id" : UUID("84cc9dc9-4682-40ea-b3bb-f8a4ce58e415") } -MongoDB server version: 4.0.5 -Server has startup warnings: -2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten] -2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine -2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten] ** See http://dochub.mongodb.org/core/prodnotes-filesystem ---- -Enable MongoDB's free cloud-based monitoring service, which will then receive and display -metrics about your deployment (disk utilization, CPU, operation statistics, etc). - -The monitoring data will be available on a MongoDB website with a unique URL accessible to you -and anyone you share the URL with. MongoDB may use this information to make product -improvements and to suggest MongoDB products and deployment options to you. - -To enable free monitoring, run the following command: db.enableFreeMonitoring() -To permanently disable this reminder, run the following command: db.disableFreeMonitoring() ---- - -> -> GOT IT -2019-02-01T15:45:32.432+0000 E QUERY [js] SyntaxError: missing ; before statement @(shell):1:4 -> -> -> use valDB -switched to db valDB -> db.addUser("valuser", "password") -2019-02-01T15:46:47.221+0000 E QUERY [js] TypeError: db.addUser is not a function : -@(shell):1:1 -> db.createUser({user: "kristjan", pwd: "password"}) -2019-02-01T15:49:31.025+0000 E QUERY [js] Error: couldn't add user: "createUser" command requires a "roles" array : -_getErrorWithCode@src/mongo/shell/utils.js:25:13 -DB.prototype.createUser@src/mongo/shell/db.js:1491:15 -@(shell):1:1 ->db.createUser({user: kristjan, pwd: password, roles: [readWrite]}) -2019-02-01T15:51:51.052+0000 E QUERY [js] ReferenceError: kristjan is not defined : -@(shell):1:16 -> db.createUser({user: kristjan, pwd: password, roles: [readWrite]}) -2019-02-01T15:51:58.492+0000 E QUERY [js] ReferenceError: kristjan is not defined : -@(shell):1:16 -> db.createUser({user: kristjan, pwd: password, roles: [readWrite]}) -2019-02-01T15:52:08.259+0000 E QUERY [js] ReferenceError: kristjan is not defined : -@(shell):1:16 -> db.createUser({user: kristjan, pwd: password, roles: [readWrite]}) -2019-02-01T15:52:09.212+0000 E QUERY [js] ReferenceError: kristjan is not defined : -@(shell):1:16 -> $ db.createUser({user: kristjan, pwd: password, roles: [readWrite]}) -2019-02-01T15:52:11.599+0000 E QUERY [js] SyntaxError: missing ; before statement @(shell):1:2 -> db.createUser({user: kristjan, pwd: password, roles: [readWrite]}) -2019-02-01T15:52:23.784+0000 E QUERY [js] ReferenceError: kristjan is not defined : -@(shell):1:16 -> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]}) -2019-02-01T15:52:53.218+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser : -_getErrorWithCode@src/mongo/shell/utils.js:25:13 -DB.prototype.createUser@src/mongo/shell/db.js:1491:15 -@(shell):1:1 -> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]}) -2019-02-01T15:53:27.144+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser : -_getErrorWithCode@src/mongo/shell/utils.js:25:13 -DB.prototype.createUser@src/mongo/shell/db.js:1491:15 -@(shell):1:1 -> -> -> db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]}) -Successfully added user: { "user" : "kristjan", "roles" : [ "readWrite" ] } -> -> -> show dbs -admin 0.000GB -config 0.000GB -local 0.000GB -> use testDB -switched to db testDB -> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]}) -2019-02-01T15:55:35.453+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser : -_getErrorWithCode@src/mongo/shell/utils.js:25:13 -DB.prototype.createUser@src/mongo/shell/db.js:1491:15 -@(shell):1:1 -> db.createUser({user: kristjan, pwd: password, roles: [readWrite]}) -2019-02-01T15:55:52.582+0000 E QUERY [js] ReferenceError: kristjan is not defined : -@(shell):1:16 -> -> db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]}) -Successfully added user: { "user" : "kristjan", "roles" : [ "readWrite" ] } -> -> -> show dbs -admin 0.000GB -config 0.000GB -local 0.000GB -> -> -> -> -> -> db.testdata.insert({txt: "test"}) -WriteResult({ "nInserted" : 1 }) -> -> -> use test2 -switched to db test2 -> db.createUser({user: "test", pwd: "test", roles: ["readWrite"]}) -Successfully added user: { "user" : "test", "roles" : [ "readWrite" ] } -> db.txts.insert({txt: "written by test user"})WriteResult({ "nInserted" : 1 }) -> -> -> exit -bye -# -# -# clera -/bin/sh: 9: clera: not found -# ls -bin dev home lib64 opt run sys var -boot docker-entrypoint-initdb.d js-yaml.js media proc sbin tmp -data etc lib mnt root srv usr -# -# -# clera -/bin/sh: 13: clera: not found -# ls -bin dev home lib64 opt run sys var -boot docker-entrypoint-initdb.d js-yaml.js media proc sbin tmp -data etc lib mnt root srv usr -# -# -# clear - -# -# mongo -MongoDB shell version v4.0.5 -connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb -Implicit session: session { "id" : UUID("409e49e5-61d2-454e-8e22-e721379386c9") } -MongoDB server version: 4.0.5 -> -> -> show dbs -2019-02-01T15:58:28.512+0000 E QUERY [js] Error: listDatabases failed:{ - "ok" : 0, - "errmsg" : "command listDatabases requires authentication", - "code" : 13, - "codeName" : "Unauthorized" -} : -_getErrorWithCode@src/mongo/shell/utils.js:25:13 -Mongo.prototype.getDBs@src/mongo/shell/mongo.js:124:1 -shellHelper.show@src/mongo/shell/utils.js:876:19 -shellHelper@src/mongo/shell/utils.js:766:15 -@(shellhelp2):1:1 ->use texts -switched to db texts -> use derps -switched to db derps -> -> -> use testDB -switched to db testDB -> show collections -Warning: unable to run listCollections, attempting to approximate collection names by parsing connectionStatus -> db.show -testDB.show -> -> db.list -testDB.list -> testDB.list.findAll() -2019-02-01T16:00:33.108+0000 E QUERY [js] ReferenceError: testDB is not defined : -@(shell):1:1 -> -> -> db.list.findAll() -2019-02-01T16:00:40.688+0000 E QUERY [js] TypeError: db.list.findAll is not a function : -@(shell):1:1 -> -> -> db.list.findOne() -2019-02-01T16:00:48.507+0000 E QUERY [js] Error: error: { - "ok" : 0, - "errmsg" : "command find requires authentication", - "code" : 13, - "codeName" : "Unauthorized" -} : -_getErrorWithCode@src/mongo/shell/utils.js:25:13 -DBCommandCursor@src/mongo/shell/query.js:708:1 -DBQuery.prototype._exec@src/mongo/shell/query.js:113:28 -DBQuery.prototype.hasNext@src/mongo/shell/query.js:288:5 -DBCollection.prototype.findOne@src/mongo/shell/collection.js:260:10 -@(shell):1:1 ->db.auth("kristjan", "password") -1 -> -> db.list.findAll() -2019-02-01T16:01:04.681+0000 E QUERY [js] TypeError: db.list.findAll is not a function : -@(shell):1:1 -> db.testdata.findAll() -2019-02-01T16:01:11.772+0000 E QUERY [js] TypeError: db.testdata.findAll is not a function : -@(shell):1:1 -> -> -> db.testdata.find_all() -2019-02-01T16:01:19.018+0000 E QUERY [js] TypeError: db.testdata.find_all is not a function : -@(shell):1:1 -> -> -> db.testdata.findOne -function (query, fields, options, readConcern, collation) { - var cursor = this.find(query, fields, -1 /* limit */, 0 /* skip*/, 0 /* batchSize */, options); - - if (readConcern) { - cursor = cursor.readConcern(readConcern); - } - - if (collation) { - cursor = cursor.collation(collation); - } - - if (!cursor.hasNext()) - return null; - var ret = cursor.next(); - if (cursor.hasNext()) - throw Error("findOne has more than 1 result!"); - if (ret.$err) - throw _getErrorWithCode(ret, "error " + tojson(ret)); - return ret; -} ->db.testdata.find \ No newline at end of file diff --git a/dockerfiles/python-env/Dockerfile b/dockerfiles/python-env/Dockerfile index d688ce0..266d331 100644 --- a/dockerfiles/python-env/Dockerfile +++ b/dockerfiles/python-env/Dockerfile @@ -22,5 +22,5 @@ RUN apt-get install -y \ ENV PYTHONIOENCODING UTF-8 RUN pip3 install \ - yaml \ + pyyaml \ flask_cors diff --git a/src/backend_flask/app.py b/src/backend_flask/app.py index 0161dac..4eb9312 100644 --- a/src/backend_flask/app.py +++ b/src/backend_flask/app.py @@ -23,6 +23,7 @@ import smtplib from email.mime.text import MIMEText from copy import deepcopy as DC from pathlib import Path +from pymongo import MongoClient import argparse log = logging.getLogger(__name__) @@ -377,8 +378,12 @@ def api_senses_update(): if __name__ == "__main__": + print("Starting app.py main()") aparser = argparse.ArgumentParser(description="Arguments for app.py") aparser.add_argument("--config-file", type=str, help="check ./conf_files/") + aparser.add_argument("--dbuser", type=str) + aparser.add_argument("--dbpass", type=str) + aparser.add_argument("--dbaddr", type=str) args = aparser.parse_args() config = None @@ -392,7 +397,18 @@ if __name__ == "__main__": else: logging.basicConfig(filename=logfile, level=logging.INFO) + # db login + client = MongoClient( + "mongodb://{}".format(args.dbaddr), + username=args.dbuser, + password=args.dbpass, + authSource="valdb", + authMechanism='SCRAM-SHA-256' + ) + valdb = client.valdb + # log.info("[*] Starting app.py with config:\n%s".format(config)) - print("[*] Starting app.py with config:\n{}".format(config)) + log.info("[*] Starting app.py with config:\n{}".format(config)) + sys.exit() app.run(host=str(config["host"]), port=int(config["port"]))