lkrsnik/cjvt-valency
1
0
Fork 0
forked from kristjan/cjvt-valency
Code Issues Pull Requests Releases Wiki Activity

tested db, fill, backend

Browse Source
This commit is contained in:
voje 2019-03-23 20:27:23 +01:00
parent 7645c8724d
commit 8c6d6ab8ab
8 changed files with 32 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,3 +3,4 @@ data/samples/
*.pyc
src/frontend_vue/node_modules/
src/frontend_vue/dist/
dockerfiles/database/create.js

16
Makefile
View File

@ -15,13 +15,13 @@ OUTPUT = "db"
OUTDIR = "/home/voje/workdir/test_out"
DBADDR = "0.0.0.0:27017" # don't use localhost
DB_ADM_USER = testadmin
DB_ADM_PASS = testadminpass
DB_USR_USER = testuser
DB_USR_PASS = testuserpass
DB_ADM_USER = valadmin
DB_ADM_PASS = valadminpass
DB_USR_USER = valuser
DB_USR_PASS = valuserpass
export
.PHONY: python-env fill_database
.PHONY: python-env fill-database
all:
echo "Select an argument"
@ -54,7 +54,7 @@ data/samples:
cd data; tar xzvf samples.tar.gz
# from inside python-env container:
fill_database: data/samples
fill-database: data/samples
python3 src/pkg/cjvt-corpusparser/corpusparser/main.py --kres-folder $(KRES_FOLDER) \
--ssj-file $(SSJ_FILE) --kres-srl-folder $(KRES_SRL_FOLDER) \
--output $(OUTPUT) --outdir $(OUTDIR) --dbaddr $(DBADDR) \
@ -76,4 +76,6 @@ backend-env: python-env-install
backend-dev: python-env-install
cd ./src/backend_flask; python3 app.py --config-file ./conf_files/dev_conf.yaml
cd ./src/backend_flask; python3 app.py \
--config-file ./conf_files/dev_conf.yaml \
--dbuser $(DB_USR_USER) --dbpass $(DB_USR_PASS) --dbaddr $(DBADDR)

4
README.md
View File

@ -5,12 +5,14 @@ Required submodules:
```bash
$ git submodule init
$ git submodule update
```
## Components
### Database (2 containers)
Set db admin, user, pass, etc in Makefile.
Spin up the database service and create users:
```bash
# $ make database-clean # opt
@ -33,7 +35,7 @@ $ make python-env
$ make python-env-install
# run the code
$ make fill_database
$ make fill-database
```
If all goes well, we should be able to inspect the database on `0.0.0.0:8087`.

3
dockerfiles/database/create.js
View File

@ -1,3 +0,0 @@
db.auth("testadmin", "testadminpass")
use valdb
db.createUser({user: "testuser", pwd: "testuserpass", roles: ["readWrite"]})

16
dockerfiles/database/mongo_init_db.sh
View File

@ -1,16 +0,0 @@
#!/bin/bash
echo "exiting - check script"
exit
# obviously, change root and password!
# db names: nameDB
# collection names: lower case, plural
# user names?
# mongo admin -u root -p password --eval "db.getSiblingDB('vlDB').addUser('vluser', 'password')"
docker exec -it mongo-container /bin/bash
mongo -u root -p example
use valDB
db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]})

316
dockerfiles/database/mongo_terminator.log
View File

@ -1,316 +0,0 @@
>
> exit()
2019-02-01T15:44:26.624+0000 E QUERY [js] ReferenceError: exit is not defined :
@(shell):1:1
>
>
> exit
bye
# exit
┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
└──╼ $
┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
└──╼ $
┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
└──╼ $clear
┌─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
└──╼ $
┌─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
└──╼ $docker exec -it mng3_mongo.1.tz20az7xgqjhu4nxzokcifeyz /bin/sh
# mongo
MongoDB shell version v4.0.5
connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("37ea3984-719e-4381-b5da-806ef3fac970") }
MongoDB server version: 4.0.5
> exit
bye
#
# mongo -admin root -password example
Error parsing command line: unrecognised option '-admin'
try 'mongo --help' for more information
#
# mongo -h
MongoDB shell version v4.0.5
usage: mongo [options] [db address] [file names (ending in .js)]
db address can be:
foo foo database on local machine
192.168.0.5/foo foo database on 192.168.0.5 machine
192.168.0.5:9999/foo foo database on 192.168.0.5 machine on port 9999
Options:
--shell run the shell after executing files
--nodb don't connect to mongod on startup - no
'db address' arg expected
--norc will not run the ".mongorc.js" file on
start up
--quiet be less chatty
--port arg port to connect to
--host arg server to connect to
--eval arg evaluate javascript
-h [ --help ] show this usage information
--version show version information
--verbose increase verbosity
--ipv6 enable IPv6 support (disabled by default)
--disableJavaScriptJIT disable the Javascript Just In Time
compiler
--enableJavaScriptJIT enable the Javascript Just In Time
compiler
--disableJavaScriptProtection allow automatic JavaScript function
marshalling
--ssl use SSL for all connections
--sslCAFile arg Certificate Authority file for SSL
--sslPEMKeyFile arg PEM certificate/key file for SSL
--sslPEMKeyPassword arg password for key in PEM file for SSL
--sslCRLFile arg Certificate Revocation List file for SSL
--sslAllowInvalidHostnames allow connections to servers with
non-matching hostnames
--sslAllowInvalidCertificates allow connections to servers with invalid
certificates
--sslFIPSMode activate FIPS 140-2 mode at startup
--sslDisabledProtocols arg Comma separated list of TLS protocols to
disable [TLS1_0,TLS1_1,TLS1_2]
--retryWrites automatically retry write operations upon
transient network errors
--disableImplicitSessions do not automatically create and use
implicit sessions
--jsHeapLimitMB arg set the js scope's heap size limit
Authentication Options:
-u [ --username ] arg username for authentication
-p [ --password ] arg password for authentication
--authenticationDatabase arg user source (defaults to dbname)
--authenticationMechanism arg authentication mechanism
--gssapiServiceName arg (=mongodb) Service name to use when authenticating
using GSSAPI/Kerberos
--gssapiHostName arg Remote host name to use for purpose of
GSSAPI/Kerberos authentication
file names: a list of files to run. files have to end in .js and will exit after unless --shell is specified
#^[[A^[[A^C
# mongo -u root -p example
MongoDB shell version v4.0.5
connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("84cc9dc9-4682-40ea-b3bb-f8a4ce58e415") }
MongoDB server version: 4.0.5
Server has startup warnings:
2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten]
2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten] ** See http://dochub.mongodb.org/core/prodnotes-filesystem
---
Enable MongoDB's free cloud-based monitoring service, which will then receive and display
metrics about your deployment (disk utilization, CPU, operation statistics, etc).
The monitoring data will be available on a MongoDB website with a unique URL accessible to you
and anyone you share the URL with. MongoDB may use this information to make product
improvements and to suggest MongoDB products and deployment options to you.
To enable free monitoring, run the following command: db.enableFreeMonitoring()
To permanently disable this reminder, run the following command: db.disableFreeMonitoring()
---
>
> GOT IT
2019-02-01T15:45:32.432+0000 E QUERY [js] SyntaxError: missing ; before statement @(shell):1:4
>
>
> use valDB
switched to db valDB
> db.addUser("valuser", "password")
2019-02-01T15:46:47.221+0000 E QUERY [js] TypeError: db.addUser is not a function :
@(shell):1:1
> db.createUser({user: "kristjan", pwd: "password"})
2019-02-01T15:49:31.025+0000 E QUERY [js] Error: couldn't add user: "createUser" command requires a "roles" array :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
@(shell):1:1
>db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
2019-02-01T15:51:51.052+0000 E QUERY [js] ReferenceError: kristjan is not defined :
@(shell):1:16
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
2019-02-01T15:51:58.492+0000 E QUERY [js] ReferenceError: kristjan is not defined :
@(shell):1:16
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
2019-02-01T15:52:08.259+0000 E QUERY [js] ReferenceError: kristjan is not defined :
@(shell):1:16
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
2019-02-01T15:52:09.212+0000 E QUERY [js] ReferenceError: kristjan is not defined :
@(shell):1:16
> $ db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
2019-02-01T15:52:11.599+0000 E QUERY [js] SyntaxError: missing ; before statement @(shell):1:2
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
2019-02-01T15:52:23.784+0000 E QUERY [js] ReferenceError: kristjan is not defined :
@(shell):1:16
> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]})
2019-02-01T15:52:53.218+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
@(shell):1:1
> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]})
2019-02-01T15:53:27.144+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
@(shell):1:1
>
>
> db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]})
Successfully added user: { "user" : "kristjan", "roles" : [ "readWrite" ] }
>
>
> show dbs
admin 0.000GB
config 0.000GB
local 0.000GB
> use testDB
switched to db testDB
> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]})
2019-02-01T15:55:35.453+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
@(shell):1:1
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
2019-02-01T15:55:52.582+0000 E QUERY [js] ReferenceError: kristjan is not defined :
@(shell):1:16
>
> db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]})
Successfully added user: { "user" : "kristjan", "roles" : [ "readWrite" ] }
>
>
> show dbs
admin 0.000GB
config 0.000GB
local 0.000GB
>
>
>
>
>
> db.testdata.insert({txt: "test"})
WriteResult({ "nInserted" : 1 })
>
>
> use test2
switched to db test2
> db.createUser({user: "test", pwd: "test", roles: ["readWrite"]})
Successfully added user: { "user" : "test", "roles" : [ "readWrite" ] }
> db.txts.insert({txt: "written by test user"})WriteResult({ "nInserted" : 1 })
>
>
> exit
bye
#
#
# clera
/bin/sh: 9: clera: not found
# ls
bin dev home lib64 opt run sys var
boot docker-entrypoint-initdb.d js-yaml.js media proc sbin tmp
data etc lib mnt root srv usr
#
#
# clera
/bin/sh: 13: clera: not found
# ls
bin dev home lib64 opt run sys var
boot docker-entrypoint-initdb.d js-yaml.js media proc sbin tmp
data etc lib mnt root srv usr
#
#
# clear
#
# mongo
MongoDB shell version v4.0.5
connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("409e49e5-61d2-454e-8e22-e721379386c9") }
MongoDB server version: 4.0.5
>
>
> show dbs
2019-02-01T15:58:28.512+0000 E QUERY [js] Error: listDatabases failed:{
"ok" : 0,
"errmsg" : "command listDatabases requires authentication",
"code" : 13,
"codeName" : "Unauthorized"
} :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
Mongo.prototype.getDBs@src/mongo/shell/mongo.js:124:1
shellHelper.show@src/mongo/shell/utils.js:876:19
shellHelper@src/mongo/shell/utils.js:766:15
@(shellhelp2):1:1
>use texts
switched to db texts
> use derps
switched to db derps
>
>
> use testDB
switched to db testDB
> show collections
Warning: unable to run listCollections, attempting to approximate collection names by parsing connectionStatus
> db.show
testDB.show
>
> db.list
testDB.list
> testDB.list.findAll()
2019-02-01T16:00:33.108+0000 E QUERY [js] ReferenceError: testDB is not defined :
@(shell):1:1
>
>
> db.list.findAll()
2019-02-01T16:00:40.688+0000 E QUERY [js] TypeError: db.list.findAll is not a function :
@(shell):1:1
>
>
> db.list.findOne()
2019-02-01T16:00:48.507+0000 E QUERY [js] Error: error: {
"ok" : 0,
"errmsg" : "command find requires authentication",
"code" : 13,
"codeName" : "Unauthorized"
} :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
DBCommandCursor@src/mongo/shell/query.js:708:1
DBQuery.prototype._exec@src/mongo/shell/query.js:113:28
DBQuery.prototype.hasNext@src/mongo/shell/query.js:288:5
DBCollection.prototype.findOne@src/mongo/shell/collection.js:260:10
@(shell):1:1
>db.auth("kristjan", "password")
1
>
> db.list.findAll()
2019-02-01T16:01:04.681+0000 E QUERY [js] TypeError: db.list.findAll is not a function :
@(shell):1:1
> db.testdata.findAll()
2019-02-01T16:01:11.772+0000 E QUERY [js] TypeError: db.testdata.findAll is not a function :
@(shell):1:1
>
>
> db.testdata.find_all()
2019-02-01T16:01:19.018+0000 E QUERY [js] TypeError: db.testdata.find_all is not a function :
@(shell):1:1
>
>
> db.testdata.findOne
function (query, fields, options, readConcern, collation) {
var cursor = this.find(query, fields, -1 /* limit */, 0 /* skip*/, 0 /* batchSize */, options);
if (readConcern) {
cursor = cursor.readConcern(readConcern);
}
if (collation) {
cursor = cursor.collation(collation);
}
if (!cursor.hasNext())
return null;
var ret = cursor.next();
if (cursor.hasNext())
throw Error("findOne has more than 1 result!");
if (ret.$err)
throw _getErrorWithCode(ret, "error " + tojson(ret));
return ret;
}
>db.testdata.find

2
dockerfiles/python-env/Dockerfile
View File

@ -22,5 +22,5 @@ RUN apt-get install -y \
ENV PYTHONIOENCODING UTF-8
RUN pip3 install \
yaml \
pyyaml \
flask_cors

18
src/backend_flask/app.py
View File

@ -23,6 +23,7 @@ import smtplib
from email.mime.text import MIMEText
from copy import deepcopy as DC
from pathlib import Path
from pymongo import MongoClient
import argparse
log = logging.getLogger(__name__)
@ -377,8 +378,12 @@ def api_senses_update():
if __name__ == "__main__":
print("Starting app.py main()")
aparser = argparse.ArgumentParser(description="Arguments for app.py")
aparser.add_argument("--config-file", type=str, help="check ./conf_files/")
aparser.add_argument("--dbuser", type=str)
aparser.add_argument("--dbpass", type=str)
aparser.add_argument("--dbaddr", type=str)
args = aparser.parse_args()
config = None
@ -392,7 +397,18 @@ if __name__ == "__main__":
else:
logging.basicConfig(filename=logfile, level=logging.INFO)
# db login
client = MongoClient(
"mongodb://{}".format(args.dbaddr),
username=args.dbuser,
password=args.dbpass,
authSource="valdb",
authMechanism='SCRAM-SHA-256'
)
valdb = client.valdb
# log.info("[*] Starting app.py with config:\n%s".format(config))
print("[*] Starting app.py with config:\n{}".format(config))
log.info("[*] Starting app.py with config:\n{}".format(config))
sys.exit()
app.run(host=str(config["host"]), port=int(config["port"]))