tested db, fill, backend
This commit is contained in:
parent
7645c8724d
commit
8c6d6ab8ab
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -3,3 +3,4 @@ data/samples/
|
|||
*.pyc
|
||||
src/frontend_vue/node_modules/
|
||||
src/frontend_vue/dist/
|
||||
dockerfiles/database/create.js
|
||||
|
|
|
|||
16
Makefile
16
Makefile
|
|
@ -15,13 +15,13 @@ OUTPUT = "db"
|
|||
OUTDIR = "/home/voje/workdir/test_out"
|
||||
DBADDR = "0.0.0.0:27017" # don't use localhost
|
||||
|
||||
DB_ADM_USER = testadmin
|
||||
DB_ADM_PASS = testadminpass
|
||||
DB_USR_USER = testuser
|
||||
DB_USR_PASS = testuserpass
|
||||
DB_ADM_USER = valadmin
|
||||
DB_ADM_PASS = valadminpass
|
||||
DB_USR_USER = valuser
|
||||
DB_USR_PASS = valuserpass
|
||||
export
|
||||
|
||||
.PHONY: python-env fill_database
|
||||
.PHONY: python-env fill-database
|
||||
|
||||
all:
|
||||
echo "Select an argument"
|
||||
|
|
@ -54,7 +54,7 @@ data/samples:
|
|||
cd data; tar xzvf samples.tar.gz
|
||||
|
||||
# from inside python-env container:
|
||||
fill_database: data/samples
|
||||
fill-database: data/samples
|
||||
python3 src/pkg/cjvt-corpusparser/corpusparser/main.py --kres-folder $(KRES_FOLDER) \
|
||||
--ssj-file $(SSJ_FILE) --kres-srl-folder $(KRES_SRL_FOLDER) \
|
||||
--output $(OUTPUT) --outdir $(OUTDIR) --dbaddr $(DBADDR) \
|
||||
|
|
@ -76,4 +76,6 @@ backend-env: python-env-install
|
|||
|
||||
|
||||
backend-dev: python-env-install
|
||||
cd ./src/backend_flask; python3 app.py --config-file ./conf_files/dev_conf.yaml
|
||||
cd ./src/backend_flask; python3 app.py \
|
||||
--config-file ./conf_files/dev_conf.yaml \
|
||||
--dbuser $(DB_USR_USER) --dbpass $(DB_USR_PASS) --dbaddr $(DBADDR)
|
||||
|
|
|
|||
|
|
@ -5,13 +5,15 @@ Required submodules:
|
|||
|
||||
```bash
|
||||
$ git submodule init
|
||||
$ git submodule update
|
||||
```
|
||||
|
||||
## Components
|
||||
|
||||
|
||||
### Database (2 containers)
|
||||
Spin up the database service and create users:
|
||||
Set db admin, user, pass, etc in Makefile.
|
||||
Spin up the database service and create users:
|
||||
```bash
|
||||
# $ make database-clean # opt
|
||||
$ make database-service
|
||||
|
|
@ -33,7 +35,7 @@ $ make python-env
|
|||
$ make python-env-install
|
||||
|
||||
# run the code
|
||||
$ make fill_database
|
||||
$ make fill-database
|
||||
```
|
||||
|
||||
If all goes well, we should be able to inspect the database on `0.0.0.0:8087`.
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
db.auth("testadmin", "testadminpass")
|
||||
use valdb
|
||||
db.createUser({user: "testuser", pwd: "testuserpass", roles: ["readWrite"]})
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
#!/bin/bash
|
||||
echo "exiting - check script"
|
||||
exit
|
||||
|
||||
# obviously, change root and password!
|
||||
|
||||
# db names: nameDB
|
||||
# collection names: lower case, plural
|
||||
# user names?
|
||||
|
||||
# mongo admin -u root -p password --eval "db.getSiblingDB('vlDB').addUser('vluser', 'password')"
|
||||
|
||||
docker exec -it mongo-container /bin/bash
|
||||
mongo -u root -p example
|
||||
use valDB
|
||||
db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]})
|
||||
|
|
@ -1,316 +0,0 @@
|
|||
|
||||
>
|
||||
> exit()
|
||||
2019-02-01T15:44:26.624+0000 E QUERY [js] ReferenceError: exit is not defined :
|
||||
@(shell):1:1
|
||||
>
|
||||
>
|
||||
> exit
|
||||
bye
|
||||
# exit
|
||||
┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
|
||||
└──╼ $
|
||||
┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
|
||||
└──╼ $
|
||||
┌─[✗]─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
|
||||
└──╼ $clear
|
||||
|
||||
┌─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
|
||||
└──╼ $
|
||||
┌─[kristjan@k-arch]─[/home/kristjan/Desktop/workdir]
|
||||
└──╼ $docker exec -it mng3_mongo.1.tz20az7xgqjhu4nxzokcifeyz /bin/sh
|
||||
# mongo
|
||||
MongoDB shell version v4.0.5
|
||||
connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb
|
||||
Implicit session: session { "id" : UUID("37ea3984-719e-4381-b5da-806ef3fac970") }
|
||||
MongoDB server version: 4.0.5
|
||||
> exit
|
||||
bye
|
||||
#
|
||||
# mongo -admin root -password example
|
||||
Error parsing command line: unrecognised option '-admin'
|
||||
try 'mongo --help' for more information
|
||||
#
|
||||
# mongo -h
|
||||
MongoDB shell version v4.0.5
|
||||
usage: mongo [options] [db address] [file names (ending in .js)]
|
||||
db address can be:
|
||||
foo foo database on local machine
|
||||
192.168.0.5/foo foo database on 192.168.0.5 machine
|
||||
192.168.0.5:9999/foo foo database on 192.168.0.5 machine on port 9999
|
||||
Options:
|
||||
--shell run the shell after executing files
|
||||
--nodb don't connect to mongod on startup - no
|
||||
'db address' arg expected
|
||||
--norc will not run the ".mongorc.js" file on
|
||||
start up
|
||||
--quiet be less chatty
|
||||
--port arg port to connect to
|
||||
--host arg server to connect to
|
||||
--eval arg evaluate javascript
|
||||
-h [ --help ] show this usage information
|
||||
--version show version information
|
||||
--verbose increase verbosity
|
||||
--ipv6 enable IPv6 support (disabled by default)
|
||||
--disableJavaScriptJIT disable the Javascript Just In Time
|
||||
compiler
|
||||
--enableJavaScriptJIT enable the Javascript Just In Time
|
||||
compiler
|
||||
--disableJavaScriptProtection allow automatic JavaScript function
|
||||
marshalling
|
||||
--ssl use SSL for all connections
|
||||
--sslCAFile arg Certificate Authority file for SSL
|
||||
--sslPEMKeyFile arg PEM certificate/key file for SSL
|
||||
--sslPEMKeyPassword arg password for key in PEM file for SSL
|
||||
--sslCRLFile arg Certificate Revocation List file for SSL
|
||||
--sslAllowInvalidHostnames allow connections to servers with
|
||||
non-matching hostnames
|
||||
--sslAllowInvalidCertificates allow connections to servers with invalid
|
||||
certificates
|
||||
--sslFIPSMode activate FIPS 140-2 mode at startup
|
||||
--sslDisabledProtocols arg Comma separated list of TLS protocols to
|
||||
disable [TLS1_0,TLS1_1,TLS1_2]
|
||||
--retryWrites automatically retry write operations upon
|
||||
transient network errors
|
||||
--disableImplicitSessions do not automatically create and use
|
||||
implicit sessions
|
||||
--jsHeapLimitMB arg set the js scope's heap size limit
|
||||
|
||||
Authentication Options:
|
||||
-u [ --username ] arg username for authentication
|
||||
-p [ --password ] arg password for authentication
|
||||
--authenticationDatabase arg user source (defaults to dbname)
|
||||
--authenticationMechanism arg authentication mechanism
|
||||
--gssapiServiceName arg (=mongodb) Service name to use when authenticating
|
||||
using GSSAPI/Kerberos
|
||||
--gssapiHostName arg Remote host name to use for purpose of
|
||||
GSSAPI/Kerberos authentication
|
||||
|
||||
file names: a list of files to run. files have to end in .js and will exit after unless --shell is specified
|
||||
#^[[A^[[A^C
|
||||
# mongo -u root -p example
|
||||
MongoDB shell version v4.0.5
|
||||
connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb
|
||||
Implicit session: session { "id" : UUID("84cc9dc9-4682-40ea-b3bb-f8a4ce58e415") }
|
||||
MongoDB server version: 4.0.5
|
||||
Server has startup warnings:
|
||||
2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten]
|
||||
2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
|
||||
2019-02-01T15:32:02.149+0000 I STORAGE [initandlisten] ** See http://dochub.mongodb.org/core/prodnotes-filesystem
|
||||
---
|
||||
Enable MongoDB's free cloud-based monitoring service, which will then receive and display
|
||||
metrics about your deployment (disk utilization, CPU, operation statistics, etc).
|
||||
|
||||
The monitoring data will be available on a MongoDB website with a unique URL accessible to you
|
||||
and anyone you share the URL with. MongoDB may use this information to make product
|
||||
improvements and to suggest MongoDB products and deployment options to you.
|
||||
|
||||
To enable free monitoring, run the following command: db.enableFreeMonitoring()
|
||||
To permanently disable this reminder, run the following command: db.disableFreeMonitoring()
|
||||
---
|
||||
|
||||
>
|
||||
> GOT IT
|
||||
2019-02-01T15:45:32.432+0000 E QUERY [js] SyntaxError: missing ; before statement @(shell):1:4
|
||||
>
|
||||
>
|
||||
> use valDB
|
||||
switched to db valDB
|
||||
> db.addUser("valuser", "password")
|
||||
2019-02-01T15:46:47.221+0000 E QUERY [js] TypeError: db.addUser is not a function :
|
||||
@(shell):1:1
|
||||
> db.createUser({user: "kristjan", pwd: "password"})
|
||||
2019-02-01T15:49:31.025+0000 E QUERY [js] Error: couldn't add user: "createUser" command requires a "roles" array :
|
||||
_getErrorWithCode@src/mongo/shell/utils.js:25:13
|
||||
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
|
||||
@(shell):1:1
|
||||
>db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
|
||||
2019-02-01T15:51:51.052+0000 E QUERY [js] ReferenceError: kristjan is not defined :
|
||||
@(shell):1:16
|
||||
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
|
||||
2019-02-01T15:51:58.492+0000 E QUERY [js] ReferenceError: kristjan is not defined :
|
||||
@(shell):1:16
|
||||
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
|
||||
2019-02-01T15:52:08.259+0000 E QUERY [js] ReferenceError: kristjan is not defined :
|
||||
@(shell):1:16
|
||||
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
|
||||
2019-02-01T15:52:09.212+0000 E QUERY [js] ReferenceError: kristjan is not defined :
|
||||
@(shell):1:16
|
||||
> $ db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
|
||||
2019-02-01T15:52:11.599+0000 E QUERY [js] SyntaxError: missing ; before statement @(shell):1:2
|
||||
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
|
||||
2019-02-01T15:52:23.784+0000 E QUERY [js] ReferenceError: kristjan is not defined :
|
||||
@(shell):1:16
|
||||
> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]})
|
||||
2019-02-01T15:52:53.218+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser :
|
||||
_getErrorWithCode@src/mongo/shell/utils.js:25:13
|
||||
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
|
||||
@(shell):1:1
|
||||
> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]})
|
||||
2019-02-01T15:53:27.144+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser :
|
||||
_getErrorWithCode@src/mongo/shell/utils.js:25:13
|
||||
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
|
||||
@(shell):1:1
|
||||
>
|
||||
>
|
||||
> db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]})
|
||||
Successfully added user: { "user" : "kristjan", "roles" : [ "readWrite" ] }
|
||||
>
|
||||
>
|
||||
> show dbs
|
||||
admin 0.000GB
|
||||
config 0.000GB
|
||||
local 0.000GB
|
||||
> use testDB
|
||||
switched to db testDB
|
||||
> db.createUser({user: "kristjan", pwd: "password", rolse: ["readWrite"]})
|
||||
2019-02-01T15:55:35.453+0000 E QUERY [js] Error: couldn't add user: "rolse" is not a valid argument to createUser :
|
||||
_getErrorWithCode@src/mongo/shell/utils.js:25:13
|
||||
DB.prototype.createUser@src/mongo/shell/db.js:1491:15
|
||||
@(shell):1:1
|
||||
> db.createUser({user: kristjan, pwd: password, roles: [readWrite]})
|
||||
2019-02-01T15:55:52.582+0000 E QUERY [js] ReferenceError: kristjan is not defined :
|
||||
@(shell):1:16
|
||||
>
|
||||
> db.createUser({user: "kristjan", pwd: "password", roles: ["readWrite"]})
|
||||
Successfully added user: { "user" : "kristjan", "roles" : [ "readWrite" ] }
|
||||
>
|
||||
>
|
||||
> show dbs
|
||||
admin 0.000GB
|
||||
config 0.000GB
|
||||
local 0.000GB
|
||||
>
|
||||
>
|
||||
>
|
||||
>
|
||||
>
|
||||
> db.testdata.insert({txt: "test"})
|
||||
WriteResult({ "nInserted" : 1 })
|
||||
>
|
||||
>
|
||||
> use test2
|
||||
switched to db test2
|
||||
> db.createUser({user: "test", pwd: "test", roles: ["readWrite"]})
|
||||
Successfully added user: { "user" : "test", "roles" : [ "readWrite" ] }
|
||||
> db.txts.insert({txt: "written by test user"})WriteResult({ "nInserted" : 1 })
|
||||
>
|
||||
>
|
||||
> exit
|
||||
bye
|
||||
#
|
||||
#
|
||||
# clera
|
||||
/bin/sh: 9: clera: not found
|
||||
# ls
|
||||
bin dev home lib64 opt run sys var
|
||||
boot docker-entrypoint-initdb.d js-yaml.js media proc sbin tmp
|
||||
data etc lib mnt root srv usr
|
||||
#
|
||||
#
|
||||
# clera
|
||||
/bin/sh: 13: clera: not found
|
||||
# ls
|
||||
bin dev home lib64 opt run sys var
|
||||
boot docker-entrypoint-initdb.d js-yaml.js media proc sbin tmp
|
||||
data etc lib mnt root srv usr
|
||||
#
|
||||
#
|
||||
# clear
|
||||
|
||||
#
|
||||
# mongo
|
||||
MongoDB shell version v4.0.5
|
||||
connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb
|
||||
Implicit session: session { "id" : UUID("409e49e5-61d2-454e-8e22-e721379386c9") }
|
||||
MongoDB server version: 4.0.5
|
||||
>
|
||||
>
|
||||
> show dbs
|
||||
2019-02-01T15:58:28.512+0000 E QUERY [js] Error: listDatabases failed:{
|
||||
"ok" : 0,
|
||||
"errmsg" : "command listDatabases requires authentication",
|
||||
"code" : 13,
|
||||
"codeName" : "Unauthorized"
|
||||
} :
|
||||
_getErrorWithCode@src/mongo/shell/utils.js:25:13
|
||||
Mongo.prototype.getDBs@src/mongo/shell/mongo.js:124:1
|
||||
shellHelper.show@src/mongo/shell/utils.js:876:19
|
||||
shellHelper@src/mongo/shell/utils.js:766:15
|
||||
@(shellhelp2):1:1
|
||||
>use texts
|
||||
switched to db texts
|
||||
> use derps
|
||||
switched to db derps
|
||||
>
|
||||
>
|
||||
> use testDB
|
||||
switched to db testDB
|
||||
> show collections
|
||||
Warning: unable to run listCollections, attempting to approximate collection names by parsing connectionStatus
|
||||
> db.show
|
||||
testDB.show
|
||||
>
|
||||
> db.list
|
||||
testDB.list
|
||||
> testDB.list.findAll()
|
||||
2019-02-01T16:00:33.108+0000 E QUERY [js] ReferenceError: testDB is not defined :
|
||||
@(shell):1:1
|
||||
>
|
||||
>
|
||||
> db.list.findAll()
|
||||
2019-02-01T16:00:40.688+0000 E QUERY [js] TypeError: db.list.findAll is not a function :
|
||||
@(shell):1:1
|
||||
>
|
||||
>
|
||||
> db.list.findOne()
|
||||
2019-02-01T16:00:48.507+0000 E QUERY [js] Error: error: {
|
||||
"ok" : 0,
|
||||
"errmsg" : "command find requires authentication",
|
||||
"code" : 13,
|
||||
"codeName" : "Unauthorized"
|
||||
} :
|
||||
_getErrorWithCode@src/mongo/shell/utils.js:25:13
|
||||
DBCommandCursor@src/mongo/shell/query.js:708:1
|
||||
DBQuery.prototype._exec@src/mongo/shell/query.js:113:28
|
||||
DBQuery.prototype.hasNext@src/mongo/shell/query.js:288:5
|
||||
DBCollection.prototype.findOne@src/mongo/shell/collection.js:260:10
|
||||
@(shell):1:1
|
||||
>db.auth("kristjan", "password")
|
||||
1
|
||||
>
|
||||
> db.list.findAll()
|
||||
2019-02-01T16:01:04.681+0000 E QUERY [js] TypeError: db.list.findAll is not a function :
|
||||
@(shell):1:1
|
||||
> db.testdata.findAll()
|
||||
2019-02-01T16:01:11.772+0000 E QUERY [js] TypeError: db.testdata.findAll is not a function :
|
||||
@(shell):1:1
|
||||
>
|
||||
>
|
||||
> db.testdata.find_all()
|
||||
2019-02-01T16:01:19.018+0000 E QUERY [js] TypeError: db.testdata.find_all is not a function :
|
||||
@(shell):1:1
|
||||
>
|
||||
>
|
||||
> db.testdata.findOne
|
||||
function (query, fields, options, readConcern, collation) {
|
||||
var cursor = this.find(query, fields, -1 /* limit */, 0 /* skip*/, 0 /* batchSize */, options);
|
||||
|
||||
if (readConcern) {
|
||||
cursor = cursor.readConcern(readConcern);
|
||||
}
|
||||
|
||||
if (collation) {
|
||||
cursor = cursor.collation(collation);
|
||||
}
|
||||
|
||||
if (!cursor.hasNext())
|
||||
return null;
|
||||
var ret = cursor.next();
|
||||
if (cursor.hasNext())
|
||||
throw Error("findOne has more than 1 result!");
|
||||
if (ret.$err)
|
||||
throw _getErrorWithCode(ret, "error " + tojson(ret));
|
||||
return ret;
|
||||
}
|
||||
>db.testdata.find
|
||||
|
|
@ -22,5 +22,5 @@ RUN apt-get install -y \
|
|||
ENV PYTHONIOENCODING UTF-8
|
||||
|
||||
RUN pip3 install \
|
||||
yaml \
|
||||
pyyaml \
|
||||
flask_cors
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import smtplib
|
|||
from email.mime.text import MIMEText
|
||||
from copy import deepcopy as DC
|
||||
from pathlib import Path
|
||||
from pymongo import MongoClient
|
||||
import argparse
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -377,8 +378,12 @@ def api_senses_update():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Starting app.py main()")
|
||||
aparser = argparse.ArgumentParser(description="Arguments for app.py")
|
||||
aparser.add_argument("--config-file", type=str, help="check ./conf_files/")
|
||||
aparser.add_argument("--dbuser", type=str)
|
||||
aparser.add_argument("--dbpass", type=str)
|
||||
aparser.add_argument("--dbaddr", type=str)
|
||||
args = aparser.parse_args()
|
||||
|
||||
config = None
|
||||
|
|
@ -392,7 +397,18 @@ if __name__ == "__main__":
|
|||
else:
|
||||
logging.basicConfig(filename=logfile, level=logging.INFO)
|
||||
|
||||
# db login
|
||||
client = MongoClient(
|
||||
"mongodb://{}".format(args.dbaddr),
|
||||
username=args.dbuser,
|
||||
password=args.dbpass,
|
||||
authSource="valdb",
|
||||
authMechanism='SCRAM-SHA-256'
|
||||
)
|
||||
valdb = client.valdb
|
||||
|
||||
# log.info("[*] Starting app.py with config:\n%s".format(config))
|
||||
print("[*] Starting app.py with config:\n{}".format(config))
|
||||
log.info("[*] Starting app.py with config:\n{}".format(config))
|
||||
sys.exit()
|
||||
|
||||
app.run(host=str(config["host"]), port=int(config["port"]))
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user