From bc4e8e2cf1ab47c5c0ec0d7869110e291c2c7f8c Mon Sep 17 00:00:00 2001 From: voje Date: Sun, 24 Mar 2019 02:08:22 +0100 Subject: [PATCH] working on api + frontend; todo: switch corpus --- README.md | 32 ++++---- src/backend_flask/app.py | 74 +++++++++++++++++-- src/backend_flask/conf_files/dev_conf.yaml | 1 + src/frontend_vue/config/config.json | 2 +- src/frontend_vue/config/config_dev.json | 2 +- src/frontend_vue/config/config_prod.json | 2 +- src/frontend_vue/package-lock.json | 40 +++++++--- src/frontend_vue/src/components/Home.vue | 7 +- src/frontend_vue/src/components/LWords.vue | 2 +- src/frontend_vue/src/components/MainDispl.vue | 4 +- src/frontend_vue/src/components/Nav.vue | 35 +++++++-- src/frontend_vue/src/main.js | 3 +- 12 files changed, 149 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index d474132..91236b2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ $ git submodule update ### Database (2 containers) -Set db admin, user, pass, etc in Makefile. +Set db admin, user, pass, etc in 'Makefile'. Spin up the database service and create users: ```bash # $ make database-clean # opt @@ -38,11 +38,15 @@ $ make python-env-install $ make fill-database ``` -If all goes well, we should be able to inspect the database on `0.0.0.0:8087`. +If all goes well, we should be able to inspect the database, filled with corpora, on `0.0.0.0:8087`. ### Flask backend (1 container) -Input: see Database +Relies heavily on the database. Set that up first. +```bash +# $ make backend=dev # development +$ make backend-prod +``` API endpoints: @@ -53,17 +57,13 @@ API endpoints: ### Vue frontend (1 container) +Relies on Flask backend. +Before running `make`, you might need to set the correct api address. +Check `./src/frontend_vue/config/config_prod.json`. +bash +``` +# $ make frontend-dev # development +$ make frontend-prod +``` -* ngnix server - - -## Deployment - -Preflight: -* get up DB -* prepare DB - -Flight: -* start backend -* start frontend - +App available on: `http://0.0.0.0:8080`. diff --git a/src/backend_flask/app.py b/src/backend_flask/app.py index 4eb9312..c4cf664 100644 --- a/src/backend_flask/app.py +++ b/src/backend_flask/app.py @@ -26,9 +26,13 @@ from pathlib import Path from pymongo import MongoClient import argparse +CORPORA = ["ssj", "kres"] + log = logging.getLogger(__name__) app = Flask(__name__) +app_index = {c: {} for c in CORPORA} + # when running vuejs via webpack # CORS(app) CORS(app, resources={r"/api/*": { @@ -54,20 +58,18 @@ def home(pathname): return redirect(url_for("index"), code=302) +# @app.route("/api/words/") +# def api_words(corpus): @app.route("/api/words") def api_words(): return json.dumps({ - "sorted_words": vallex.sorted_words, - "has_se": vallex.has_se + "sorted_words": app_index["ssj"]["words"], # todo - make corpus as arg }) - @app.route("/api/functors") def api_functors(): - res = [] - for key in sorted(vallex.functors_index.keys()): - res.append((key, len(vallex.functors_index[key]))) - return json.dumps(res) + # return array ([functor, len]) + return json.dumps(app_index["ssj"]["functors"]) @app.route("/api/register", methods=["POST"]) @@ -376,6 +378,60 @@ def api_senses_update(): vallex.db["v2_sense_map"].insert(data) return "OK" +def prepare_db(): + def helper_tid_to_token(tid, tokens): + for t in tokens: + if t["tid"] == tid: + return t + return None + + # update entries (add headwords and fuctors for indexing) + for corpus in ["ssj", "kres"]: + for e in valdb[corpus].find({}): + #! hw_tids are not array ids + hw_tids = list(set([x["from"] for x in e["srl_links"]])) + hw_tokens = [helper_tid_to_token(tid, e["tokens"]) for tid in hw_tids] + headwords = [(t["lemma"] if t["msd"][0] == "G" else t["lemma"] + "_") for t in hw_tokens] + e["headwords"] = headwords + + functors = list(set([x["afun"] for x in e["srl_links"]])) + e["headwords"] = headwords + e["functors"] = functors + + valdb[corpus].save(e) + + # create app_index (used in frontend, left side word index) + for corpus in CORPORA: + res_hws = {} + res_fns = {} + for e in valdb[corpus].find({}): + for hw in e["headwords"]: + if hw in res_hws: + res_hws[hw] += 1 + else: + res_hws[hw] = 1 + for fn in e["functors"]: + if fn in res_fns: + res_fns[fn] += 1 + else: + res_fns[fn] = 1 + + alphabetical = {} + for k, e in res_hws.items(): + fst = k[0].lower() + if fst in alphabetical: + alphabetical[fst].append((k, e)) + else: + alphabetical[fst] = [(k, e)] + + for k, e in alphabetical.items(): + alphabetical[k] = sorted(e, key=lambda x: x[0]) + app_index[corpus]["words"] = alphabetical + + functors = [(k, e) for (k, e) in res_fns.items()] + functors = sorted(functors, key=lambda x: x[0]) + app_index[corpus]["functors"] = functors + if __name__ == "__main__": print("Starting app.py main()") @@ -407,8 +463,10 @@ if __name__ == "__main__": ) valdb = client.valdb + if config["prepare_db"]: + prepare_db() + # log.info("[*] Starting app.py with config:\n%s".format(config)) log.info("[*] Starting app.py with config:\n{}".format(config)) - sys.exit() app.run(host=str(config["host"]), port=int(config["port"])) diff --git a/src/backend_flask/conf_files/dev_conf.yaml b/src/backend_flask/conf_files/dev_conf.yaml index 31dd233..3753cc7 100644 --- a/src/backend_flask/conf_files/dev_conf.yaml +++ b/src/backend_flask/conf_files/dev_conf.yaml @@ -3,4 +3,5 @@ debug: True port: 5004 host: localhost logfile: "/var/log/valency_backend.log" +prepare_db: True --- diff --git a/src/frontend_vue/config/config.json b/src/frontend_vue/config/config.json index 116f713..a304ca1 100644 --- a/src/frontend_vue/config/config.json +++ b/src/frontend_vue/config/config.json @@ -1,3 +1,3 @@ { - "api_addr": "http://193.2.76.103:5004" + "api_addr": "http://0.0.0.0:5004" } diff --git a/src/frontend_vue/config/config_dev.json b/src/frontend_vue/config/config_dev.json index d9b7448..a304ca1 100644 --- a/src/frontend_vue/config/config_dev.json +++ b/src/frontend_vue/config/config_dev.json @@ -1,3 +1,3 @@ { - "api_addr": "http://localhost:5004" + "api_addr": "http://0.0.0.0:5004" } diff --git a/src/frontend_vue/config/config_prod.json b/src/frontend_vue/config/config_prod.json index 116f713..c1a1b65 100644 --- a/src/frontend_vue/config/config_prod.json +++ b/src/frontend_vue/config/config_prod.json @@ -1,3 +1,3 @@ { - "api_addr": "http://193.2.76.103:5004" + "api_addr": "0.0.0.0:5004" } diff --git a/src/frontend_vue/package-lock.json b/src/frontend_vue/package-lock.json index 316844b..4416c52 100644 --- a/src/frontend_vue/package-lock.json +++ b/src/frontend_vue/package-lock.json @@ -116,7 +116,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, "requires": { "sprintf-js": "~1.0.2" @@ -2979,9 +2979,9 @@ } }, "esprima": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", - "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dev": true }, "esrecurse": { @@ -5128,13 +5128,13 @@ "dev": true }, "js-yaml": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz", - "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.0.tgz", + "integrity": "sha512-pZZoSxcCYco+DIKBTimr67J6Hy+EYGZDY/HCWC+iAEA9h1ByhMXAIVUXMcMFpOCxQ/xjXmPI2MkDL5HRm5eFrQ==", "dev": true, "requires": { "argparse": "^1.0.7", - "esprima": "^2.6.0" + "esprima": "^4.0.0" } }, "jsesc": { @@ -5255,9 +5255,9 @@ } }, "lodash": { - "version": "4.17.10", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz", - "integrity": "sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg==" + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, "lodash.camelcase": { "version": "4.3.0", @@ -9679,6 +9679,24 @@ "mkdirp": "~0.5.1", "sax": "~1.2.1", "whet.extend": "~0.9.9" + }, + "dependencies": { + "esprima": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", + "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", + "dev": true + }, + "js-yaml": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz", + "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^2.6.0" + } + } } }, "tapable": { diff --git a/src/frontend_vue/src/components/Home.vue b/src/frontend_vue/src/components/Home.vue index d90f426..6d5c9cd 100644 --- a/src/frontend_vue/src/components/Home.vue +++ b/src/frontend_vue/src/components/Home.vue @@ -10,7 +10,7 @@
- +
@@ -35,11 +35,6 @@ export default { LFunctors: LFunctors, MainDispl: MainDispl, }, - methods: { - navSS: function () { - return this.$root.storeGet("navSS") === "words" - } - } } diff --git a/src/frontend_vue/src/components/LWords.vue b/src/frontend_vue/src/components/LWords.vue index ac28af4..683782b 100644 --- a/src/frontend_vue/src/components/LWords.vue +++ b/src/frontend_vue/src/components/LWords.vue @@ -30,7 +30,7 @@ export default { this.$http.get(this.$root.storeGet("api_addr") + "/api/words") .then(function(response) { component.$root.store.api_error = null - component.$root.store.has_se = response.data["has_se"] + // component.$root.store.has_se = response.data["has_se"] component.letters = response.data["sorted_words"] }) .catch(function(error) { diff --git a/src/frontend_vue/src/components/MainDispl.vue b/src/frontend_vue/src/components/MainDispl.vue index aa8910e..e53461a 100644 --- a/src/frontend_vue/src/components/MainDispl.vue +++ b/src/frontend_vue/src/components/MainDispl.vue @@ -32,7 +32,7 @@       - +
@@ -206,7 +206,7 @@ export default { reload: function () { this.state = "loading" this.sentences = {} - if (this.$root.store.navSS === "functors") this.getFFrames(this.hw) + if (this.$root.store.selIndex === "functors") this.getFFrames(this.hw) else { this.getFrames(this.hw) if (this.$root.store.radio === "three") { diff --git a/src/frontend_vue/src/components/Nav.vue b/src/frontend_vue/src/components/Nav.vue index 018000a..f3e0c1c 100644 --- a/src/frontend_vue/src/components/Nav.vue +++ b/src/frontend_vue/src/components/Nav.vue @@ -6,17 +6,30 @@ - - + + {{ option }} + + + + + + + + v-on:click="updateIndex(option.val)" + > {{ option.key }} - + Uporabnik: {{ this.$root.store.username }} @@ -46,19 +59,27 @@ export default { name: "Nav", props: ["appState"], data() {return { - search_options: [ + optIndexes: [ {key: "besede", val: "words"}, {key: "udeleženske vloge", val: "functors"}, ], + optCorpora: ["kres", "ssj"], }}, methods: { - setNavSS(val) { + updateAll() { this.$root.store.radio = "one" - this.$root.store.navSS = val this.$router.push({ name: "Home" }) }, + updateCorpus(val) { + this.$root.store.selCorpus = val + this.updateAll() + }, + updateIndex(val) { + this.$root.store.selIndex = val + this.updateAll() + }, loggedIn() { return (this.$root.store.token !== null) }, diff --git a/src/frontend_vue/src/main.js b/src/frontend_vue/src/main.js index 230c6f6..6fda067 100644 --- a/src/frontend_vue/src/main.js +++ b/src/frontend_vue/src/main.js @@ -38,7 +38,8 @@ const store = { // api_addr: "http://193.2.76.103:5004", // production token: null, username: null, - navSS: "words", + selIndex: "words", + selCorpus: "kres", radio: "one", has_se: [], // used for appending (se) to certain verbs }