working on api + frontend; todo: switch corpus

2019-03-24 02:08:22 +01:00 · 2019-03-24 02:08:22 +01:00 · bc4e8e2cf1
commit bc4e8e2cf1
parent 8c6d6ab8ab
12 changed files with 149 additions and 55 deletions
--- a/README.md
+++ b/README.md
@ -12,7 +12,7 @@ $ git submodule update


 ### Database (2 containers)
-Set db admin, user, pass, etc in Makefile.  
+Set db admin, user, pass, etc in 'Makefile'.  
 Spin up the database service and create users:  
 ```bash
 # $ make database-clean  # opt
@ -38,11 +38,15 @@ $ make python-env-install
 $ make fill-database
 ```

-If all goes well, we should be able to inspect the database on `0.0.0.0:8087`.  
+If all goes well, we should be able to inspect the database, filled with corpora, on `0.0.0.0:8087`.  


 ### Flask backend (1 container)
-Input: see Database
+Relies heavily on the database. Set that up first. 
+```bash
+# $ make backend=dev  # development
+$ make backend-prod
+```

 API endpoints:

@ -53,17 +57,13 @@ API endpoints:


 ### Vue frontend (1 container)
+Relies on Flask backend.  
+Before running `make`, you might need to set the correct api address.  
+Check `./src/frontend_vue/config/config_prod.json`.  
+bash
+```
+# $ make frontend-dev  # development
+$ make frontend-prod
+```

-* ngnix server
-
-
-## Deployment
-
-Preflight:
-* get up DB
-* prepare DB
-
-Flight:
-* start backend
-* start frontend
-
+App available on: `http://0.0.0.0:8080`.  
--- a/src/backend_flask/app.py
+++ b/src/backend_flask/app.py
@ -26,9 +26,13 @@ from pathlib import Path
 from pymongo import MongoClient
 import argparse

+CORPORA = ["ssj", "kres"]
+
 log = logging.getLogger(__name__)
 app = Flask(__name__)

+app_index = {c: {} for c in CORPORA}
+
 # when running vuejs via webpack
 # CORS(app)
 CORS(app, resources={r"/api/*": {
@ -54,20 +58,18 @@ def home(pathname):
    return redirect(url_for("index"), code=302)


+# @app.route("/api/words/<corpus>")
+# def api_words(corpus):
@app.route("/api/words")
 def api_words():
    return json.dumps({
-        "sorted_words": vallex.sorted_words,
-        "has_se": vallex.has_se
+        "sorted_words": app_index["ssj"]["words"], # todo - make corpus as arg
    })

-
@app.route("/api/functors")
 def api_functors():
-    res = []
-    for key in sorted(vallex.functors_index.keys()):
-        res.append((key, len(vallex.functors_index[key])))
-    return json.dumps(res)
+    # return array ([functor, len])
+    return json.dumps(app_index["ssj"]["functors"])


@app.route("/api/register", methods=["POST"])
@ -376,6 +378,60 @@ def api_senses_update():
        vallex.db["v2_sense_map"].insert(data)
    return "OK"

+def prepare_db():
+    def helper_tid_to_token(tid, tokens):
+        for t in tokens:
+            if t["tid"] == tid:
+                return t
+        return None
+
+    # update entries (add headwords and fuctors for indexing)
+    for corpus in ["ssj", "kres"]:
+        for e in valdb[corpus].find({}):
+            #! hw_tids are not array ids
+            hw_tids = list(set([x["from"] for x in e["srl_links"]]))
+            hw_tokens = [helper_tid_to_token(tid, e["tokens"]) for tid in hw_tids]
+            headwords = [(t["lemma"] if t["msd"][0] == "G" else t["lemma"] + "_") for t in hw_tokens]
+            e["headwords"] = headwords
+
+            functors = list(set([x["afun"] for x in e["srl_links"]]))
+            e["headwords"] = headwords
+            e["functors"] = functors
+
+            valdb[corpus].save(e)
+
+    # create app_index (used in frontend, left side word index)
+    for corpus in CORPORA:
+        res_hws = {}
+        res_fns = {}
+        for e in valdb[corpus].find({}):
+            for hw in e["headwords"]:
+                if hw in res_hws:
+                    res_hws[hw] += 1
+                else:
+                    res_hws[hw] = 1
+            for fn in e["functors"]:
+                if fn in res_fns:
+                    res_fns[fn] += 1
+                else:
+                    res_fns[fn] = 1
+
+        alphabetical = {}
+        for k, e in res_hws.items():
+            fst = k[0].lower()
+            if fst in alphabetical:
+                alphabetical[fst].append((k, e))
+            else:
+                alphabetical[fst] = [(k, e)]
+
+        for k, e in alphabetical.items():
+            alphabetical[k] = sorted(e, key=lambda x: x[0]) 
+        app_index[corpus]["words"] = alphabetical
+
+        functors = [(k, e) for (k, e) in res_fns.items()]
+        functors = sorted(functors, key=lambda x: x[0])
+        app_index[corpus]["functors"] = functors
+

 if __name__ == "__main__":
    print("Starting app.py main()")
@ -407,8 +463,10 @@ if __name__ == "__main__":
    )
    valdb = client.valdb

+    if config["prepare_db"]:
+        prepare_db()
+
    # log.info("[*] Starting app.py with config:\n%s".format(config))
    log.info("[*] Starting app.py with config:\n{}".format(config))
-    sys.exit()

    app.run(host=str(config["host"]), port=int(config["port"]))
--- a/src/backend_flask/conf_files/dev_conf.yaml
+++ b/src/backend_flask/conf_files/dev_conf.yaml
@ -3,4 +3,5 @@ debug: True
 port: 5004
 host: localhost
 logfile: "/var/log/valency_backend.log"
+prepare_db: True
 ---
--- a/src/frontend_vue/config/config.json
+++ b/src/frontend_vue/config/config.json
@ -1,3 +1,3 @@
 {
-    "api_addr": "http://193.2.76.103:5004"
+    "api_addr": "http://0.0.0.0:5004"
 }
--- a/src/frontend_vue/config/config_dev.json
+++ b/src/frontend_vue/config/config_dev.json
@ -1,3 +1,3 @@
 {
-    "api_addr": "http://localhost:5004"
+    "api_addr": "http://0.0.0.0:5004"
 }
--- a/src/frontend_vue/config/config_prod.json
+++ b/src/frontend_vue/config/config_prod.json
@ -1,3 +1,3 @@
 {
-    "api_addr": "http://193.2.76.103:5004"
+    "api_addr": "0.0.0.0:5004"
 }
--- a/src/frontend_vue/package-lock.json
+++ b/src/frontend_vue/package-lock.json
@ -116,7 +116,7 @@
    "argparse": {
      "version": "1.0.10",
      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
-      "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
      "dev": true,
      "requires": {
        "sprintf-js": "~1.0.2"
@ -2979,9 +2979,9 @@
      }
    },
    "esprima": {
-      "version": "2.7.3",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz",
-      "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
      "dev": true
    },
    "esrecurse": {
@ -5128,13 +5128,13 @@
      "dev": true
    },
    "js-yaml": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
-      "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.0.tgz",
+      "integrity": "sha512-pZZoSxcCYco+DIKBTimr67J6Hy+EYGZDY/HCWC+iAEA9h1ByhMXAIVUXMcMFpOCxQ/xjXmPI2MkDL5HRm5eFrQ==",
      "dev": true,
      "requires": {
        "argparse": "^1.0.7",
-        "esprima": "^2.6.0"
+        "esprima": "^4.0.0"
      }
    },
    "jsesc": {
@ -5255,9 +5255,9 @@
      }
    },
    "lodash": {
-      "version": "4.17.10",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz",
-      "integrity": "sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg=="
+      "version": "4.17.11",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
+      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
    },
    "lodash.camelcase": {
      "version": "4.3.0",
@ -9679,6 +9679,24 @@
        "mkdirp": "~0.5.1",
        "sax": "~1.2.1",
        "whet.extend": "~0.9.9"
+      },
+      "dependencies": {
+        "esprima": {
+          "version": "2.7.3",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz",
+          "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=",
+          "dev": true
+        },
+        "js-yaml": {
+          "version": "3.7.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
+          "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+          "dev": true,
+          "requires": {
+            "argparse": "^1.0.7",
+            "esprima": "^2.6.0"
+          }
+        }
      }
    },
    "tapable": {
--- a/src/frontend_vue/src/components/Home.vue
+++ b/src/frontend_vue/src/components/Home.vue
@ -10,7 +10,7 @@
    <div class="my-home container-fluid">
      <div class="row">
        <div id="serach" class="col-sm-2 border-right fill">
-          <LWords v-if="navSS()"></LWords>
+          <LWords v-if="this.$root.store.selIndex === 'words'"></LWords>
          <LFunctors v-else></LFunctors>
        </div>
        <div class="col-sm-10">
@ -35,11 +35,6 @@ export default {
    LFunctors: LFunctors,
    MainDispl: MainDispl,
  },
-  methods: {
-    navSS: function () {
-      return this.$root.storeGet("navSS") === "words"
-    }
-  }
 }
 </script>

--- a/src/frontend_vue/src/components/LWords.vue
+++ b/src/frontend_vue/src/components/LWords.vue
@ -30,7 +30,7 @@ export default {
            this.$http.get(this.$root.storeGet("api_addr") + "/api/words")
                .then(function(response) {
                    component.$root.store.api_error = null
-                    component.$root.store.has_se = response.data["has_se"]
+                    // component.$root.store.has_se = response.data["has_se"]
                    component.letters = response.data["sorted_words"]
                })
                .catch(function(error) {
--- a/src/frontend_vue/src/components/MainDispl.vue
+++ b/src/frontend_vue/src/components/MainDispl.vue
@ -32,7 +32,7 @@
                <tr>
                    <label class="radio-inline"><input value="one" v-model="$root.store.radio" v-on:change="reload()" checked="" type="radio" name="optradio">posamezne povedi</label>&nbsp;&nbsp;
                    <label class="radio-inline"><input value="two" v-model="$root.store.radio" v-on:change="reload()" type="radio" name="optradio">skupne udeleženske vloge</label>&nbsp;&nbsp;
-                    <label v-if="this.$root.store.navSS === 'words'" class="radio-inline"><input value="three" v-model="$root.store.radio" v-on:change="reload()" type="radio" name="optradio">po meri</label>  
+                    <label v-if="this.$root.store.selIndex === 'words'" class="radio-inline"><input value="three" v-model="$root.store.radio" v-on:change="reload()" type="radio" name="optradio">po meri</label>  
                </tr>
            </table>
        </div>
@ -206,7 +206,7 @@ export default {
        reload: function () {
            this.state = "loading"
            this.sentences = {}
-            if (this.$root.store.navSS === "functors") this.getFFrames(this.hw)
+            if (this.$root.store.selIndex === "functors") this.getFFrames(this.hw)
            else {
                this.getFrames(this.hw)
                if (this.$root.store.radio === "three") {
--- a/src/frontend_vue/src/components/Nav.vue
+++ b/src/frontend_vue/src/components/Nav.vue
@ -6,11 +6,24 @@
    <b-collapse is-nav id="nav_collapse">

    <b-navbar-nav>
-        <b-nav-item-dropdown text="Prikaz" right>
-            <b-dropdown-item v-for="option in search_options" 
+        <b-nav-item-dropdown text="korpus" right>
+            <b-dropdown-item v-for="option in optCorpora"
+                :value="option"
+                :key="option"
+                v-on:click="updateCorpus(option)"
+            >
+                {{ option }}
+            </b-dropdown-item>
+        </b-nav-item-dropdown>
+    </b-navbar-nav>
+
+    <b-navbar-nav>
+        <b-nav-item-dropdown text="prikaz" right>
+            <b-dropdown-item v-for="option in optIndexes"
                :value="option.val"
                :key="option.val"
-                v-on:click="setNavSS(option.val)">
+                v-on:click="updateIndex(option.val)"
+            >
                {{ option.key }}
            </b-dropdown-item>
        </b-nav-item-dropdown>
@ -46,19 +59,27 @@ export default {
    name: "Nav",
    props: ["appState"],
    data() {return {
-        search_options: [
+        optIndexes: [
            {key: "besede", val: "words"},
            {key: "udeleženske vloge", val: "functors"},
        ],
+        optCorpora: ["kres", "ssj"],
    }},
    methods: {
-        setNavSS(val) {
+        updateAll() {
            this.$root.store.radio = "one"
-            this.$root.store.navSS = val
            this.$router.push({
                name: "Home"
            })
        },
+        updateCorpus(val) {
+            this.$root.store.selCorpus = val
+            this.updateAll()
+        },
+        updateIndex(val) {
+            this.$root.store.selIndex = val
+            this.updateAll()
+        },
        loggedIn() {
            return (this.$root.store.token !== null)
        },
--- a/src/frontend_vue/src/main.js
+++ b/src/frontend_vue/src/main.js
@ -38,7 +38,8 @@ const store = {
    // api_addr: "http://193.2.76.103:5004",  // production
    token: null,
    username: null,
-    navSS: "words",
+    selIndex: "words",
+    selCorpus: "kres",
    radio: "one",
    has_se: [],  // used for appending (se) to certain verbs
 }