From fd351bd8dbe1215b6deed771091ccc16c3ab191e Mon Sep 17 00:00:00 2001 From: voje Date: Fri, 22 Mar 2019 14:50:47 +0100 Subject: [PATCH] backend env dockerfile --- Makefile | 7 + dockerfiles/backend/Dockerfile | 0 dockerfiles/frontend/README.md | 1 - dockerfiles/main-compose.yml | 0 dockerfiles/placeholder | 0 dockerfiles/python-env/Dockerfile | 7 +- src/backend_flask/README.md | 11 + src/backend_flask/anchor | 0 src/backend_flask/app.py | 398 +++++++++++++++++++++ src/backend_flask/conf_files/dev_conf.yaml | 6 + 10 files changed, 428 insertions(+), 2 deletions(-) delete mode 100644 dockerfiles/backend/Dockerfile delete mode 100644 dockerfiles/frontend/README.md delete mode 100644 dockerfiles/main-compose.yml delete mode 100644 dockerfiles/placeholder create mode 100644 src/backend_flask/README.md delete mode 100644 src/backend_flask/anchor create mode 100644 src/backend_flask/app.py create mode 100644 src/backend_flask/conf_files/dev_conf.yaml diff --git a/Makefile b/Makefile index 49b758e..b64a148 100644 --- a/Makefile +++ b/Makefile @@ -70,3 +70,10 @@ frontend-dev: frontend-prod: cd src/frontend_vue/; $(MAKE) prod + +## Backend +backend-env: python-env-install + + +backend-dev: python-env-install + cd ./src/backend_flask; python3 app.py --config-file ./conf_files/dev_conf.yaml diff --git a/dockerfiles/backend/Dockerfile b/dockerfiles/backend/Dockerfile deleted file mode 100644 index e69de29..0000000 diff --git a/dockerfiles/frontend/README.md b/dockerfiles/frontend/README.md deleted file mode 100644 index 528dddf..0000000 --- a/dockerfiles/frontend/README.md +++ /dev/null @@ -1 +0,0 @@ -# Files in `../../frontend_vue/`. diff --git a/dockerfiles/main-compose.yml b/dockerfiles/main-compose.yml deleted file mode 100644 index e69de29..0000000 diff --git a/dockerfiles/placeholder b/dockerfiles/placeholder deleted file mode 100644 index e69de29..0000000 diff --git a/dockerfiles/python-env/Dockerfile b/dockerfiles/python-env/Dockerfile index 6b620bf..d688ce0 100644 --- a/dockerfiles/python-env/Dockerfile +++ b/dockerfiles/python-env/Dockerfile @@ -13,9 +13,14 @@ RUN pip3 install \ sklearn \ argparse \ pathlib \ - pymongo + pymongo \ + flask RUN apt-get install -y \ curl ENV PYTHONIOENCODING UTF-8 + +RUN pip3 install \ + yaml \ + flask_cors diff --git a/src/backend_flask/README.md b/src/backend_flask/README.md new file mode 100644 index 0000000..cf3b06f --- /dev/null +++ b/src/backend_flask/README.md @@ -0,0 +1,11 @@ +# backend +Using Flask mycroframework. + +Entrypoint: app.py + +Depends on packages form `../pkg`. + +## Environment +From git root, run `make python-env`. +Inside the container, run `backend-dev` to install our packages and run the app in debug mode. + diff --git a/src/backend_flask/anchor b/src/backend_flask/anchor deleted file mode 100644 index e69de29..0000000 diff --git a/src/backend_flask/app.py b/src/backend_flask/app.py new file mode 100644 index 0000000..0161dac --- /dev/null +++ b/src/backend_flask/app.py @@ -0,0 +1,398 @@ +# -*- coding: utf-8 -*- + +from flask import Flask, render_template, request, url_for, redirect + +""" +from valency import k_utils +from valency.ssj_struct import * +from valency.val_struct import * +from valency.reduce_functions import * +""" + +import logging +import sys +import json +import yaml +from flask_cors import CORS +import hashlib +import uuid +import datetime +import string +import random +import smtplib +from email.mime.text import MIMEText +from copy import deepcopy as DC +from pathlib import Path +import argparse + +log = logging.getLogger(__name__) +app = Flask(__name__) + +# when running vuejs via webpack +# CORS(app) +CORS(app, resources={r"/api/*": { + "origins": "*", +}}) + + +# for testing functions +@app.route("/test_dev") +def test_dev(): + ret = vallex.test_dev() + return(str(ret) or "edit val_struct.py: test_dev()") + + +@app.route("/") +def index(): + return(render_template("index.html")) + + +@app.route("/home", defaults={"pathname": ""}) +@app.route("/home/") +def home(pathname): + return redirect(url_for("index"), code=302) + + +@app.route("/api/words") +def api_words(): + return json.dumps({ + "sorted_words": vallex.sorted_words, + "has_se": vallex.has_se + }) + + +@app.route("/api/functors") +def api_functors(): + res = [] + for key in sorted(vallex.functors_index.keys()): + res.append((key, len(vallex.functors_index[key]))) + return json.dumps(res) + + +@app.route("/api/register", methods=["POST"]) +def api_register(): + USERS_COLL = "v2_users" + b = request.get_data() + data = json.loads(b.decode()) + username = data["username"] + password = data["password"] + email = data["email"] + if ( + username == "" or + password == "" or + email == "" + ): + return "ERR" + existing = list(vallex.db[USERS_COLL].find({ + "$or": [{"username": username}, {"email": email}] + })) + if len(existing) > 0: + return "ERR: Username or email already exists." + entry = { + "username": username, + "hpass": hashlib.sha256( + password.encode("utf-8")).hexdigest(), + "email": hashlib.sha256( + email.encode("utf-8")).hexdigest() + } + vallex.db[USERS_COLL].insert(entry) + return "OK" + + +@app.route("/api/login", methods=["POST"]) +def api_login(): + USERS_COLL = "v2_users" + TOKENS_COLL = "v2_user_tokens" + b = request.get_data() + data = json.loads(b.decode()) + username = data["username"] + password = data["password"] + hpass = hashlib.sha256(password.encode("utf-8")).hexdigest() + + db_user = list(vallex.db[USERS_COLL].find({ + "username": username, + "hpass": hpass + })) + if len(db_user) == 0: + return json.dumps({"token": None}) + + # update or create token + token = uuid.uuid4().hex + token_entry = { + "username": username, + "date": datetime.datetime.utcnow(), + "token": token + } + vallex.db[TOKENS_COLL].update( + {"username": token_entry["username"]}, + token_entry, + upsert=True + ) + return json.dumps({"token": token}) + + +def send_new_pass_mail(recipient, new_pass): + # dtime = str(datetime.datetime.now()) + SENDER = "valencaglagolov@gmail.com" + msg = MIMEText( + "PoĊĦiljamo vam novo geslo za " + "vstop v aplikacijo Vezljivostni vzorci slovenskih glagolov.\n" + "Geslo: {}.".format(new_pass) + ) + msg["Subject"] = "Pozabljeno geslo" + msg["From"] = SENDER + msg["To"] = recipient + + try: + server = smtplib.SMTP("smtp.gmail.com", 587) + server.ehlo() + server.starttls() + server.login( + SENDER, + "rapid limb soapy fermi" + ) + server.sendmail(SENDER, [recipient], msg.as_string()) + server.close() + log.info("Sent new password.") + except Error as e: + log.error("Sending new password failed") + log.error(e) + + +@app.route("/api/new_pass", methods=["POST"]) +def api_new_pass(): + b = request.get_data() + data = json.loads(b.decode()) + username = data["username"] + email = data["email"] + hemail = hashlib.sha256(email.encode("utf-8")).hexdigest() + db_res = list(vallex.db.v2_users.find({ + "username": username, + "email": hemail + })) + # check if user is valid + if len(db_res) == 0: + return json.dumps({"confirmation": False}) + # create a new password + new_pass = "".join([random.choice( + string.ascii_letters + string.digits) for i in range(10)]) + # update locally + hpass = hashlib.sha256(new_pass.encode("utf-8")).hexdigest() + vallex.db.v2_users.update( + { + "username": username, + "email": hemail + }, + {"$set": { + "hpass": hpass + }} + ) + # send via mail + send_new_pass_mail(email, new_pass) + return json.dumps({"confirmation": True}) + + +def prepare_frames(ret_frames): + # append sentences + for frame in ret_frames: + frame.sentences = [] + unique_sids = {".".join(x.split(".")[:-1]): x for x in frame.tids} + log.debug(str(unique_sids)) + frame.sentences = [] + frame.aggr_sent = {} + for sid, tid in unique_sids.items(): + hwl = vallex.get_token(tid)["lemma"] + tmp_idx = len(frame.sentences) + if hwl not in frame.aggr_sent: + frame.aggr_sent[hwl] = [] + frame.aggr_sent[hwl].append(tmp_idx) + frame.sentences.append( + vallex.get_tokenized_sentence(tid) + ) + # return (n-frames, rendered template) + # json frames + json_ret = {"frames": []} + for frame in ret_frames: + json_ret["frames"].append(DC(frame.to_json())) + return json.dumps(json_ret) + + +@app.route("/api/frames") +def api_get_frames(): + hw = request.args.get("hw") + if hw is None: + return json.dumps({"error": "Headword not found."}) + + rf_name = request.args.get("rf", "reduce_0") # 2nd is default + RF = reduce_functions[rf_name]["f"] + entry = vallex.entries[hw] + ret_frames = RF(entry.raw_frames, vallex) + return prepare_frames(ret_frames) + + +@app.route("/api/functor-frames") +def api_get_functor_frames(): + functor = request.args.get("functor") + if functor is None: + return json.dumps({"error": "Missing argument: functor."}) + rf_name = request.args.get("rf", "reduce_0") # 2nd is default + RF = reduce_functions[rf_name]["f"] + raw_frames = vallex.functors_index[functor] + ret_frames = RF(raw_frames, vallex) + return prepare_frames(ret_frames) + + +def token_to_username(token): + COLLNAME = "v2_user_tokens" + key = { + "token": token + } + res = list(vallex.db[COLLNAME].find(key)) + if len(res) != 1: + return None + username = res[0]["username"] + # update deletion interval + vallex.db[COLLNAME].update( + key, {"$set": {"date": datetime.datetime.utcnow()}}) + return username + + +@app.route("/api/token", methods=["POST"]) +def api_token(): + # check if token is valid + b = request.get_data() + data = json.loads(b.decode()) + token = data.get("token") + # user = data.get("user") + user = token_to_username(token) + confirm = (user is not None) + return json.dumps({ + "confirmation": confirm, + "username": user + }) + + +@app.route("/api/senses/get") +def api_senses_get(): + # returns senses and mapping for hw + hw = request.args.get("hw") + senses = list(vallex.db["v2_senses"].find({ + "hw": hw + })) + sense_map_query = list(vallex.db["v2_sense_map"].find({ + "hw": hw + })) + # aggregation by max date possible on DB side + # but no simple way of returning full entries + # aggregate hw and ssj_id by max date + sense_map_aggr = {} + for sm in sense_map_query: + key = sm["hw"] + sm["ssj_id"] + if key in sense_map_aggr: + sense_map_aggr[key] = max( + [sm, sense_map_aggr[key]], key=lambda x: x["date"]) + else: + sense_map_aggr[key] = sm + sense_map_list = [x[1] for x in sense_map_aggr.items()] + sense_map = {} + for el in sense_map_list: + sense_map[el["ssj_id"]] = el + for k, e in sense_map.items(): + del(e["_id"]) + del(e["date"]) + for e in senses: + del(e["_id"]) + if "date" in e: + del(e["date"]) + + # sort senses: user defined first, sskj second + # sskj senses sorted by sskj sense_id + user_senses = [s for s in senses if s["author"] != "SSKJ"] + sskj_senses = [s for s in senses if s["author"] == "SSKJ"] + + def sorting_helper(sense): + arr = sense["sense_id"].split("-") + return "{:03d}-{:03d}-{:03d}".format( + int(arr[1]), int(arr[2]), int(arr[3])) + + sskj_senses = sorted(sskj_senses, key=sorting_helper) + senses = user_senses + sskj_senses + + return json.dumps({ + "senses": senses, + "sense_map": sense_map, + }) + + +@app.route("/api/senses/update", methods=["POST"]) +def api_senses_update(): + b = request.get_data() + data = json.loads(b.decode()) + token = data.get("token") + hw = data.get("hw") + sense_map = data.get("sense_map") + new_senses = data.get("new_senses") + + username = token_to_username(token) + if username is None: + log.debug("Not a user.") + return "Not a user." + + # store new senses, + # create new sense_ids + id_map = {} + for ns in new_senses: + tmp_dt = datetime.datetime.utcnow() + new_sense_id = "{}-{}".format( + username, + hashlib.sha256("{}{}{}".format( + username, + ns["desc"], + str(tmp_dt) + ).encode("utf-8")).hexdigest()[:10] + ) + frontend_sense_id = ns["sense_id"] + ns["sense_id"] = new_sense_id + ns["date"] = tmp_dt + id_map[frontend_sense_id] = new_sense_id + + # insert into db + vallex.db["v2_senses"].insert(ns) + + # replace tmp_id with mongo's _id + for ssj_id, el in sense_map.items(): + sense_id = el["sense_id"] + if sense_id in id_map.keys(): + sense_id = id_map[sense_id] + data = { + "user": username, + "hw": hw, + "ssj_id": ssj_id, + "sense_id": sense_id, + "date": datetime.datetime.utcnow() + } + # vallex.db["v2_sense_map"].update(key, data, upsert=True) + vallex.db["v2_sense_map"].insert(data) + return "OK" + + +if __name__ == "__main__": + aparser = argparse.ArgumentParser(description="Arguments for app.py") + aparser.add_argument("--config-file", type=str, help="check ./conf_files/") + args = aparser.parse_args() + + config = None + with Path(args.config_file).open("r") as fp: + config = list(yaml.safe_load_all(fp))[0] + + app.debug = bool(config["debug"]) + logfile = config["logfile"] + if app.debug: + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + else: + logging.basicConfig(filename=logfile, level=logging.INFO) + + # log.info("[*] Starting app.py with config:\n%s".format(config)) + print("[*] Starting app.py with config:\n{}".format(config)) + + app.run(host=str(config["host"]), port=int(config["port"])) diff --git a/src/backend_flask/conf_files/dev_conf.yaml b/src/backend_flask/conf_files/dev_conf.yaml new file mode 100644 index 0000000..31dd233 --- /dev/null +++ b/src/backend_flask/conf_files/dev_conf.yaml @@ -0,0 +1,6 @@ +--- +debug: True +port: 5004 +host: localhost +logfile: "/var/log/valency_backend.log" +---