Main brez modelov

.gitignore

@@ -3,6 +3,9 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+#sublime config
+sftp-config.json
+
 # C extensions
 *.so
 
@@ -64,4 +67,14 @@ target/
 .ipynb_checkpoints
 
 # JetBrains
-.idea
+.idea
+
+# Related to Development folder
+mnt/
+
+gen/
+swagger_server/requets_db/dbs/*
+
+.env
+
+classla/classla_resources

ATEapi/.dockerignore

@@ -0,0 +1,26 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+README.md

ATEapi/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+.DS_Store
+.vscode/
+model/term_extractor/pytorch_model.bin

ATEapi/Dockerfile

@@ -0,0 +1,31 @@
+# For more information, please refer to https://aka.ms/vscode-docker-python
+FROM python:3.8
+
+EXPOSE 5000
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED=1
+
+# RUN apk add --no-cache gcc musl-dev
+RUN pip install gunicorn
+
+# Install pip requirements
+COPY requirements.txt .
+RUN python -m pip install -r requirements.txt
+
+WORKDIR /app
+COPY . /app
+
+
+ADD https://kt-cloud.ijs.si/index.php/s/T4qtSKxbxgqr6c5/download/pytorch_model.bin ./model/term_extractor/
+
+# Creates a non-root user with an explicit UID and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
+USER appuser
+
+# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
+CMD ["gunicorn", "-t 0", "--bind", "0.0.0.0:5000", "main:app"]

ATEapi/README.md

@@ -0,0 +1,57 @@
+# Automatic Term Extraction API
+
+## 1. Description
+In this repo, we wrote an API to inference SloBERTa term extractor, whose model has been trained with [RSDO5 corpus](https://www.clarin.si/repository/xmlui/handle/11356/1400). Feel free to check out this [repo](https://github.com/honghanhh/ate-2022) for better understanding about the methodology.
+
+---
+
+## 2. Requirements
+
+Please install all the necessary libraries noted in [requirements.txt](./requirements.txt) using this command:
+
+```
+pip install -r requirements.txt
+```
+
+## 3. Implementation
+
+Download the model from [pytorch_model.bin](https://kt-cloud.ijs.si/index.php/s/T4qtSKxbxgqr6c5) and save it into `./model/term_extractor/`.
+
+Run the following command on the terminal:
+
+```python
+main.py
+```
+
+A link will be show on the terminal so that you can access to the API and test requests.
+```python
+ * Serving Flask app 'main'
+ * Debug mode: on
+WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on http://127.0.0.1:5000
+Press CTRL+C to quit
+ * Restarting with stat
+ * Debugger is active!
+ * Debugger PIN: 185-520-689
+```
+
+We suggest you use POSTMAN or Hoppscotch to test the API with an uploaded file as the output whose format is CONLL-like (plese check out [temp_1.conllu](temp_1.conllu)). See the demonstration as the image below.
+
+![demo](./imgs/demo.png)
+
+## 4. Docker version
+Run the following command:
+```python
+docker build -t ate .  
+docker run -d -p 5000:5000 ate
+```
+
+## 5. References
+The term extraction tool is an updated version of Tran et al. (2022), using the SloBERTa model.
+
+Hanh Thi Hong Tran, Matej Martinc, Andraz Repar, Antoine Doucet and Senja Pollak: A Transformer-based Sequence-labeling Approach to the Slovenian Cross-domain Automatic Term Extraction. Proc. of Jezikovne tehnologije in digitalna humanistika, 2022.
+
+## 6. Contributors:
+- 🐮 [TRAN Thi Hong Hanh](https://github.com/honghanhh) 🐮
+- Matej Martinc
+- Senja Pollak

ATEapi/canonical_utils.py

@@ -0,0 +1,108 @@
+import os
+import classla
+# classla.download('sl', logging_level='WARNING')
+
+from lemmagen3 import Lemmatizer
+
+def lem_adj(gender, wrd):
+    lem = Lemmatizer()
+    if gender == 'm':
+        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-male.bin'))
+    elif gender == 'f':
+        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-female.bin'))
+    elif gender == 'n':
+        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-neutral.bin'))
+
+    form = lem.lemmatize(wrd)
+    return form
+
+
+def process_nlp_pipeline(lang, text):
+    nlp = classla.Pipeline(lang=lang, processors='tokenize,pos,lemma', tokenize_pretokenized=True, logging_level='WARNING')
+    doc = nlp(text)
+    return doc
+
+
+def get_adj_msd(head, word):
+    feats = head.feats
+    feats_dict = {}
+    feats = feats.strip().split('|')
+    for f in feats:
+        f = f.strip().split('=')
+        feats_dict[f[0]] = f[1]
+    gender = feats_dict['Gender']
+    #print(gender)
+    #gender = gender.strip().split('=')[1]
+    if gender == 'Masc' and len(word.xpos) == 6:
+        msd = word.xpos[:-1]+'ny'
+    elif gender == 'Masc' and len(word.xpos) == 7:
+        msd = word.xpos[:-1]+'y'
+    elif gender == 'Fem':
+        msd = word.xpos[:-1]+'n'
+    elif gender == 'Neut':
+        msd = word.xpos[:-1]+'n'
+    else:
+        msd = None
+    return msd
+
+
+def subfinder(mylist, pattern):
+    matches = []
+    for i in range(len(mylist)):
+        if mylist[i].text.lower() == pattern[0] and [t.text.lower() for t in mylist[i:i+len(pattern)]] == pattern:
+            matches.append(mylist[i:i+len(pattern)])
+    return matches
+
+
+def find_canon(term):
+    head = None
+    pre = []
+    post = []
+    for word in term.words:
+        if word.upos == 'NOUN' or word.upos == 'PROPN':
+            head = word
+            break
+    if head is None:
+        if len(term.words) == 1:
+            head2 = term.words[0]
+            lem = Lemmatizer()
+            lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
+            head_form = lem.lemmatize(head2.text.lower())
+            return head_form
+        else:
+            return ' '.join([w.text for w in term.words])  # just return the input because we do not cover such case
+    else:
+        for word in term.words:
+            if word.id < head.id:
+                pre.append(word)
+            elif word.id > head.id:
+                post.append(word)
+
+    canon = []
+    for el in pre:
+        msd = get_adj_msd(head, el)
+        if msd is None:
+            canon.append(el.lemma.lower())
+        else:
+            if msd[0] == 'A' and msd[3] == 'm':
+                form = lem_adj('m', el.text.lower())
+                canon.append(form)
+            elif msd[0] == 'A' and msd[3] == 'f':
+                form = lem_adj('f', el.text.lower())
+                canon.append(form)
+            elif msd[0] == 'A' and msd[3] == 'n':
+                form = lem_adj('n', el.text.lower())
+                canon.append(form)
+
+    lem = Lemmatizer()
+    lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
+    head_form = lem.lemmatize(head.text.lower())
+    canon.append(head_form)
+    for el in post:
+        canon.append(el.text)
+    return ' '.join(canon)
+
+def process(forms):
+    text = '\n'.join(forms)
+    doc = process_nlp_pipeline('sl', text)
+    return [find_canon(sent) for sent in doc.sentences]

ATEapi/docker-compose.debug.yml

@@ -0,0 +1,14 @@
+version: '3.4'
+
+services:
+  atedocker:
+    image: atedocker
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    command: ["sh", "-c", "pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m flask run --no-debugger --no-reload --host 0.0.0.0 --port 5000"]
+    ports:
+      - 5000:5000
+      - 5678:5678
+    environment:
+      - FLASK_APP=main.py

ATEapi/docker-compose.yml

@@ -0,0 +1,10 @@
+version: '3.4'
+
+services:
+  atedocker:
+    image: atedocker
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    ports:
+      - 5001:5000

ATEapi/main.py

@@ -0,0 +1,67 @@
+# Dependencies
+import torch
+import torch.nn.functional as F
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+from utils import *
+from canonical_utils import *
+from flask import Flask, request, jsonify
+from transformers import AutoTokenizer, AutoModelForTokenClassification 
+
+# Your API definition
+app = Flask(__name__)
+
+label_list=["n", "B-T", "T"]
+tokenizer = AutoTokenizer.from_pretrained('./model/term_extractor/')
+model = AutoModelForTokenClassification.from_pretrained('./model/term_extractor/', num_labels=len(label_list)).to(device)
+
+@app.route('/predict',methods=['POST'])
+def predict():
+    frame = read_conll(request.files['file'])
+    sequences = [' '.join(x) for x in frame.word]
+    lemma, pos, msd = frame.lemma, frame.pos, frame.msd
+    preds = []
+    probs = []
+    for seq in sequences:
+        tokens = tokenizer(seq, padding=True, truncation=True, return_tensors="pt").to(device)
+        output = model(**tokens).logits.argmax(-1)
+        prob = F.softmax(model(**tokens).logits, dim=2)
+        probs.append(prob[0].tolist())
+        preds.append([label_list[key] for key in output[0].tolist()])
+
+    texts, final_preds, final_probs = [], [], []
+    for seq, pred, prob in list(zip(sequences, preds, probs)):
+        t, p, p1 = remap(tokenizer, seq, pred, prob)
+        texts.append(t)
+        final_preds.append(p)
+        final_probs.append(p1)
+    predicted_terms, prob_terms, lemma_terms, pos_terms, msd_terms = extract_terms_full(final_preds, final_probs, texts, lemma, pos, msd)
+    df = pd.DataFrame({'terms':predicted_terms,
+                    'raw_prob':prob_terms,
+                    'lemma':lemma_terms,
+                    'pos':pos_terms,
+                    'msd':msd_terms})
+    df = df.drop_duplicates(subset=['lemma','pos'], keep='first')
+    # print(df.head(5))
+    df['prob'] = pd.Series(dtype='float')
+    for i in range(len(df)):
+        temp = [float(x) for x in df['raw_prob'].iloc[i].split(' ')]
+        df['prob'].iloc[i] = round(sum(temp)/len(temp),4)
+
+    df = df.sort_values(by=['lemma','prob'], ascending=True)
+    df = df.drop_duplicates(subset=['lemma'], keep='last')
+    df['canonical'] = process(df['terms'])
+    df = df[['terms', 'canonical', 'lemma','pos','msd','prob']].rename(columns={'prob':'ranking'})
+    # sort by ranking 
+    # print(df.head(5))
+    df = df[df['pos'] != 'PUNCT']
+    df = df.query("terms.str.len() > 2")
+    df = df.sort_values('ranking', ascending=False).drop_duplicates(subset=['terms','lemma'], keep = 'first').sort_index()
+    print(df.head(5))
+    return df.to_json(orient='records')
+    # return jsonify(df.to_dict(orient='records'))
+
+
+if __name__ == '__main__':
+
+    app.run(debug=True)

ATEapi/model/term_extractor/config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "EMBEDDIA/sloberta",
+  "architectures": [
+    "CamembertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "camembert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.20.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 32005
+}

ATEapi/model/term_extractor/special_tokens_map.json

@@ -0,0 +1,19 @@
+{
+  "additional_special_tokens": [
+    "<s>NOTUSED",
+    "</s>NOTUSED"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

ATEapi/model/term_extractor/tokenizer_config.json

@@ -0,0 +1,26 @@
+{
+  "additional_special_tokens": [
+    "<s>NOTUSED",
+    "</s>NOTUSED"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "EMBEDDIA/sloberta",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "tokenizer_class": "CamembertTokenizer",
+  "unk_token": "<unk>"
+}

ATEapi/requirements.txt

@@ -0,0 +1,7 @@
+classla==1.1.0
+Flask==2.2.2
+lemmagen3==3.3.2
+numpy==1.21.6
+pandas==1.3.5
+torch
+transformers==4.20.1

ATEapi/temp_1.conllu

@@ -0,0 +1,789 @@
+# newpar id = 1
+# sent_id = 1.1
+# text = UNIVERZA V LJUBLJANI FAKULTETA ZA UPRAVO Magistrsko delo RAZVOJ VARSTVA PRAVIC ZAVEZANCEV V SLOVENSKEM DAVČNEM POSTOPKU Helena Truden Ljubljana, januar 2012 UNIVERZA V LJUBLJANI FAKULTETA ZA UPRAVO MAGISTRSKO DELO RAZVOJ VARSTVA PRAVIC ZAVEZANCEV V SLOVENSKEM DAVČNEM POSTOPKU Kandidatka: Helena Truden Vpisna številka: 04037934 Študijski program: magistrski študijski program Uprava druga stopnja Mentorica: doc. dr. Polonca Kovač Ljubljana, januar 2012 IZJAVA O AVTORSTVU MAGISTRSKEGA DELA Podpisana Helena Truden, študentka magistrskega študijskega programa Uprava druga stopnja, z vpisno številko 04037934, sem avtorica magistrskega dela z naslovom: Razvoj varstva pravic zavezancev v slovenskem davčnem postopku. S svojim podpisom zagotavljam, da: - je predloženo delo izključno rezultat mojega lastnega raziskovalnega dela, - sem poskrbela, da so dela in mnenja drugih avtorjev in avtoric, ki jih uporabljam v predloženem delu, navedena oz. citirana v skladu s fakultetnimi navodili, - sem poskrbela, da so vsa dela in mnenja drugih avtorjev oz. avtoric navedena v seznamu virov, ki je sestavni element predloženega dela in je zapisan v skladu s fakultetnimi navodili, - sem pridobila vsa dovoljenja za uporabo avtorskih del, ki so v celoti prenesena v predloženo delo in sem to tudi jasno zapisala v predloženem delu, - se zavedam, da je plagiatorstvo – predstavljanje tujih del, bodisi v obliki citata bodisi v obliki skoraj dobesednega parafraziranja bodisi v grafični obliki, s katerim so tuje misli oz. ideje predstavljene kot moje lastne – kaznivo po zakonu (Zakon o avtorski in sorodnih pravicah, Ur. list RS, št. 21/95), kršitev pa se sankcionira tudi z ukrepi po pravilih Univerze v Ljubljani in Fakultete za upravo, - se zavedam posledic, ki jih dokazano plagiatorstvo lahko predstavlja za predloženo delo in za moj status na Fakulteti za upravo, - je elektronska oblika identična s tiskano obliko magistrskega dela ter soglašam z objavo dela v zbirki »Dela FU«. Magistrsko delo je lektorirala: Martina Kočevar, učiteljica slovenskega jezika. Ljubljana, 16. 12. 2011 Podpis avtorice: POVZETEK Predmet raziskovanja magistrskega dela je razvoj varstva pravic zavezancev v slovenskem davčnem postopku.
+1	UNIVERZA	univerza	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+2	V	v	ADP	Sl	Case=Loc	_	_	_	_
+3	LJUBLJANI	Ljubljana	PROPN	Npfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+4	FAKULTETA	fakulteta	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+5	ZA	za	ADP	Sa	Case=Acc	_	_	_	_
+6	UPRAVO	uprava	NOUN	Ncfsa	Case=Acc|Gender=Fem|Number=Sing	_	_	_	_
+7	Magistrsko	magistrski	ADJ	Agpnsn	Case=Nom|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+8	delo	delo	NOUN	Ncnsn	Case=Nom|Gender=Neut|Number=Sing	_	_	_	_
+9	RAZVOJ	razvoj	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+10	VARSTVA	varstvo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+11	PRAVIC	pravica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+12	ZAVEZANCEV	zavezanec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+13	V	v	ADP	Sl	Case=Loc	_	_	_	_
+14	SLOVENSKEM	slovenski	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+15	DAVČNEM	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+16	POSTOPKU	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+17	Helena	Helena	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+18	Truden	Truden	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+19	Ljubljana	Ljubljana	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+20	,	,	PUNCT	Z	_	_	_	_	_
+21	januar	januar	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+22	2012	2012	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+23	UNIVERZA	univerza	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+24	V	v	ADP	Sl	Case=Loc	_	_	_	_
+25	LJUBLJANI	Ljubljana	PROPN	Npfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+26	FAKULTETA	fakulteta	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+27	ZA	za	ADP	Sa	Case=Acc	_	_	_	_
+28	UPRAVO	uprava	NOUN	Ncfsa	Case=Acc|Gender=Fem|Number=Sing	_	_	_	_
+29	MAGISTRSKO	magistrski	ADJ	Agpnsn	Case=Nom|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+30	DELO	delo	NOUN	Ncnsn	Case=Nom|Gender=Neut|Number=Sing	_	_	_	_
+31	RAZVOJ	razvoj	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+32	VARSTVA	varstvo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+33	PRAVIC	pravica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+34	ZAVEZANCEV	zavezanec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+35	V	v	ADP	Sl	Case=Loc	_	_	_	_
+36	SLOVENSKEM	slovenski	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+37	DAVČNEM	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+38	POSTOPKU	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+39	Kandidatka	kandidatka	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+40	:	:	PUNCT	Z	_	_	_	_	_
+41	Helena	Helena	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+42	Truden	Truden	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+43	Vpisna	vpisen	ADJ	Agpfsn	Case=Nom|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+44	številka	številka	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+45	:	:	PUNCT	Z	_	_	_	_	_
+46	04037934	04037934	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+47	Študijski	študijski	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+48	program	program	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+49	:	:	PUNCT	Z	_	_	_	_	_
+50	magistrski	magistrski	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+51	študijski	študijski	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+52	program	program	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+53	Uprava	uprava	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+54	druga	drug	ADJ	Mlpfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+55	stopnja	stopnja	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+56	Mentorica	mentorica	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+57	:	:	PUNCT	Z	_	_	_	_	_
+58	doc.	doc.	X	Y	Abbr=Yes	_	_	_	_
+59	dr.	dr.	X	Y	Abbr=Yes	_	_	_	_
+60	Polonca	Polonca	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+61	Kovač	Kovač	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+62	Ljubljana	Ljubljana	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+63	,	,	PUNCT	Z	_	_	_	_	_
+64	januar	januar	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+65	2012	2012	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+66	IZJAVA	izjava	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+67	O	o	ADP	Sl	Case=Loc	_	_	_	_
+68	AVTORSTVU	avtorstvo	NOUN	Ncnsl	Case=Loc|Gender=Neut|Number=Sing	_	_	_	_
+69	MAGISTRSKEGA	magistrski	ADJ	Agpnsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+70	DELA	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+71	Podpisana	podpisan	ADJ	Appfsn	Case=Nom|Degree=Pos|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+72	Helena	Helena	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+73	Truden	Truden	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+74	,	,	PUNCT	Z	_	_	_	_	_
+75	študentka	študentka	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+76	magistrskega	magistrski	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+77	študijskega	študijski	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+78	programa	program	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+79	Uprava	uprava	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+80	druga	drug	ADJ	Mlpfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+81	stopnja	stopnja	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+82	,	,	PUNCT	Z	_	_	_	_	_
+83	z	z	ADP	Si	Case=Ins	_	_	_	_
+84	vpisno	vpisen	ADJ	Agpfsi	Case=Ins|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+85	številko	številka	NOUN	Ncfsi	Case=Ins|Gender=Fem|Number=Sing	_	_	_	_
+86	04037934	04037934	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	SpaceAfter=No
+87	,	,	PUNCT	Z	_	_	_	_	_
+88	sem	biti	AUX	Va-r1s-n	Mood=Ind|Number=Sing|Person=1|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+89	avtorica	avtorica	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+90	magistrskega	magistrski	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+91	dela	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+92	z	z	ADP	Si	Case=Ins	_	_	_	_
+93	naslovom	naslov	NOUN	Ncmsi	Case=Ins|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+94	:	:	PUNCT	Z	_	_	_	_	_
+95	Razvoj	razvoj	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+96	varstva	varstvo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+97	pravic	pravica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+98	zavezancev	zavezanec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+99	v	v	ADP	Sl	Case=Loc	_	_	_	_
+100	slovenskem	slovenski	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+101	davčnem	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+102	postopku	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+103	.	.	PUNCT	Z	_	_	_	_	_
+104	S	z	ADP	Si	Case=Ins	_	_	_	_
+105	svojim	svoj	DET	Px-msi	Case=Ins|Gender=Masc|Number=Sing|Poss=Yes|PronType=Prs|Reflex=Yes	_	_	_	_
+106	podpisom	podpis	NOUN	Ncmsi	Case=Ins|Gender=Masc|Number=Sing	_	_	_	_
+107	zagotavljam	zagotavljati	VERB	Vmpr1s	Aspect=Imp|Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin	_	_	_	SpaceAfter=No
+108	,	,	PUNCT	Z	_	_	_	_	_
+109	da	da	SCONJ	Cs	_	_	_	_	SpaceAfter=No
+110	:	:	PUNCT	Z	_	_	_	_	_
+111	-	-	PUNCT	Z	_	_	_	_	_
+112	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+113	predloženo	predložen	ADJ	Appnsn	Case=Nom|Degree=Pos|Gender=Neut|Number=Sing|VerbForm=Part	_	_	_	_
+114	delo	delo	NOUN	Ncnsn	Case=Nom|Gender=Neut|Number=Sing	_	_	_	_
+115	izključno	izključno	ADV	Rgp	Degree=Pos	_	_	_	_
+116	rezultat	rezultat	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+117	mojega	moj	DET	Ps1nsgs	Case=Gen|Gender=Neut|Number=Sing|Number[psor]=Sing|Person=1|Poss=Yes|PronType=Prs	_	_	_	_
+118	lastnega	lasten	ADJ	Agpnsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+119	raziskovalnega	raziskovalen	ADJ	Agpnsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+120	dela	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	SpaceAfter=No
+121	,	,	PUNCT	Z	_	_	_	_	_
+122	-	-	PUNCT	Z	_	_	_	_	_
+123	sem	biti	AUX	Va-r1s-n	Mood=Ind|Number=Sing|Person=1|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+124	poskrbela	poskrbeti	VERB	Vmep-sf	Aspect=Perf|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	SpaceAfter=No
+125	,	,	PUNCT	Z	_	_	_	_	_
+126	da	da	SCONJ	Cs	_	_	_	_	_
+127	so	biti	AUX	Va-r3p-n	Mood=Ind|Number=Plur|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+128	dela	delo	NOUN	Ncnpn	Case=Nom|Gender=Neut|Number=Plur	_	_	_	_
+129	in	in	CCONJ	Cc	_	_	_	_	_
+130	mnenja	mnenje	NOUN	Ncnpn	Case=Nom|Gender=Neut|Number=Plur	_	_	_	_
+131	drugih	drug	ADJ	Mlpmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+132	avtorjev	avtor	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+133	in	in	CCONJ	Cc	_	_	_	_	_
+134	avtoric	avtorica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	SpaceAfter=No
+135	,	,	PUNCT	Z	_	_	_	_	_
+136	ki	ki	SCONJ	Cs	_	_	_	_	_
+137	jih	on	PRON	Pp3mpa--y	Case=Acc|Gender=Masc|Number=Plur|Person=3|PronType=Prs|Variant=Short	_	_	_	_
+138	uporabljam	uporabljati	VERB	Vmpr1s	Aspect=Imp|Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin	_	_	_	_
+139	v	v	ADP	Sl	Case=Loc	_	_	_	_
+140	predloženem	predložen	ADJ	Appmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	_
+141	delu	del	NOUN	Ncmsl	Case=Loc|Gender=Neut|Number=Sing	_	_	_	SpaceAfter=No
+142	,	,	PUNCT	Z	_	_	_	_	_
+143	navedena	naveden	ADJ	Appfsn	Case=Nom|Degree=Pos|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+144	oz.	oz.	X	Y	Abbr=Yes	_	_	_	_
+145	citirana	citiran	ADJ	Appfsn	Case=Nom|Degree=Pos|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+146	v	v	ADP	Sl	Case=Loc	_	_	_	_
+147	skladu	sklad	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+148	s	z	ADP	Si	Case=Ins	_	_	_	_
+149	fakultetnimi	fakulteten	ADJ	Agpnpi	Case=Ins|Degree=Pos|Gender=Neut|Number=Plur	_	_	_	_
+150	navodili	navodilo	NOUN	Ncnpi	Case=Ins|Gender=Neut|Number=Plur	_	_	_	SpaceAfter=No
+151	,	,	PUNCT	Z	_	_	_	_	_
+152	-	-	PUNCT	Z	_	_	_	_	_
+153	sem	biti	AUX	Va-r1s-n	Mood=Ind|Number=Sing|Person=1|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+154	poskrbela	poskrbeti	VERB	Vmep-sf	Aspect=Perf|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	SpaceAfter=No
+155	,	,	PUNCT	Z	_	_	_	_	_
+156	da	da	SCONJ	Cs	_	_	_	_	_
+157	so	biti	AUX	Va-r3p-n	Mood=Ind|Number=Plur|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+158	vsa	ves	DET	Pg-npn	Case=Nom|Gender=Neut|Number=Plur|PronType=Tot	_	_	_	_
+159	dela	delo	NOUN	Ncnpn	Case=Nom|Gender=Neut|Number=Plur	_	_	_	_
+160	in	in	CCONJ	Cc	_	_	_	_	_
+161	mnenja	mnenje	NOUN	Ncnpn	Case=Nom|Gender=Neut|Number=Plur	_	_	_	_
+162	drugih	drug	ADJ	Mlpmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+163	avtorjev	avtor	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+164	oz.	oz.	X	Y	Abbr=Yes	_	_	_	_
+165	avtoric	avtorica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+166	navedena	naveden	ADJ	Appnpn	Case=Nom|Degree=Pos|Gender=Neut|Number=Plur|VerbForm=Part	_	_	_	_
+167	v	v	ADP	Sl	Case=Loc	_	_	_	_
+168	seznamu	seznam	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+169	virov	vir	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+170	,	,	PUNCT	Z	_	_	_	_	_
+171	ki	ki	SCONJ	Cs	_	_	_	_	_
+172	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+173	sestavni	sestaven	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+174	element	element	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+175	predloženega	predložen	ADJ	Appnsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing|VerbForm=Part	_	_	_	_
+176	dela	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+177	in	in	CCONJ	Cc	_	_	_	_	_
+178	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+179	zapisan	zapisan	ADJ	Appmsnn	Case=Nom|Definite=Ind|Degree=Pos|Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	_
+180	v	v	ADP	Sl	Case=Loc	_	_	_	_
+181	skladu	sklad	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+182	s	z	ADP	Si	Case=Ins	_	_	_	_
+183	fakultetnimi	fakulteten	ADJ	Agpnpi	Case=Ins|Degree=Pos|Gender=Neut|Number=Plur	_	_	_	_
+184	navodili	navodilo	NOUN	Ncnpi	Case=Ins|Gender=Neut|Number=Plur	_	_	_	SpaceAfter=No
+185	,	,	PUNCT	Z	_	_	_	_	_
+186	-	-	PUNCT	Z	_	_	_	_	_
+187	sem	biti	AUX	Va-r1s-n	Mood=Ind|Number=Sing|Person=1|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+188	pridobila	pridobiti	VERB	Vmep-sf	Aspect=Perf|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+189	vsa	ves	DET	Pg-npa	Case=Acc|Gender=Neut|Number=Plur|PronType=Tot	_	_	_	_
+190	dovoljenja	dovoljenje	NOUN	Ncnpa	Case=Acc|Gender=Neut|Number=Plur	_	_	_	_
+191	za	za	ADP	Sa	Case=Acc	_	_	_	_
+192	uporabo	uporaba	NOUN	Ncfsa	Case=Acc|Gender=Fem|Number=Sing	_	_	_	_
+193	avtorskih	avtorski	ADJ	Agpnpg	Case=Gen|Degree=Pos|Gender=Neut|Number=Plur	_	_	_	_
+194	del	delo	NOUN	Ncnpg	Case=Gen|Gender=Neut|Number=Plur	_	_	_	SpaceAfter=No
+195	,	,	PUNCT	Z	_	_	_	_	_
+196	ki	ki	SCONJ	Cs	_	_	_	_	_
+197	so	biti	AUX	Va-r3p-n	Mood=Ind|Number=Plur|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+198	v	v	ADP	Sl	Case=Loc	_	_	_	_
+199	celoti	celota	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+200	prenesena	prenesen	ADJ	Appnpn	Case=Nom|Degree=Pos|Gender=Neut|Number=Plur|VerbForm=Part	_	_	_	_
+201	v	v	ADP	Sa	Case=Acc	_	_	_	_
+202	predloženo	predložen	ADJ	Appnsa	Case=Acc|Degree=Pos|Gender=Neut|Number=Sing|VerbForm=Part	_	_	_	_
+203	delo	delo	NOUN	Ncnsa	Case=Acc|Gender=Neut|Number=Sing	_	_	_	_
+204	in	in	CCONJ	Cc	_	_	_	_	_
+205	sem	biti	AUX	Va-r1s-n	Mood=Ind|Number=Sing|Person=1|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+206	to	ta	DET	Pd-nsa	Case=Acc|Gender=Neut|Number=Sing|PronType=Dem	_	_	_	_
+207	tudi	tudi	PART	Q	_	_	_	_	_
+208	jasno	jasno	ADV	Rgp	Degree=Pos	_	_	_	_
+209	zapisala	zapisati	VERB	Vmep-sf	Aspect=Perf|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+210	v	v	ADP	Sl	Case=Loc	_	_	_	_
+211	predloženem	predložen	ADJ	Appmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	_
+212	delu	delo	NOUN	Ncnsl	Case=Loc|Gender=Neut|Number=Sing	_	_	_	SpaceAfter=No
+213	,	,	PUNCT	Z	_	_	_	_	_
+214	-	-	PUNCT	Z	_	_	_	_	_
+215	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+216	zavedam	zavedati	VERB	Vmpr1s	Aspect=Imp|Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin	_	_	_	SpaceAfter=No
+217	,	,	PUNCT	Z	_	_	_	_	_
+218	da	da	SCONJ	Cs	_	_	_	_	_
+219	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+220	plagiatorstvo	plagiatorstvo	NOUN	Ncnsn	Case=Nom|Gender=Neut|Number=Sing	_	_	_	_
+221	–	–	PUNCT	Z	_	_	_	_	_
+222	predstavljanje	predstavljanje	NOUN	Ncnsn	Case=Nom|Gender=Neut|Number=Sing	_	_	_	_
+223	tujih	tuj	ADJ	Agpnpg	Case=Gen|Degree=Pos|Gender=Neut|Number=Plur	_	_	_	_
+224	del	delo	NOUN	Ncnpg	Case=Gen|Gender=Neut|Number=Plur	_	_	_	SpaceAfter=No
+225	,	,	PUNCT	Z	_	_	_	_	_
+226	bodisi	bodisi	CCONJ	Cc	_	_	_	_	_
+227	v	v	ADP	Sl	Case=Loc	_	_	_	_
+228	obliki	oblika	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+229	citata	citat	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+230	bodisi	bodisi	CCONJ	Cc	_	_	_	_	_
+231	v	v	ADP	Sl	Case=Loc	_	_	_	_
+232	obliki	oblika	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+233	skoraj	skoraj	PART	Q	_	_	_	_	_
+234	dobesednega	dobeseden	ADJ	Agpnsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+235	parafraziranja	parafraziranje	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+236	bodisi	bodisi	CCONJ	Cc	_	_	_	_	_
+237	v	v	ADP	Sl	Case=Loc	_	_	_	_
+238	grafični	grafičen	ADJ	Agpfsl	Case=Loc|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+239	obliki	oblika	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+240	,	,	PUNCT	Z	_	_	_	_	_
+241	s	z	ADP	Si	Case=Ins	_	_	_	_
+242	katerim	kateri	DET	Pq-msi	Case=Ins|Gender=Masc|Number=Sing|PronType=Int	_	_	_	_
+243	so	biti	AUX	Va-r3p-n	Mood=Ind|Number=Plur|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+244	tuje	tuj	ADJ	Agpfpn	Case=Nom|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+245	misli	misel	NOUN	Ncfpn	Case=Nom|Gender=Fem|Number=Plur	_	_	_	_
+246	oz.	oz.	X	Y	Abbr=Yes	_	_	_	_
+247	ideje	ideja	NOUN	Ncfpn	Case=Nom|Gender=Fem|Number=Plur	_	_	_	_
+248	predstavljene	predstavljen	ADJ	Appfpn	Case=Nom|Degree=Pos|Gender=Fem|Number=Plur|VerbForm=Part	_	_	_	_
+249	kot	kot	SCONJ	Cs	_	_	_	_	_
+250	moje	moj	DET	Ps1fpns	Case=Nom|Gender=Fem|Number=Plur|Number[psor]=Sing|Person=1|Poss=Yes|PronType=Prs	_	_	_	_
+251	lastne	lasten	ADJ	Agpfpn	Case=Nom|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+252	–	–	PUNCT	Z	_	_	_	_	_
+253	kaznivo	kazniv	ADJ	Agpnsn	Case=Nom|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+254	po	po	ADP	Sl	Case=Loc	_	_	_	_
+255	zakonu	zakon	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+256	(	(	PUNCT	Z	_	_	_	_	SpaceAfter=No
+257	Zakon	zakon	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+258	o	o	ADP	Sl	Case=Loc	_	_	_	_
+259	avtorski	avtorski	ADJ	Agpfsl	Case=Loc|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+260	in	in	CCONJ	Cc	_	_	_	_	_
+261	sorodnih	soroden	ADJ	Agpfpl	Case=Loc|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+262	pravicah	pravica	NOUN	Ncfpl	Case=Loc|Gender=Fem|Number=Plur	_	_	_	SpaceAfter=No
+263	,	,	PUNCT	Z	_	_	_	_	_
+264	Ur.	ur.	X	Y	Abbr=Yes	_	_	_	_
+265	list	list	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+266	RS	RS	PROPN	Npfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+267	,	,	PUNCT	Z	_	_	_	_	_
+268	št.	št.	X	Y	Abbr=Yes	_	_	_	_
+269	21	21	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	SpaceAfter=No
+270	/	/	PUNCT	Z	_	_	_	_	SpaceAfter=No
+271	95	95	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	SpaceAfter=No
+272	)	)	PUNCT	Z	_	_	_	_	SpaceAfter=No
+273	,	,	PUNCT	Z	_	_	_	_	_
+274	kršitev	kršitev	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+275	pa	pa	CCONJ	Cc	_	_	_	_	_
+276	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+277	sankcionira	sankcionirati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+278	tudi	tudi	PART	Q	_	_	_	_	_
+279	z	z	ADP	Si	Case=Ins	_	_	_	_
+280	ukrepi	ukrep	NOUN	Ncmpi	Case=Ins|Gender=Masc|Number=Plur	_	_	_	_
+281	po	po	ADP	Sl	Case=Loc	_	_	_	_
+282	pravilih	pravilo	NOUN	Ncnpl	Case=Loc|Gender=Neut|Number=Plur	_	_	_	_
+283	Univerze	univerza	NOUN	Ncfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	_
+284	v	v	ADP	Sl	Case=Loc	_	_	_	_
+285	Ljubljani	Ljubljana	PROPN	Npfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+286	in	in	CCONJ	Cc	_	_	_	_	_
+287	Fakultete	fakulteta	NOUN	Ncfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	_
+288	za	za	ADP	Sa	Case=Acc	_	_	_	_
+289	upravo	uprava	NOUN	Ncfsa	Case=Acc|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+290	,	,	PUNCT	Z	_	_	_	_	_
+291	-	-	PUNCT	Z	_	_	_	_	_
+292	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+293	zavedam	zavedati	VERB	Vmpr1s	Aspect=Imp|Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin	_	_	_	_
+294	posledic	posledica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	SpaceAfter=No
+295	,	,	PUNCT	Z	_	_	_	_	_
+296	ki	ki	SCONJ	Cs	_	_	_	_	_
+297	jih	on	PRON	Pp3mpa--y	Case=Acc|Gender=Masc|Number=Plur|Person=3|PronType=Prs|Variant=Short	_	_	_	_
+298	dokazano	dokazan	ADJ	Appnsn	Case=Nom|Degree=Pos|Gender=Neut|Number=Sing|VerbForm=Part	_	_	_	_
+299	plagiatorstvo	plagiatorstvo	NOUN	Ncnsn	Case=Nom|Gender=Neut|Number=Sing	_	_	_	_
+300	lahko	lahko	ADV	Rgp	Degree=Pos	_	_	_	_
+301	predstavlja	predstavljati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+302	za	za	ADP	Sa	Case=Acc	_	_	_	_
+303	predloženo	predložen	ADJ	Appnsa	Case=Acc|Degree=Pos|Gender=Neut|Number=Sing|VerbForm=Part	_	_	_	_
+304	delo	delo	NOUN	Ncnsa	Case=Acc|Gender=Neut|Number=Sing	_	_	_	_
+305	in	in	CCONJ	Cc	_	_	_	_	_
+306	za	za	ADP	Sa	Case=Acc	_	_	_	_
+307	moj	moj	DET	Ps1msas	Case=Acc|Gender=Masc|Number=Sing|Number[psor]=Sing|Person=1|Poss=Yes|PronType=Prs	_	_	_	_
+308	status	status	NOUN	Ncmsan	Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing	_	_	_	_
+309	na	na	ADP	Sl	Case=Loc	_	_	_	_
+310	Fakulteti	fakulteta	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+311	za	za	ADP	Sa	Case=Acc	_	_	_	_
+312	upravo	uprava	NOUN	Ncfsa	Case=Acc|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+313	,	,	PUNCT	Z	_	_	_	_	_
+314	-	-	PUNCT	Z	_	_	_	_	_
+315	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+316	elektronska	elektronski	ADJ	Agpfsn	Case=Nom|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+317	oblika	oblika	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+318	identična	identičen	ADJ	Agpfsn	Case=Nom|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+319	s	z	ADP	Si	Case=Ins	_	_	_	_
+320	tiskano	tiskan	ADJ	Appfsi	Case=Ins|Degree=Pos|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+321	obliko	oblika	NOUN	Ncfsi	Case=Ins|Gender=Fem|Number=Sing	_	_	_	_
+322	magistrskega	magistrski	ADJ	Agpnsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+323	dela	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+324	ter	ter	CCONJ	Cc	_	_	_	_	_
+325	soglašam	soglašati	VERB	Vmpr1s	Aspect=Imp|Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin	_	_	_	_
+326	z	z	ADP	Si	Case=Ins	_	_	_	_
+327	objavo	objava	NOUN	Ncfsi	Case=Ins|Gender=Fem|Number=Sing	_	_	_	_
+328	dela	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+329	v	v	ADP	Sl	Case=Loc	_	_	_	_
+330	zbirki	zbirka	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+331	»	»	PUNCT	Z	_	_	_	_	SpaceAfter=No
+332	Dela	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+333	FU	FU	PROPN	Npmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+334	«	«	PUNCT	Z	_	_	_	_	SpaceAfter=No
+335	.	.	PUNCT	Z	_	_	_	_	_
+336	Magistrsko	magistrski	ADJ	Agpnsa	Case=Acc|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+337	delo	delo	NOUN	Ncnsa	Case=Acc|Gender=Neut|Number=Sing	_	_	_	_
+338	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+339	lektorirala	lektorirati	VERB	Vmbp-sf	Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	SpaceAfter=No
+340	:	:	PUNCT	Z	_	_	_	_	_
+341	Martina	Martina	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+342	Kočevar	Kočevar	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+343	,	,	PUNCT	Z	_	_	_	_	_
+344	učiteljica	učiteljica	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+345	slovenskega	slovenski	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+346	jezika	jezik	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+347	.	.	PUNCT	Z	_	_	_	_	_
+348	Ljubljana	Ljubljana	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+349	,	,	PUNCT	Z	_	_	_	_	_
+350	16.	16.	NUM	Mdo	NumForm=Digit|NumType=Ord	_	_	_	_
+351	12.	12.	NUM	Mdo	NumForm=Digit|NumType=Ord	_	_	_	_
+352	2011	2011	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+353	Podpis	podpis	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+354	avtorice	avtorica	NOUN	Ncfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+355	:	:	PUNCT	Z	_	_	_	_	_
+356	POVZETEK	povzetek	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+357	Predmet	predmet	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+358	raziskovanja	raziskovanje	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+359	magistrskega	magistrski	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+360	dela	delo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+361	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+362	razvoj	razvoj	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+363	varstva	varstvo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+364	pravic	pravica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+365	zavezancev	zavezanec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+366	v	v	ADP	Sl	Case=Loc	_	_	_	_
+367	slovenskem	slovenski	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+368	davčnem	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+369	postopku	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+370	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.2
+# text = Davčni postopek je posebni upravni postopek, v katerem davčni organ odmeri višino davka, ki jo mora zavezanec plačati.
+1	Davčni	davčen	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+2	postopek	postopek	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+3	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+4	posebni	poseben	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+5	upravni	upraven	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+6	postopek	postopek	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+7	,	,	PUNCT	Z	_	_	_	_	_
+8	v	v	ADP	Sl	Case=Loc	_	_	_	_
+9	katerem	kateri	DET	Pq-msl	Case=Loc|Gender=Masc|Number=Sing|PronType=Int	_	_	_	_
+10	davčni	davčen	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+11	organ	organ	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+12	odmeri	odmeriti	VERB	Vmer3s	Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+13	višino	višina	NOUN	Ncfsa	Case=Acc|Gender=Fem|Number=Sing	_	_	_	_
+14	davka	davek	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+15	,	,	PUNCT	Z	_	_	_	_	_
+16	ki	ki	SCONJ	Cs	_	_	_	_	_
+17	jo	on	PRON	Pp3fsa--y	Case=Acc|Gender=Fem|Number=Sing|Person=3|PronType=Prs|Variant=Short	_	_	_	_
+18	mora	morati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+19	zavezanec	zavezanec	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+20	plačati	plačati	VERB	Vmen	Aspect=Perf|VerbForm=Inf	_	_	_	SpaceAfter=No
+21	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.3
+# text = V davčnem postopku prihaja do konflikta javnega in zasebnih interesov.
+1	V	v	ADP	Sl	Case=Loc	_	_	_	_
+2	davčnem	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+3	postopku	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+4	prihaja	prihajati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+5	do	do	ADP	Sg	Case=Gen	_	_	_	_
+6	konflikta	konflikt	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+7	javnega	javen	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+8	in	in	CCONJ	Cc	_	_	_	_	_
+9	zasebnih	zaseben	ADJ	Agpmpg	Case=Gen|Degree=Pos|Gender=Masc|Number=Plur	_	_	_	_
+10	interesov	interes	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+11	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.4
+# text = Namen davčnega organa je namreč, da pobere čim višji znesek davka, interes zavezanca pa je, da je znesek davka, ki ga bo plačal, čim nižji.
+1	Namen	namen	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+2	davčnega	davčen	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+3	organa	organ	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+4	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+5	namreč	namreč	CCONJ	Cc	_	_	_	_	SpaceAfter=No
+6	,	,	PUNCT	Z	_	_	_	_	_
+7	da	da	SCONJ	Cs	_	_	_	_	_
+8	pobere	pobrati	VERB	Vmer3s	Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+9	čim	čim	ADV	Rgp	Degree=Pos	_	_	_	_
+10	višji	visok	ADJ	Agcmsay	Case=Acc|Definite=Def|Degree=Cmp|Gender=Masc|Number=Sing	_	_	_	_
+11	znesek	znesek	NOUN	Ncmsan	Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing	_	_	_	_
+12	davka	davek	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+13	,	,	PUNCT	Z	_	_	_	_	_
+14	interes	interes	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+15	zavezanca	zavezanec	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+16	pa	pa	CCONJ	Cc	_	_	_	_	_
+17	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	SpaceAfter=No
+18	,	,	PUNCT	Z	_	_	_	_	_
+19	da	da	SCONJ	Cs	_	_	_	_	_
+20	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+21	znesek	znesek	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+22	davka	davek	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+23	,	,	PUNCT	Z	_	_	_	_	_
+24	ki	ki	SCONJ	Cs	_	_	_	_	_
+25	ga	on	PRON	Pp3msa--y	Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs|Variant=Short	_	_	_	_
+26	bo	biti	AUX	Va-f3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Fut|VerbForm=Fin	_	_	_	_
+27	plačal	plačati	VERB	Vmep-sm	Aspect=Perf|Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	SpaceAfter=No
+28	,	,	PUNCT	Z	_	_	_	_	_
+29	čim	čim	ADV	Rgp	Degree=Pos	_	_	_	_
+30	nižji	nizek	ADJ	Agcmsny	Case=Nom|Definite=Def|Degree=Cmp|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+31	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.5
+# text = Kljub temu, da ima javni interes prednost pred interesi posameznikov, pa v davčnih postopkih pogosto prihaja do napak, zaradi katerih so kršene pravice zavezancev.
+1	Kljub	kljub	ADP	Sd	Case=Dat	_	_	_	_
+2	temu	ta	DET	Pd-nsd	Case=Dat|Gender=Neut|Number=Sing|PronType=Dem	_	_	_	SpaceAfter=No
+3	,	,	PUNCT	Z	_	_	_	_	_
+4	da	da	SCONJ	Cs	_	_	_	_	_
+5	ima	imeti	VERB	Vmpr3s-n	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+6	javni	javen	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+7	interes	interes	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+8	prednost	prednost	NOUN	Ncfsa	Case=Acc|Gender=Fem|Number=Sing	_	_	_	_
+9	pred	pred	ADP	Si	Case=Ins	_	_	_	_
+10	interesi	interes	NOUN	Ncmpi	Case=Ins|Gender=Masc|Number=Plur	_	_	_	_
+11	posameznikov	posameznik	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+12	,	,	PUNCT	Z	_	_	_	_	_
+13	pa	pa	CCONJ	Cc	_	_	_	_	_
+14	v	v	ADP	Sl	Case=Loc	_	_	_	_
+15	davčnih	davčen	ADJ	Agpmpl	Case=Loc|Degree=Pos|Gender=Masc|Number=Plur	_	_	_	_
+16	postopkih	postopek	NOUN	Ncmpl	Case=Loc|Gender=Masc|Number=Plur	_	_	_	_
+17	pogosto	pogosto	ADV	Rgp	Degree=Pos	_	_	_	_
+18	prihaja	prihajati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+19	do	do	ADP	Sg	Case=Gen	_	_	_	_
+20	napak	napaka	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	SpaceAfter=No
+21	,	,	PUNCT	Z	_	_	_	_	_
+22	zaradi	zaradi	ADP	Sg	Case=Gen	_	_	_	_
+23	katerih	kateri	DET	Pq-fpg	Case=Gen|Gender=Masc|Number=Plur|PronType=Int	_	_	_	_
+24	so	biti	AUX	Va-r3p-n	Mood=Ind|Number=Plur|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+25	kršene	kršen	ADJ	Appfpn	Case=Nom|Degree=Pos|Gender=Fem|Number=Plur|VerbForm=Part	_	_	_	_
+26	pravice	pravica	NOUN	Ncfpn	Case=Nom|Gender=Fem|Number=Plur	_	_	_	_
+27	zavezancev	zavezanec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+28	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.6
+# text = V teh primerih je pomembno, da zakon, ki ureja davčni postopek, ureja tudi pravice zavezancev na ta način, da s tem ni ogrožen javni interes. Zakon, ki ureja davčni postopek, je v Republiki Sloveniji od leta 1996 poseben Zakon o davčnem postopku, ki pa je eden od zakonov, ki se najpogosteje spreminja.
+1	V	v	ADP	Sl	Case=Loc	_	_	_	_
+2	teh	ta	DET	Pd-mpl	Case=Loc|Gender=Masc|Number=Plur|PronType=Dem	_	_	_	_
+3	primerih	primer	NOUN	Ncmpl	Case=Loc|Gender=Masc|Number=Plur	_	_	_	_
+4	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+5	pomembno	pomemben	ADJ	Agpnsn	Case=Nom|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	SpaceAfter=No
+6	,	,	PUNCT	Z	_	_	_	_	_
+7	da	da	SCONJ	Cs	_	_	_	_	_
+8	zakon	zakon	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+9	,	,	PUNCT	Z	_	_	_	_	_
+10	ki	ki	SCONJ	Cs	_	_	_	_	_
+11	ureja	urejati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+12	davčni	davčen	ADJ	Agpmsay	Case=Acc|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+13	postopek	postopek	NOUN	Ncmsan	Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+14	,	,	PUNCT	Z	_	_	_	_	_
+15	ureja	urejati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+16	tudi	tudi	PART	Q	_	_	_	_	_
+17	pravice	pravica	NOUN	Ncfpa	Case=Acc|Gender=Fem|Number=Plur	_	_	_	_
+18	zavezancev	zavezanec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+19	na	na	ADP	Sa	Case=Acc	_	_	_	_
+20	ta	ta	DET	Pd-msa	Case=Acc|Gender=Masc|Number=Sing|PronType=Dem	_	_	_	_
+21	način	način	NOUN	Ncmsan	Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+22	,	,	PUNCT	Z	_	_	_	_	_
+23	da	da	SCONJ	Cs	_	_	_	_	_
+24	s	z	ADP	Si	Case=Ins	_	_	_	_
+25	tem	ta	DET	Pd-nsi	Case=Ins|Gender=Neut|Number=Sing|PronType=Dem	_	_	_	_
+26	ni	biti	AUX	Va-r3s-y	Mood=Ind|Number=Sing|Person=3|Polarity=Neg|Tense=Pres|VerbForm=Fin	_	_	_	_
+27	ogrožen	ogrožen	ADJ	Appmsnn	Case=Nom|Definite=Ind|Degree=Pos|Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	_
+28	javni	javen	ADJ	Agpmsny	Case=Nom|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+29	interes	interes	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+30	.	.	PUNCT	Z	_	_	_	_	_
+31	Zakon	zakon	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+32	,	,	PUNCT	Z	_	_	_	_	_
+33	ki	ki	SCONJ	Cs	_	_	_	_	_
+34	ureja	urejati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+35	davčni	davčen	ADJ	Agpmsay	Case=Acc|Definite=Def|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+36	postopek	postopek	NOUN	Ncmsan	Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+37	,	,	PUNCT	Z	_	_	_	_	_
+38	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+39	v	v	ADP	Sl	Case=Loc	_	_	_	_
+40	Republiki	republika	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+41	Sloveniji	Slovenija	PROPN	Npfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+42	od	od	ADP	Sg	Case=Gen	_	_	_	_
+43	leta	leto	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+44	1996	1996	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+45	poseben	poseben	ADJ	Agpmsnn	Case=Nom|Definite=Ind|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+46	Zakon	zakon	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+47	o	o	ADP	Sl	Case=Loc	_	_	_	_
+48	davčnem	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+49	postopku	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+50	,	,	PUNCT	Z	_	_	_	_	_
+51	ki	ki	SCONJ	Cs	_	_	_	_	_
+52	pa	pa	CCONJ	Cc	_	_	_	_	_
+53	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+54	eden	eden	NUM	Mlpmsn	Case=Nom|Gender=Masc|Number=Sing|NumForm=Word|NumType=Card	_	_	_	_
+55	od	od	ADP	Sg	Case=Gen	_	_	_	_
+56	zakonov	zakon	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+57	,	,	PUNCT	Z	_	_	_	_	_
+58	ki	ki	SCONJ	Cs	_	_	_	_	_
+59	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+60	najpogosteje	pogosto	ADV	Rgs	Degree=Sup	_	_	_	_
+61	spreminja	spreminjati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	SpaceAfter=No
+62	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.7
+# text = V 15 letih smo tako dobili kar tri nove Zakone o davčnem postopku, sedaj od leta 2007 velja ZDavP-2, ki pa je bil tudi že štirikrat noveliran.
+1	V	v	ADP	Sl	Case=Loc	_	_	_	_
+2	15	15	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+3	letih	leto	NOUN	Ncnpl	Case=Loc|Gender=Neut|Number=Plur	_	_	_	_
+4	smo	biti	AUX	Va-r1p-n	Mood=Ind|Number=Plur|Person=1|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+5	tako	tako	ADV	Rgp	Degree=Pos	_	_	_	_
+6	dobili	dobiti	VERB	Vmep-pm	Aspect=Perf|Gender=Masc|Number=Plur|VerbForm=Part	_	_	_	_
+7	kar	kar	ADV	Rgp	Degree=Pos	_	_	_	_
+8	tri	trije	NUM	Mlcmpa	Case=Acc|Gender=Masc|Number=Plur|NumForm=Word|NumType=Card	_	_	_	_
+9	nove	nov	ADJ	Agpmpa	Case=Acc|Degree=Pos|Gender=Masc|Number=Plur	_	_	_	_
+10	Zakone	zakon	NOUN	Ncmpa	Case=Acc|Gender=Masc|Number=Plur	_	_	_	_
+11	o	o	ADP	Sl	Case=Loc	_	_	_	_
+12	davčnem	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+13	postopku	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+14	,	,	PUNCT	Z	_	_	_	_	_
+15	sedaj	sedaj	ADV	Rgp	Degree=Pos	_	_	_	_
+16	od	od	ADP	Sg	Case=Gen	_	_	_	_
+17	leta	leto	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+18	2007	2007	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+19	velja	veljati	VERB	Vmpr3s	Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+20	ZDavP	ZDavP	PROPN	Npmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+21	-	-	PUNCT	Z	_	_	_	_	SpaceAfter=No
+22	2	2	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	SpaceAfter=No
+23	,	,	PUNCT	Z	_	_	_	_	_
+24	ki	ki	SCONJ	Cs	_	_	_	_	_
+25	pa	pa	CCONJ	Cc	_	_	_	_	_
+26	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+27	bil	biti	AUX	Va-p-sm	Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	_
+28	tudi	tudi	PART	Q	_	_	_	_	_
+29	že	že	PART	Q	_	_	_	_	_
+30	štirikrat	štirikrat	ADV	Rgp	Degree=Pos	_	_	_	_
+31	noveliran	noveliran	ADJ	Appmsnn	Case=Nom|Definite=Ind|Degree=Pos|Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	SpaceAfter=No
+32	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.8
+# text = Pogosto so te spremembe ravno v smislu povečevanja pravic zavezancev, dodajajo pa se tudi instituti, ki zavezance spodbujajo k temu, da bi v čim večji meri in pravočasno poravnali svoje davčne obveznosti.
+1	Pogosto	pogosto	ADV	Rgp	Degree=Pos	_	_	_	_
+2	so	biti	VERB	Va-r3p-n	Mood=Ind|Number=Plur|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+3	te	ta	DET	Pd-fpn	Case=Nom|Gender=Fem|Number=Plur|PronType=Dem	_	_	_	_
+4	spremembe	sprememba	NOUN	Ncfpn	Case=Nom|Gender=Fem|Number=Plur	_	_	_	_
+5	ravno	ravno	PART	Q	_	_	_	_	_
+6	v	v	ADP	Sl	Case=Loc	_	_	_	_
+7	smislu	smisel	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+8	povečevanja	povečevanje	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+9	pravic	pravica	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+10	zavezancev	zavezanec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+11	,	,	PUNCT	Z	_	_	_	_	_
+12	dodajajo	dodajati	VERB	Vmpr3p	Aspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+13	pa	pa	CCONJ	Cc	_	_	_	_	_
+14	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+15	tudi	tudi	PART	Q	_	_	_	_	_
+16	instituti	institut	NOUN	Ncmpn	Case=Nom|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+17	,	,	PUNCT	Z	_	_	_	_	_
+18	ki	ki	SCONJ	Cs	_	_	_	_	_
+19	zavezance	zavezanec	NOUN	Ncmpa	Case=Acc|Gender=Masc|Number=Plur	_	_	_	_
+20	spodbujajo	spodbujati	VERB	Vmpr3p	Aspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+21	k	k	ADP	Sd	Case=Dat	_	_	_	_
+22	temu	ta	DET	Pd-nsd	Case=Dat|Gender=Neut|Number=Sing|PronType=Dem	_	_	_	SpaceAfter=No
+23	,	,	PUNCT	Z	_	_	_	_	_
+24	da	da	SCONJ	Cs	_	_	_	_	_
+25	bi	biti	AUX	Va-c	Mood=Cnd|VerbForm=Fin	_	_	_	_
+26	v	v	ADP	Sl	Case=Loc	_	_	_	_
+27	čim	čim	ADV	Rgp	Degree=Pos	_	_	_	_
+28	večji	velik	ADJ	Agcfsl	Case=Loc|Degree=Cmp|Gender=Fem|Number=Sing	_	_	_	_
+29	meri	mera	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+30	in	in	CCONJ	Cc	_	_	_	_	_
+31	pravočasno	pravočasno	ADV	Rgp	Degree=Pos	_	_	_	_
+32	poravnali	poravnati	VERB	Vmep-pm	Aspect=Perf|Gender=Masc|Number=Plur|VerbForm=Part	_	_	_	_
+33	svoje	svoj	DET	Px-fpa	Case=Acc|Gender=Fem|Number=Plur|Poss=Yes|PronType=Prs|Reflex=Yes	_	_	_	_
+34	davčne	davčen	ADJ	Agpfpa	Case=Acc|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+35	obveznosti	obveznost	NOUN	Ncfpa	Case=Acc|Gender=Fem|Number=Plur	_	_	_	SpaceAfter=No
+36	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.9
+# text = Poleg tega se nadrejeno uporabljajo procesne določbe področne zakonodaje EU in RS, podrejeno pa ZUP. K spremembam davčne zakonodaje pa je pripomoglo tudi dejstvo, da se tudi v Sloveniji spopadamo z gospodarsko krizo, ki je prizadela tudi zavezance.
+1	Poleg	poleg	ADP	Sg	Case=Gen	_	_	_	_
+2	tega	ta	DET	Pd-nsg	Case=Gen|Gender=Neut|Number=Sing|PronType=Dem	_	_	_	_
+3	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+4	nadrejeno	nadrejeno	ADV	Rgp	Degree=Pos	_	_	_	_
+5	uporabljajo	uporabljati	VERB	Vmpr3p	Aspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+6	procesne	procesen	ADJ	Agpfpn	Case=Nom|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+7	določbe	določba	NOUN	Ncfpn	Case=Nom|Gender=Fem|Number=Plur	_	_	_	_
+8	področne	področen	ADJ	Agpfsg	Case=Gen|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+9	zakonodaje	zakonodaja	NOUN	Ncfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	_
+10	EU	EU	PROPN	Npfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	_
+11	in	in	CCONJ	Cc	_	_	_	_	_
+12	RS	RS	PROPN	Npfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+13	,	,	PUNCT	Z	_	_	_	_	_
+14	podrejeno	podrejen	ADJ	Appnsn	Case=Nom|Degree=Pos|Gender=Neut|Number=Sing|VerbForm=Part	_	_	_	_
+15	pa	pa	CCONJ	Cc	_	_	_	_	_
+16	ZUP	ZUP	PROPN	Npfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+17	.	.	PUNCT	Z	_	_	_	_	_
+18	K	k	ADP	Sd	Case=Dat	_	_	_	_
+19	spremembam	sprememba	NOUN	Ncfpd	Case=Dat|Gender=Fem|Number=Plur	_	_	_	_
+20	davčne	davčen	ADJ	Agpfsg	Case=Gen|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+21	zakonodaje	zakonodaja	NOUN	Ncfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	_
+22	pa	pa	CCONJ	Cc	_	_	_	_	_
+23	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+24	pripomoglo	pripomoči	VERB	Vmep-sn	Aspect=Perf|Gender=Neut|Number=Sing|VerbForm=Part	_	_	_	_
+25	tudi	tudi	PART	Q	_	_	_	_	_
+26	dejstvo	dejstvo	NOUN	Ncnsn	Case=Nom|Gender=Neut|Number=Sing	_	_	_	SpaceAfter=No
+27	,	,	PUNCT	Z	_	_	_	_	_
+28	da	da	SCONJ	Cs	_	_	_	_	_
+29	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+30	tudi	tudi	PART	Q	_	_	_	_	_
+31	v	v	ADP	Sl	Case=Loc	_	_	_	_
+32	Sloveniji	Slovenija	PROPN	Npfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+33	spopadamo	spopadati	VERB	Vmpr1p	Aspect=Imp|Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Fin	_	_	_	_
+34	z	z	ADP	Si	Case=Ins	_	_	_	_
+35	gospodarsko	gospodarski	ADJ	Agpfsi	Case=Ins|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+36	krizo	kriza	NOUN	Ncfsi	Case=Ins|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+37	,	,	PUNCT	Z	_	_	_	_	_
+38	ki	ki	SCONJ	Cs	_	_	_	_	_
+39	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+40	prizadela	prizadeti	VERB	Vmep-sf	Aspect=Perf|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+41	tudi	tudi	PART	Q	_	_	_	_	_
+42	zavezance	zavezanec	NOUN	Ncmpa	Case=Acc|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+43	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.10
+# text = Mnogi zavezanci se soočajo s finančnimi težavami, zato se je Zakon o davčnem postopku ravno v luči gospodarske krize spremenil v tej smeri, da zavezancem olajša njihov položaj in jim omogoči več različnih možnosti za poplačilo davčnih obveznosti, predvsem z odlogi plačila in obročnim plačevanjem. Možnost odpisa, delnega odpisa, odloga in obročnega poplačila davčnih obveznosti se je od uvedbe prvega Zakona o davčnem postopku (1996) pa do danes povečala.
+1	Mnogi	mnog	DET	Pi-mpn	Case=Nom|Gender=Masc|Number=Plur|PronType=Ind	_	_	_	_
+2	zavezanci	zavezanec	NOUN	Ncmpn	Case=Nom|Gender=Masc|Number=Plur	_	_	_	_
+3	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+4	soočajo	soočati	VERB	Vmpr3p	Aspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+5	s	z	ADP	Si	Case=Ins	_	_	_	_
+6	finančnimi	finančen	ADJ	Agpfpi	Case=Ins|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+7	težavami	težava	NOUN	Ncfpi	Case=Ins|Gender=Fem|Number=Plur	_	_	_	SpaceAfter=No
+8	,	,	PUNCT	Z	_	_	_	_	_
+9	zato	zato	CCONJ	Cc	_	_	_	_	_
+10	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+11	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+12	Zakon	zakon	NOUN	Ncmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	_
+13	o	o	ADP	Sl	Case=Loc	_	_	_	_
+14	davčnem	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+15	postopku	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+16	ravno	ravno	PART	Q	_	_	_	_	_
+17	v	v	ADP	Sl	Case=Loc	_	_	_	_
+18	luči	luč	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	_
+19	gospodarske	gospodarski	ADJ	Agpfsg	Case=Gen|Degree=Pos|Gender=Fem|Number=Sing	_	_	_	_
+20	krize	kriza	NOUN	Ncfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	_
+21	spremenil	spremeniti	VERB	Vmep-sm	Aspect=Perf|Gender=Masc|Number=Sing|VerbForm=Part	_	_	_	_
+22	v	v	ADP	Sl	Case=Loc	_	_	_	_
+23	tej	ta	DET	Pd-fsl	Case=Loc|Gender=Fem|Number=Sing|PronType=Dem	_	_	_	_
+24	smeri	smer	NOUN	Ncfsl	Case=Loc|Gender=Fem|Number=Sing	_	_	_	SpaceAfter=No
+25	,	,	PUNCT	Z	_	_	_	_	_
+26	da	da	SCONJ	Cs	_	_	_	_	_
+27	zavezancem	zavezanec	NOUN	Ncmpd	Case=Dat|Gender=Masc|Number=Plur	_	_	_	_
+28	olajša	olajšati	VERB	Vmer3s	Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+29	njihov	njihov	DET	Ps3msap	Case=Acc|Gender=Masc|Number=Sing|Number[psor]=Plur|Person=3|Poss=Yes|PronType=Prs	_	_	_	_
+30	položaj	položaj	NOUN	Ncmsan	Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing	_	_	_	_
+31	in	in	CCONJ	Cc	_	_	_	_	_
+32	jim	on	PRON	Pp3mpd--y	Case=Dat|Gender=Masc|Number=Plur|Person=3|PronType=Prs|Variant=Short	_	_	_	_
+33	omogoči	omogočiti	VERB	Vmer3s	Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
+34	več	več	DET	Rgc	PronType=Ind	_	_	_	_
+35	različnih	različen	ADJ	Agpfpg	Case=Gen|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+36	možnosti	možnost	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+37	za	za	ADP	Sa	Case=Acc	_	_	_	_
+38	poplačilo	poplačilo	NOUN	Ncnsa	Case=Acc|Gender=Neut|Number=Sing	_	_	_	_
+39	davčnih	davčen	ADJ	Agpfpg	Case=Gen|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+40	obveznosti	obveznost	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	SpaceAfter=No
+41	,	,	PUNCT	Z	_	_	_	_	_
+42	predvsem	predvsem	PART	Q	_	_	_	_	_
+43	z	z	ADP	Si	Case=Ins	_	_	_	_
+44	odlogi	odlog	NOUN	Ncmpi	Case=Ins|Gender=Masc|Number=Plur	_	_	_	_
+45	plačila	plačilo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+46	in	in	CCONJ	Cc	_	_	_	_	_
+47	obročnim	obročen	ADJ	Agpnsi	Case=Ins|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+48	plačevanjem	plačevanje	NOUN	Ncnsi	Case=Ins|Gender=Neut|Number=Sing	_	_	_	SpaceAfter=No
+49	.	.	PUNCT	Z	_	_	_	_	_
+50	Možnost	možnost	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+51	odpisa	odpis	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+52	,	,	PUNCT	Z	_	_	_	_	_
+53	delnega	delen	ADJ	Agpmsg	Case=Gen|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+54	odpisa	odpis	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+55	,	,	PUNCT	Z	_	_	_	_	_
+56	odloga	odlog	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+57	in	in	CCONJ	Cc	_	_	_	_	_
+58	obročnega	obročen	ADJ	Agpnsg	Case=Gen|Degree=Pos|Gender=Neut|Number=Sing	_	_	_	_
+59	poplačila	poplačilo	NOUN	Ncnsg	Case=Gen|Gender=Neut|Number=Sing	_	_	_	_
+60	davčnih	davčen	ADJ	Agpfpg	Case=Gen|Degree=Pos|Gender=Fem|Number=Plur	_	_	_	_
+61	obveznosti	obveznost	NOUN	Ncfpg	Case=Gen|Gender=Fem|Number=Plur	_	_	_	_
+62	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+63	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+64	od	od	ADP	Sg	Case=Gen	_	_	_	_
+65	uvedbe	uvedba	NOUN	Ncfsg	Case=Gen|Gender=Fem|Number=Sing	_	_	_	_
+66	prvega	prvi	ADJ	Mlomsg	Case=Gen|Gender=Masc|Number=Sing|NumType=Ord	_	_	_	_
+67	Zakona	zakon	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+68	o	o	ADP	Sl	Case=Loc	_	_	_	_
+69	davčnem	davčen	ADJ	Agpmsl	Case=Loc|Degree=Pos|Gender=Masc|Number=Sing	_	_	_	_
+70	postopku	postopek	NOUN	Ncmsl	Case=Loc|Gender=Masc|Number=Sing	_	_	_	_
+71	(	(	PUNCT	Z	_	_	_	_	SpaceAfter=No
+72	1996	1996	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	SpaceAfter=No
+73	)	)	PUNCT	Z	_	_	_	_	_
+74	pa	pa	CCONJ	Cc	_	_	_	_	_
+75	do	do	ADP	Sg	Case=Gen	_	_	_	_
+76	danes	danes	ADV	Rgp	Degree=Pos	_	_	_	_
+77	povečala	povečati	VERB	Vmep-sf	Aspect=Perf|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	SpaceAfter=No
+78	.	.	PUNCT	Z	_	_	_	_	_
+
+# sent_id = 1.11
+# text = Od prvotnih dovoljenih do 6 mesecev odloga oziroma največ do 6 mesečnih obrokov, se je možnost povečala na 12 mesecev odloga oziroma največ do 12 mesečnih obrokov (ZDavP-1), do 24 mesecev oziroma do največ 24 mesečnih obrokov.
+1	Od	od	ADP	Sg	Case=Gen	_	_	_	_
+2	prvotnih	prvoten	ADJ	Agpmpg	Case=Gen|Degree=Pos|Gender=Masc|Number=Plur	_	_	_	_
+3	dovoljenih	dovoljen	ADJ	Appmpg	Case=Gen|Degree=Pos|Gender=Masc|Number=Plur|VerbForm=Part	_	_	_	_
+4	do	do	ADP	Sg	Case=Gen	_	_	_	_
+5	6	6	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+6	mesecev	mesec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+7	odloga	odlog	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+8	oziroma	oziroma	CCONJ	Cc	_	_	_	_	_
+9	največ	največ	DET	Rgs	PronType=Ind	_	_	_	_
+10	do	do	ADP	Sg	Case=Gen	_	_	_	_
+11	6	6	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+12	mesečnih	mesečen	ADJ	Agpmpg	Case=Gen|Degree=Pos|Gender=Masc|Number=Plur	_	_	_	_
+13	obrokov	obrok	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+14	,	,	PUNCT	Z	_	_	_	_	_
+15	se	se	PRON	Px------y	PronType=Prs|Reflex=Yes|Variant=Short	_	_	_	_
+16	je	biti	AUX	Va-r3s-n	Mood=Ind|Number=Sing|Person=3|Polarity=Pos|Tense=Pres|VerbForm=Fin	_	_	_	_
+17	možnost	možnost	NOUN	Ncfsn	Case=Nom|Gender=Fem|Number=Sing	_	_	_	_
+18	povečala	povečati	VERB	Vmep-sf	Aspect=Perf|Gender=Fem|Number=Sing|VerbForm=Part	_	_	_	_
+19	na	na	ADP	Sa	Case=Acc	_	_	_	_
+20	12	12	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+21	mesecev	mesec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+22	odloga	odlog	NOUN	Ncmsg	Case=Gen|Gender=Masc|Number=Sing	_	_	_	_
+23	oziroma	oziroma	CCONJ	Cc	_	_	_	_	_
+24	največ	največ	DET	Rgs	PronType=Ind	_	_	_	_
+25	do	do	ADP	Sg	Case=Gen	_	_	_	_
+26	12	12	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+27	mesečnih	mesečen	ADJ	Agpmpg	Case=Gen|Degree=Pos|Gender=Masc|Number=Plur	_	_	_	_
+28	obrokov	obrok	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+29	(	(	PUNCT	Z	_	_	_	_	SpaceAfter=No
+30	ZDavP	ZDavP	PROPN	Npmsn	Case=Nom|Gender=Masc|Number=Sing	_	_	_	SpaceAfter=No
+31	-	-	PUNCT	Z	_	_	_	_	SpaceAfter=No
+32	1	1	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	SpaceAfter=No
+33	)	)	PUNCT	Z	_	_	_	_	SpaceAfter=No
+34	,	,	PUNCT	Z	_	_	_	_	_
+35	do	do	ADP	Sg	Case=Gen	_	_	_	_
+36	24	24	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+37	mesecev	mesec	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	_
+38	oziroma	oziroma	CCONJ	Cc	_	_	_	_	_
+39	do	do	ADP	Sg	Case=Gen	_	_	_	_
+40	največ	največ	DET	Rgs	PronType=Ind	_	_	_	_
+41	24	24	NUM	Mdc	NumForm=Digit|NumType=Card	_	_	_	_
+42	mesečnih	mesečen	ADJ	Agpmpg	Case=Gen|Degree=Pos|Gender=Masc|Number=Plur	_	_	_	_
+43	obrokov	obrok	NOUN	Ncmpg	Case=Gen|Gender=Masc|Number=Plur	_	_	_	SpaceAfter=No
+44	.	.	PUNCT	Z	_	_	_	_	_

ATEapi/utils.py

@@ -0,0 +1,90 @@
+import os
+import glob
+import json
+import numpy as np
+import pandas as pd
+import warnings
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+def read_conll(f, lowercase=True, max_example=None):
+    examples = []
+    # with open(in_file) as f:
+    word, lemma, pos, msd, label = [], [], [], [], []
+    for line in f.readlines():
+        sp = line.decode('utf-8').strip().split('\t')
+        # print(sp)
+        if len(sp) == 10:
+            if '-' not in sp[0]:
+                word.append(sp[1].lower() if lowercase else sp[1])
+                lemma.append(sp[2].lower() if lowercase else sp[2])
+                pos.append(sp[3])
+                msd.append(sp[4])
+                label.append('n')
+        elif len(word) > 0:
+            examples.append({'word': word, 'lemma': lemma, 'pos': pos, 'msd':msd,   'labels': label})
+            word, lemma, pos, msd, label = [], [], [], [], []
+            if (max_example is not None) and (len(examples) == max_example):
+                break
+    if len(word) > 0:
+        examples.append({'word': word, 'lemma': lemma, 'pos': pos,  'msd':msd, 'labels': label})
+    return pd.DataFrame(examples)
+
+
+def extract_terms_full(token_predictions,token_probabilities, val_texts, lemma_texts, pos_texts, msd_texts):
+    extracted_terms = list()
+    extracted_probs = list()
+    lemma_terms = list()
+    pos_terms = list()
+    msd_terms = list()
+    # go over all predictions
+    for i in range(len(token_predictions)):
+        pred = token_predictions[i]
+        prob = token_probabilities[i]
+        txt  = val_texts[i]
+        lemma  = lemma_texts[i]
+        pos  = pos_texts[i]
+        msd  = msd_texts[i]
+        for j in range(len(pred)):
+            if pred[j] == "B-T":
+                term = txt[j]
+                prob_term = str(round(prob[j][1],4))
+                lemma_term = lemma[j]
+                pos_term = pos[j]
+                msd_term = msd[j]
+                for k in range(j+1,len(pred)):
+                    if pred[k]=="T": 
+                        term+=" "+txt[k]
+                        prob_term+=" "+str(round(prob[k][2],4))
+                        lemma_term+=" "+ lemma[k]
+                        pos_term+=" "+ pos[k]
+                        msd_term+=" "+ msd[k]
+                    else: break
+                extracted_terms.append(term)
+                extracted_probs.append(prob_term)
+                lemma_terms.append(lemma_term)
+                pos_terms.append(pos_term)
+                msd_terms.append(msd_term)
+    return extracted_terms, extracted_probs, lemma_terms, pos_terms, msd_terms
+
+def remap(tokenizer, texts, preds, probs):
+    text = texts.split(' ')
+    len_token = []
+    res_pred = []
+    final_pred = []
+    res_prob = []
+    final_prob = []
+    for i in range(len(text)):
+        temp = len(tokenizer(text[i], add_special_tokens=False)['input_ids'])
+        len_token.append(temp)
+    for val in len_token:
+        res_pred.append(val)
+        res_pred.extend([0]*(val-1))    
+        res_prob.append(val)
+        res_prob.extend([0]*(val-1))  
+    for p1, p2 in zip(res_pred, preds[1:len(preds)-1]):
+        if p1 != 0:
+            final_pred.append(p2)
+    for p1, p2 in zip(res_prob, probs[1:len(probs)-1]):
+        if p1 != 0:
+            final_prob.append(p2)
+    return text, final_pred, final_prob

Dockerfile

@@ -1,10 +1,12 @@
-FROM python:3.6-alpine
+FROM python:3.9
+
+
+RUN apt-get update && apt-get install python3-pil tesseract-ocr libtesseract-dev tesseract-ocr-eng tesseract-ocr-slv tesseract-ocr-script-latn ffmpeg libsm6 libxext6 libgl1 libmagic1 -y
 
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
-
 COPY requirements.txt /usr/src/app/
-
+# RUN pip install --upgrade pip
 RUN pip3 install --no-cache-dir -r requirements.txt
 
 COPY . /usr/src/app
@@ -13,4 +15,4 @@ EXPOSE 8080
 
 ENTRYPOINT ["python3"]
 
-CMD ["-m", "swagger_server"]
+CMD ["-m", "swagger_server"]

README.md

@@ -1,49 +1,69 @@
-# Swagger generated server
+# Luščilnik terminoloških kandidatov
 
-## Overview
+## Delovanje
-This server was generated by the [swagger-codegen](https://github.com/swagger-api/swagger-codegen) project. By using the
-[OpenAPI-Spec](https://github.com/swagger-api/swagger-core/wiki) from a remote server, you can easily generate a server stub.  This
-is an example of building a swagger-enabled Flask server.
 
-This example uses the [Connexion](https://github.com/zalando/connexion) library on top of Flask.
+Luščilnik je izdelan kot spletna storitev, ki omogoča luščenje terminoloških kandidatov iz zbirke besedil uporabnika v običajnih datotečnih formatih za zapis besedil, kot tudi luščenje terminoloških kandidatov iz korpusa oss.
 
-## Requirements
+Spletna storitev omogoča celotno luščenje v enem samem koraku, zagon posameznih segmentov obdelave ter omogoča tako sinhrono delovanje, kot tudi asinhrono delovanje, kadar uporabnik storitve pričakuje, da bo čas luščenja prevelik.
-Python 3.5.2+
 
-## Usage
+Luščenje je mogoče izvajati na dva načina, pri čemer je bolj zapleten način luščenja terminov iz uporabniških besedil. Postopek luščenja je sledeč:
-To run the server, please execute the following from the root directory:
+
+1. Pretvorba vhodnih datotek v čisto besedilo
+2. Segmentacija in označevanje besedil z POS označevalnikom classla
+3. Luščenje terminologije
+4. Iskanje kanoničnih oblik
+5. Iskanje dobrih primerov uporabe
+
+Kadar gre za luščenje terminov iz korpusa OSS, uporabnik poda obseg besedil preko filtra leta, vrste dokumentov, ključnih besed in udk vrstilca. Ker so dokumenti že predobdelani (pretvorjeni in označeni) se 1. in 2. korak lahko preskočita.
+
+Natančen opis klicev in uporabe storitev je dostopen na http://rsdo.lhrs.feri.um.si:8080/ui/
+
+## Komponente
+
+### Programski vmesnik za luščenje 
+
+Programski vmesnik za luščenje sestavlja python flask strežnik, v katerem tečejo orkestracijske storitve, ki omogočajo izvajanje luščenja na način REST. Zraven teče še swagger strežnik z opisom api-ja, ki uporabnikom omogoča lažjo integracijo v svoje storitve. Vmesnikovo delovanje je odvisno od ostalih kompoment sistema.
+
+
+### SloBERT luščilnik 
+
+Programski vmesnik do luščilnika, naučenega na RSDO5 učnem korpusu, ki omogoča luščenje iz besedil v conllu obiliki.
+Natančen opis in izvorni repoitorij dela projekta se nahaja na https://github.com/honghanhh/ate-docker
+
+### Kanonizator besednih zvez 
+
+Programski vmesnik do kanonizatorja besednih zvez, ki se uporablja za pretvorbo najdenih terminoloških kandidatov iz lematizirane ali neosnovne oblike, v kanonično obliko.
+Natančen opis in izvorni repozitorij dela projekta se nahaja na: https://github.com/honghanhh/ate-docker
+
+### Podatkovna baza korpusa OSS 
+
+Podatkovna baza korpusa OSS vsebuje obdelana besedila iz korpusa OSS ter hkrati tudi iskalne indekse, ki omogočajo hitrejše poizvedbe po podatkih. Prav tako so v podatkovni bazi predizračunani hevristični indeksi za iskanje terminoloških kandidatov ter procedure za obdelavo uporabniških besedil na enak način. Ker je sama vsebina podatkovne baze prevelika, so dodane samo skripte za kreiranje strukture tabele, sam korpus pa je v bazo potrebno po instalaciji uvoziti in zagnati njegovo obdelavo.
+
+Uporabljena je odprtokodna podatkovna baza MariaDB z ColumnStore hrambo. 
+
+## Zahteve
+Za izvajanje priporočamo docker izvajalno okolje
+Posamezni deli imajo ločene zahteve, ki se lahko razberejo v Dockerfile skriptah
+
+
+## Uporaba
+Za zagon storitve je potrebno najprej ustrezno konfigurirati izvajaln okolje docker. To je možno preko datoteke docker_compose.yml. Ta je prednastavljena tako, da pričakuje, da bo v nadmapi izvorne kode ustvarjena mapa, v katero se bodo naložili jezikovni modeli, saj se tako ne prenaša v izvajalno okolje docker. Uporabnik lahko to seveda spremeni po želji.
+
+Za uporabo korpusa OSS je potrebno slednjega ustrezno uvoziti v podatkovno bazo. Objavljen je na clarin.si 
+
+Izgradnja slike se zažene z 
 
 ```
-pip3 install -r requirements.txt
+docker-compose build
-python3 -m swagger_server
 ```
 
-and open your browser to here:
+zagon pa z 
 
 ```
-http://localhost:8080/ui/
+docker-compose up
 ```
 
-Your Swagger definition lives here:
+## Avtorji
 
-```
+Pri ustvarjanju projekta so sodelovali:
-http://localhost:8080/swagger.json
+Marko Ferme, Hanh Thi Hong Tran, Klemen Kac, Matej Martinc, Milan Ojsteršek, Vid Podpečan, Senja Pollak, Marko Pranjić, Andraž Repar, Kristjan Žagar
-```
-
-To launch the integration tests, use tox:
-```
-sudo pip install tox
-tox
-```
-
-## Running with Docker
-
-To run the server on a Docker container, please execute the following from the root directory:
-
-```bash
-# building the image
-docker build -t swagger_server .
-
-# starting up a container
-docker run -p 8080:8080 swagger_server
-```

canonical_forms/Dockerfile

@@ -0,0 +1,25 @@
+# pull official base image
+FROM python:3.8-slim-buster
+
+# set work directory
+WORKDIR /usr/src/app
+
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# install system dependencies
+RUN apt-get update && apt-get install -y netcat
+#git
+
+# install dependencies
+RUN pip install --upgrade pip
+COPY ./requirements.txt /usr/src/app/requirements.txt
+RUN pip install -r requirements.txt
+#RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
+
+# copy project
+COPY . /usr/src/app/
+
+# run entrypoint.sh
+ENTRYPOINT ["/usr/src/app/entrypoint.sh"]

canonical_forms/Dockerfile.prod

@@ -0,0 +1,74 @@
+###########
+# BUILDER #
+###########
+
+# pull official base image
+FROM python:3.8-slim-buster as builder
+
+# set work directory
+WORKDIR /usr/src/app
+
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# install system dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends gcc
+
+# lint
+#RUN pip install --upgrade pip
+#RUN pip install flake8
+#COPY . /usr/src/app/
+#RUN flake8 --ignore=E501,F401 .
+
+# install python dependencies
+COPY ./requirements.txt .
+RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
+
+
+#########
+# FINAL #
+#########
+
+# pull official base image
+FROM python:3.8-slim-buster
+
+# create directory for the app user
+RUN mkdir -p /home/app
+
+# create the app user
+RUN addgroup --system app && adduser --system --group app
+
+
+# create the appropriate directories
+ENV HOME=/home/app
+ENV APP_HOME=/home/app/web
+RUN mkdir $APP_HOME
+WORKDIR $APP_HOME
+
+# install dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends netcat
+#git
+COPY --from=builder /usr/src/app/wheels /wheels
+COPY --from=builder /usr/src/app/requirements.txt .
+RUN pip install --upgrade pip
+RUN pip install --no-cache /wheels/*
+
+# copy entrypoint-prod.sh
+COPY ./entrypoint.prod.sh $APP_HOME
+
+# copy project
+COPY . $APP_HOME
+
+# chown all the files to the app user
+RUN chown -R app:app $APP_HOME
+
+# a workaround for permission errors for the classla_volume
+RUN mkdir $HOME/classla_resources && chown app:app $HOME/classla_resources
+
+# change to the app user
+USER app
+
+# run entrypoint.prod.sh
+ENTRYPOINT ["/home/app/web/entrypoint.prod.sh"]

canonical_forms/app.py

@@ -0,0 +1,53 @@
+import os
+import json
+
+from flask import (
+    Flask,
+    jsonify,
+    send_from_directory,
+    request,
+    redirect,
+    url_for
+)
+import werkzeug
+werkzeug.cached_property = werkzeug.utils.cached_property
+from werkzeug.utils import secure_filename
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+#import flask.scaffold
+#flask.helpers._endpoint_from_view_func = flask.scaffold._endpoint_from_view_func # fix
+from flask_restx import Api, Resource, fields, abort, reqparse
+
+from flask_socketio import SocketIO
+
+import canonizer
+
+
+app = Flask(__name__)
+app.wsgi_app = ProxyFix(app.wsgi_app)
+api = Api(app, version='1.0',
+          title='API services',
+          description='REST API for canonizer')
+ns = api.namespace('rest_api', description='REST services API')
+socketio = SocketIO(app)
+
+
+canonizer_input = api.model('CanonizerInput', {
+    'forms': fields.List(fields.String, required=True, description='list of input forms')
+})
+canonizer_output = api.model('CanonizerOutput', {
+    'canonical_forms': fields.List(fields.String, description='list of canonical forms')
+})
+
+
+@ns.route('/canonize')
+class Canonizer(Resource):
+    @ns.doc('compute canonical forms')
+    @ns.expect(canonizer_input, validate=True)
+    @ns.marshal_with(canonizer_output)
+    def post(self):
+        return {'canonical_forms': canonizer.process(api.payload['forms'])}
+
+
+if __name__ == '__main__':
+    socketio.run(app, host='0.0.0.0', debug=True, use_reloader=True)

canonical_forms/canonizer.py

@@ -0,0 +1,199 @@
+import argparse
+import csv
+import os
+import string
+
+import classla
+from lemmagen3 import Lemmatizer
+
+
+classla.download("sl", logging_level="WARNING")
+classla_nlp_pipeline = classla.Pipeline(
+    lang="sl",
+    processors="tokenize,pos,lemma,depparse",
+    tokenize_pretokenized=True,
+    logging_level="WARNING",
+)
+
+
+def _resolve_lemmagen_model_loc(model_name):
+    basedir = os.path.dirname(__file__)
+    return os.path.join(basedir, "lemmagen_models", model_name)
+
+
+_canon_lemmatizer = Lemmatizer()
+_canon_lemmatizer.load_model(_resolve_lemmagen_model_loc("kanon.bin"))
+canon_lemma = _canon_lemmatizer.lemmatize
+
+ADJ_LEMMATIZER_LOC_MAP = {
+    ("m", "s"): _resolve_lemmagen_model_loc("kanon-adj-male.bin"),
+    ("m", "p"): _resolve_lemmagen_model_loc("kanon-adj-male-plural.bin"),
+    ("f", "s"): _resolve_lemmagen_model_loc("kanon-adj-female.bin"),
+    ("f", "p"): _resolve_lemmagen_model_loc("kanon-adj-female-plural.bin"),
+    ("n", "s"): _resolve_lemmagen_model_loc("kanon-adj-neutral.bin"),
+    ("n", "p"): _resolve_lemmagen_model_loc("kanon-adj-neutral-plural.bin"),
+}
+
+_ADJ_LEMMATIZER_CACHE = {}
+
+
+def lem_adj(gender, number, wrd):
+    lem_key = (gender, number)
+    if lem_key not in _ADJ_LEMMATIZER_CACHE:
+        assert lem_key in ADJ_LEMMATIZER_LOC_MAP
+        lemmatizer_model_loc = ADJ_LEMMATIZER_LOC_MAP[lem_key]
+        lemmatizer = Lemmatizer()
+        lemmatizer.load_model(lemmatizer_model_loc)
+        _ADJ_LEMMATIZER_CACHE[lem_key] = lemmatizer
+    lemmatizer = _ADJ_LEMMATIZER_CACHE[lem_key]
+    return lemmatizer.lemmatize(wrd)
+
+
+def get_adj_msd(head, word):
+    feats = head.feats
+    feats_dict = {}
+    feats = feats.strip().split("|")
+    for f in feats:
+        f = f.strip().split("=")
+        feats_dict[f[0]] = f[1]
+    gender = feats_dict["Gender"]
+    if gender == "Masc" and len(word.xpos) == 6:
+        msd = word.xpos[:-1] + "ny"
+    elif gender == "Masc" and len(word.xpos) == 7:
+        msd = word.xpos[:-1] + "y"
+    elif gender == "Fem":
+        msd = word.xpos[:-1] + "n"
+    elif gender == "Neut":
+        msd = word.xpos[:-1] + "n"
+    else:
+        # msd = None
+        msd = "qqqqqq"  # hacky but it means that adverbs are just copied over to the canonical form
+    return msd
+
+
+def _is_single_acronym(term):
+    # (single word, all uppercase and length less than 5 characters)
+    if len(term.words) == 1:
+        word = term.words[0].text
+        return len(word) < 5 and word.isupper()
+    return False
+
+
+def _join_term_words(term):
+    return " ".join([w.text for w in term.words])
+
+
+def _process_pre(pre, head, gender, number):
+    canon = []
+    for el in pre:
+        msd = get_adj_msd(head, el)
+        if msd[0] == "A":
+            form = lem_adj(gender, number, el.text.lower())
+            canon.append(form)
+        else:
+            canon.append(el.lemma.lower())
+    return canon
+
+
+def find_canon(term):
+    if _is_single_acronym(term):
+        return term.words[0].text
+
+    head = None
+    pre = []
+    post = []
+
+    for word in term.words:
+        if word.head == 0:
+            head = word
+        elif head is None:
+            pre.append(word)
+        else:
+            post.append(word)
+    ## special case where all words are proper nouns and each word is canonized independently
+    if all(w.upos == "PROPN" for w in term.words):
+        canon_name = [canon_lemma(w.text) for w in term.words]
+        return " ".join(canon_name)
+
+    if head is None:
+        if len(term.words) == 1:
+            head2 = term.words[0]
+            return canon_lemma(head2.text.lower())
+        else:
+            # just return the input because we do not cover such case
+            return _join_term_words(term)
+    if head.upos == "VERB":  # if the term is not a noun phrase
+        # just return the input because we do not cover such case
+        return _join_term_words(term)
+    if head.upos == "ADJ":
+        if len(term.words) == 1:  # for single word adjectives, return male form
+            return lem_adj("m", "s", term.words[0].text.lower())
+        else:
+            # just return the input because we do not cover such case
+            return _join_term_words(term)
+
+    gender = head.xpos[2]
+    number = head.xpos[3]
+    ending = head.lemma[-1]
+    if gender == "f" and number == "p" and ending in "ie":  # sani, hlače
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    elif gender == "m" and number == "p" and ending == "i":  # možgani
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    elif gender == "n" and number == "p" and ending == "a":  # vrata
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    else:
+        canon = _process_pre(pre, head, gender, "s")
+        head_form = canon_lemma(head.text.lower())
+        canon.append(head_form)
+
+    for el in post:
+        canon.append(el.text)
+    return " ".join(canon)
+
+
+def process(forms):
+    text = "\n".join(forms)
+    doc = classla_nlp_pipeline(text)
+    canonical_forms = []
+    for term in doc.sentences:
+        try:
+            canonical_form = find_canon(term)
+        except Exception:
+            canonical_form = _join_term_words(term)
+        canonical_forms.append(canonical_form)
+    return canonical_forms
+
+
+def read_csv(fname, columnID=0):
+    data = []
+    with open(fname) as csvfile:
+        try:
+            dialect = csv.Sniffer().sniff(csvfile.read(2048))
+        except csv.Error:
+            print("Warning: cannot determine delimiter, assuming Excel CSV dialect.")
+            dialect = "excel"
+        csvfile.seek(0)
+        reader = csv.reader(csvfile, dialect)
+        for i, row in enumerate(reader):
+            try:
+                data.append(row[columnID].strip(string.punctuation))
+            except:
+                print("Error, line {}".format(i))
+    return data
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Converter to canonical form in Slovene language"
+    )
+    parser.add_argument("csv_file", type=argparse.FileType("r"), help="Input csv file")
+    parser.add_argument("column_id", type=int, help="CSV column number (zero indexed)")
+    args = parser.parse_args()
+
+    data = read_csv(args.csv_file.name, columnID=args.column_id)
+    results = process(data)
+    for canon in results:
+        print("{}".format(canon))

canonical_forms/entrypoint.prod.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# patching reldi-tokeniser for multi-threaded environment
+sed -i 's/signal(/pass;#signal(/g' /usr/local/lib/python3.8/site-packages/classla/submodules/reldi_tokeniser/tokeniser.py
+
+# pre-download models
+python -c "import classla;classla.download('sl', logging_level='INFO')"
+
+exec "$@"

canonical_forms/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# patching reldi-tokeniser for multi-threaded environment
+echo "Patching tokeniser.py..."
+sed -i 's/signal(/pass;#signal(/g' /usr/local/lib/python3.8/site-packages/classla/submodules/reldi_tokeniser/tokeniser.py
+
+# pre-download models
+python -c "import classla;classla.download('sl', logging_level='INFO')"
+
+exec "$@"

canonical_forms/requirements.txt

@@ -0,0 +1,8 @@
+Flask==2.1.*
+flask-restx
+gunicorn==20.1.0
+classla==1.0.1
+lemmagen3==3.3.1
+flask-socketio==5.1.0
+simple-websocket==0.2.0
+markupsafe==2.0.1  # to resolve issue with missing soft_unicode function

docker-compose.yml

@@ -0,0 +1,58 @@
+version: '3.1'
+services:
+  flask-serv:
+    build: .
+    restart: always
+    image: flask-server
+    ports:
+      - "8080:8080"
+    volumes: 
+      - ../classla/classla_resources:/root/classla_resources
+    environment:
+      - PYTHONUNBUFFERED=1
+      - MDB_DATABASE=conllus_150k
+      - MDB_HOST=mariadb
+      - MDB_PORT=3306
+      - MDB_USER=rsdo5
+      - MDB_PASSWORD=rsdo5mysql
+  
+  tika2:
+    image: apache/tika:1.28.4-full
+    entrypoint: [ "/bin/sh", "-c", "java -jar /tika-server-1.28.4.jar -h 0.0.0.0 -p 9999" ]
+    restart: always
+    ports:
+      - "9999:9999"
+  
+  ate-api:
+    build: ./ATEapi
+    restart: always
+    volumes:
+      - ../classla/classla_resources:/home/appuser/classla_resources
+  
+  canonizer:
+    container_name: canonizer_service
+    build:
+      context: ./canonical_forms/web
+      dockerfile: Dockerfile.prod
+    command: gunicorn --bind 0.0.0.0:5000 app:app
+    volumes:
+      - ../classla/classla_resources:/home/app/classla_resources
+    environment:
+      - FLASK_APP=app.py
+      - FLASK_ENV=production
+      - APP_FOLDER=/home/app/web
+
+  mariadb:
+    image: mariadb/columnstore
+    restart: always
+    container_name: mariadbcs
+    ports:
+      - "3306:3306"
+    volumes:
+      - mdbdata:/var/lib/mysql
+      - mdbcs-data:/var/lib/columnstore
+      - mdbcs-etc:/etc/columnstore
+volumes:
+    mdbdata:
+    mdbcs-data:
+    mdbcs-etc:

oss/conllus_150k_documents.sql

@@ -0,0 +1,43 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `documents`
+--
+
+DROP TABLE IF EXISTS `documents`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `documents` (
+  `document_id` int(11) DEFAULT NULL,
+  `ngramscount` int(11) DEFAULT NULL,
+  `maxtf` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  5:58:49

oss/conllus_150k_documents0.sql

@@ -0,0 +1,43 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `documents`
+--
+
+DROP TABLE IF EXISTS `documents`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `documents` (
+  `document_id` int(11) DEFAULT NULL,
+  `ngramscount` int(11) DEFAULT NULL,
+  `maxtf` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  6:00:02

oss/conllus_150k_metadata.sql

@@ -0,0 +1,47 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `metadata`
+--
+
+DROP TABLE IF EXISTS `metadata`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `metadata` (
+  `xml_id` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `udk` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci DEFAULT NULL,
+  `kljucnabeseda` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci DEFAULT NULL,
+  `vrsta` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci DEFAULT NULL,
+  `leto` int(11) DEFAULT NULL,
+  `tipologija` int(11) DEFAULT NULL,
+  `document_id` int(11) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  5:58:51

oss/conllus_150k_metadata0.sql

@@ -0,0 +1,47 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `metadata`
+--
+
+DROP TABLE IF EXISTS `metadata`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `metadata` (
+  `xml_id` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `udk` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci DEFAULT NULL,
+  `kljucnabeseda` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci DEFAULT NULL,
+  `vrsta` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci DEFAULT NULL,
+  `leto` int(11) DEFAULT NULL,
+  `tipologija` int(11) DEFAULT NULL,
+  `document_id` int(11) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  6:00:03

oss/conllus_150k_ngrams_upos_df.sql

@@ -0,0 +1,43 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `ngrams_upos_df`
+--
+
+DROP TABLE IF EXISTS `ngrams_upos_df`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `ngrams_upos_df` (
+  `ngram` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `df` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  5:58:50

oss/conllus_150k_ngrams_upos_df0.sql

@@ -0,0 +1,43 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `ngrams_upos_df`
+--
+
+DROP TABLE IF EXISTS `ngrams_upos_df`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `ngrams_upos_df` (
+  `ngram` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `df` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  6:00:03

oss/conllus_150k_ngrams_upos_tf.sql

@@ -0,0 +1,44 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `ngrams_upos_tf`
+--
+
+DROP TABLE IF EXISTS `ngrams_upos_tf`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `ngrams_upos_tf` (
+  `ngram` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `document_id` int(11) DEFAULT NULL,
+  `tf` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  5:58:51

oss/conllus_150k_ngrams_upos_tf0.sql

@@ -0,0 +1,44 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `ngrams_upos_tf`
+--
+
+DROP TABLE IF EXISTS `ngrams_upos_tf`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `ngrams_upos_tf` (
+  `ngram` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `document_id` int(11) DEFAULT NULL,
+  `tf` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  6:00:04

oss/conllus_150k_ngrams_upos_tf_filter.sql

@@ -0,0 +1,45 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `ngrams_upos_tf_filter`
+--
+
+DROP TABLE IF EXISTS `ngrams_upos_tf_filter`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `ngrams_upos_tf_filter` (
+  `ngram` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `lemma` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `document_id` int(11) DEFAULT NULL,
+  `tf` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  5:58:49

oss/conllus_150k_ngrams_upos_tf_filter0.sql

@@ -0,0 +1,45 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `ngrams_upos_tf_filter`
+--
+
+DROP TABLE IF EXISTS `ngrams_upos_tf_filter`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `ngrams_upos_tf_filter` (
+  `ngram` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `lemma` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `document_id` int(11) DEFAULT NULL,
+  `tf` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  6:00:01

oss/conllus_150k_os2022_ngrams.sql

@@ -0,0 +1,48 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `os2022_ngrams`
+--
+
+DROP TABLE IF EXISTS `os2022_ngrams`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `os2022_ngrams` (
+  `file_id` int(11) NOT NULL,
+  `sent_id` float(12,10) NOT NULL,
+  `ngram_len` int(11) NOT NULL,
+  `frequency_g_t` int(11) NOT NULL,
+  `gram_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `lemma_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `xpos_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  5:58:52

oss/conllus_150k_os2022_ngrams0.sql

@@ -0,0 +1,48 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `os2022_ngrams`
+--
+
+DROP TABLE IF EXISTS `os2022_ngrams`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `os2022_ngrams` (
+  `file_id` int(11) NOT NULL,
+  `sent_id` float(12,10) NOT NULL,
+  `ngram_len` int(11) NOT NULL,
+  `frequency_g_t` int(11) NOT NULL,
+  `gram_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `lemma_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `upos_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL,
+  `xpos_text` text CHARACTER SET utf8 COLLATE utf8_slovenian_ci NOT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  6:00:05

oss/conllus_150k_routines.sql

@@ -0,0 +1,33 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Dumping routines for database 'conllus_150k'
+--
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  5:58:52

oss/conllus_150k_routines0.sql

@@ -0,0 +1,33 @@
+CREATE DATABASE  IF NOT EXISTS `conllus_150k` /*!40100 DEFAULT CHARACTER SET latin1 */;
+USE `conllus_150k`;
+-- MySQL dump 10.13  Distrib 8.0.31, for Win64 (x86_64)
+--
+-- Host: 164.8.252.72    Database: conllus_150k
+-- ------------------------------------------------------
+-- Server version	5.5.5-10.5.4-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Dumping routines for database 'conllus_150k'
+--
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-12-07  6:00:05

requirements.txt

@@ -1,5 +1,18 @@
 connexion >= 2.6.0
 connexion[swagger-ui] >= 2.6.0
-python_dateutil == 2.6.0
+python_dateutil == 2.7.3
 setuptools >= 21.0.0
 swagger-ui-bundle >= 0.0.2
+peewee==3.15.0
+mariadb==1.0.11
+classla==1.1.0
+python-decouple==3.6
+pandas==1.3.3
+python-docx==0.8.11
+lxml==4.8.0
+PyPDF2==2.10.4
+requests
+pytesseract==0.3.10
+opencv-python==4.5.2.54
+numpy==1.20.3
+python-magic==0.4.27

swagger_server/.env.template

@@ -0,0 +1,5 @@
+MDB_DATABASE=
+MDB_HOST=
+MDB_PORT=
+MDB_USER=
+MDB_PASSWORD=

swagger_server/__main__.py

@@ -4,7 +4,6 @@ import connexion
 
 from swagger_server import encoder
 
-
 def main():
     app = connexion.App(__name__, specification_dir='./swagger/')
     app.app.json_encoder = encoder.JSONEncoder

swagger_server/controllers/doc2text_controller.py

@@ -1,30 +0,0 @@
-import connexion
-import six
-
-from swagger_server import util
-
-
-def datoteka_v_besedilo_post(file=None):  # noqa: E501
-    """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo
-
-     # noqa: E501
-
-    :param file: 
-    :type file: strstr
-
-    :rtype: str
-    """
-    return 'do some magic!'
-
-
-def get_text_ocr(file=None):  # noqa: E501
-    """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
-
-     # noqa: E501
-
-    :param file: 
-    :type file: strstr
-
-    :rtype: str
-    """
-    return 'do some magic!'

swagger_server/controllers/extract_controller.py

@@ -1,21 +1,88 @@
+import codecs
+import os
+
 import connexion
-import six
+import json
+from pathlib import Path
+from swagger_server.models.izlusci_async_body import IzlusciAsyncBody  # noqa: E501
+from swagger_server.models.izlusci_sync_body import IzlusciSyncBody  # noqa: E501
+from swagger_server.requets_db.models.vrsta import JobManager
+from swagger_server.utils import cl_utils
+from swagger_server.util import get_random_filename, create_random_file_in_tmp_folder
+import requests
+from werkzeug.utils import secure_filename
 
-from swagger_server.models.izlusci_body import IzlusciBody  # noqa: E501
+ATEapi_endpoint = "http://ate-api:5000/predict"
-from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat  # noqa: E501
+
-from swagger_server import util
+# endpoint below to be used only for development purposes (don't need to run docker)
+# ATEapi_endpoint = "http://localhost:5000/predict"
 
 
-def get_candidates(body):  # noqa: E501
+def do_izlusci(conllus, prepovedane_besede):
-    """Izlusci terminološke kandidate iz seznama besedil v conllu obliki
+    tmp_file_path = ""
+    try:
+        big_conllu = cl_utils.multipla_conllus_to_one_from_conllus_arr(conllus)
+        tmp_file_path = create_random_file_in_tmp_folder(big_conllu, ".conllu")
+        fp = open(tmp_file_path, 'rb')
+        try:
+            files = [
+                ('file', ('temp_1.conllu', fp, 'application/octet-stream'))
+            ]
+            res = requests.post(ATEapi_endpoint, files=files)
+            data = json.loads(res.text)
+        finally:
+            fp.close()
+            os.remove(tmp_file_path)
+
+        ret = {'terminoloski_kandidati': [
+            {
+                'POSoznake': tk['msd'],
+                'kandidat': tk['terms'],  # more to bit lemma al terms?
+                'kanonicnaoblika': tk['canonical'],
+                'ranking': tk['ranking'],
+                'podporneutezi': [
+                    0.0,  # ????????
+                    0.0  # ??????
+                ],
+                'pogostostpojavljanja': [0, 0]  # ???????
+            }
+            for tk in data if tk['terms'] not in prepovedane_besede
+        ]}
+        return ret, 200
+    except Exception as e:
+        return str(e), 500
+
+
+def get_candidates_async(body):  # noqa: E501
+    """Izlusci terminološke kandidate iz seznama besedil v conllu obliki [asinhrono, ustvari novi job]
 
      # noqa: E501
 
-    :param body: 
+    :param body:
+    :type body: dict | bytes
+
+    :rtype: str
+    """
+    if connexion.request.is_json:
+        body = IzlusciAsyncBody.from_dict(connexion.request.get_json())  # noqa: E501
+    job, is_old_job = JobManager.create_job(4, json.dumps(body.to_dict()))
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+    return ret, 200
+
+
+def get_candidates_sync(body):  # noqa: E501
+    """Izlusci terminološke kandidate iz seznama besedil v conllu obliki [sihrono, rezultat v sami zahtevi]
+
+     # noqa: E501
+
+    :param body:
     :type body: dict | bytes
 
     :rtype: List[TerminoloskiKandidat]
     """
     if connexion.request.is_json:
-        body = IzlusciBody.from_dict(connexion.request.get_json())  # noqa: E501
+        body = IzlusciSyncBody.from_dict(connexion.request.get_json())  # noqa: E501
-    return 'do some magic!'
+
+    return do_izlusci(body.conllus, body.prepovedane_besede)

swagger_server/controllers/jobs_controller.py

@@ -0,0 +1,233 @@
+import datetime
+import json
+import os.path
+import traceback
+
+import peewee
+import asyncio
+import concurrent.futures as cf
+
+from swagger_server.controllers.extract_controller import do_izlusci
+from swagger_server.models.job_response import JobResponse  # noqa: E501
+from swagger_server.requets_db.models.vrsta import (Job)
+from threading import Thread
+from swagger_server.utils import cl_utils
+from swagger_server.utils import txt_utils
+from werkzeug.datastructures import FileStorage
+import threading
+import time
+
+CLASSLA_CONCURANCE_LIMIT = 3
+DOC2TEXT_CONCURANCE_LIMIT = 3
+ATEAPI_CONCURANCE_LIMIT = 2
+
+classla_sem = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT)
+doc2text_sem = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT)
+ateapi_sem = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT)
+
+
+def delete_job(job_id):  # noqa: E501
+    """Izbriše job
+
+     # noqa: E501
+
+    :param job_id:
+    :type job_id: int
+
+    :rtype: str
+    """
+    return 'Endpoint currently disabled'
+
+
+def get_job_status(job_id):  # noqa: E501
+    """Vrne status
+
+     # noqa: E501
+
+    :param job_id:
+    :type job_id: int
+
+    :rtype: JobResponse
+    """
+    try:
+        job = Job.get_by_id(job_id)
+        if job.started_on is None:
+            return JobResponse(job_status="waiting in que", created_on=job.created_on), 200
+        if job.started_on is not None and job.finished_on is None:
+            return JobResponse(job_status="currently processing", created_on=job.created_on,
+                               started_on=job.started_on), 200
+        if job.started_on is not None and job.finished_on is not None:
+            return JobResponse(job_status="finished processing", created_on=job.created_on, started_on=job.started_on,
+                               finished_on=job.finished_on, job_result=job.job_output), 200
+    except peewee.DoesNotExist:
+        return "Job with this ID does not exist", 404
+
+
+def clear_up_unfinished_jobs():
+    """
+    In case server crashed while jobs were in queue...
+    """
+    Job.update(started_on=None).where(Job.started_on.is_null(False), Job.finished_on.is_null()).execute()
+    if os.path.exists('tmp'):
+        for tmp_file in os.listdir('tmp'):
+            if not Job.select().where(Job.input_file == tmp_file).exists():
+                os.remove(f'tmp/{tmp_file}')
+
+
+async def try_do_jobs():
+    with cf.ThreadPoolExecutor(max_workers=3) as ex:
+        ex.submit(try_do_jobs_classla)
+        ex.submit(try_do_jobs_doc2text)
+        ex.submit(try_do_jobs_ateapi)
+
+
+### Job looping
+def try_do_jobs_ateapi():
+    while True:
+        try:
+            if ateapi_sem._value > 0:
+                unfinished_jobs = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 4) \
+                    .limit(ateapi_sem._value)
+                with cf.ThreadPoolExecutor(max_workers=ATEAPI_CONCURANCE_LIMIT) as ex:
+                    [ex.submit(execute_ateapi_job, job) for job in unfinished_jobs]
+        except Exception as e:
+            print(f"Exception in try_do_jobs_ateapi")
+            traceback.print_exc()
+        finally:
+            time.sleep(3)
+
+
+### Job looping
+def try_do_jobs_classla():
+    time.sleep(15)  # wait for tokenizers to load for classla ...
+    while True:
+        try:
+            if cl_utils.nlp_loaded:
+                if classla_sem._value > 0:
+                    unfinished_jobs_txt = Job.select() \
+                        .where(Job.finished_on.is_null(), Job.job_type == 2,
+                               Job.input_file.is_null(False)) \
+                        .limit(classla_sem._value)
+
+                    unfinished_jobs_no_txt = Job.select() \
+                        .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
+                               Job.input_file.is_null()) \
+                        .limit(classla_sem._value)
+
+                    unfinished_jobs = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
+                    unfinished_jobs = unfinished_jobs[:classla_sem._value]
+                    with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT) as ex:
+                        [ex.submit(execute_classla_job, job) for job in unfinished_jobs]
+
+        except Exception as e:
+            print(f"Exception in try_do_jobs_classla")
+            traceback.print_exc()
+        finally:
+            time.sleep(3)
+
+
+### Job looping
+def try_do_jobs_doc2text():
+    while True:
+        try:
+            if doc2text_sem._value > 0:
+                unfinished_jobs = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type << [1, 12, 3, 32]) \
+                    .limit(doc2text_sem._value)
+                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT) as ex:
+                    [ex.submit(execute_doc2text_job, job) for job in unfinished_jobs]
+        except Exception as e:
+            print(f"Exception in try_do_jobs_doc2text")
+            traceback.print_exc()
+        finally:
+            time.sleep(3)
+
+
+async def prep_jobs(tasks):
+    await asyncio.gather(*tasks)
+
+
+def execute_doc2text_job(job: Job):
+    try:
+        doc2text_sem.acquire()
+        del_file = False
+        job.started_on = datetime.datetime.utcnow()
+        job.save()
+
+        tmp_file_path = job.input_file
+        if not os.path.exists(tmp_file_path):
+            job.finished_on = datetime.datetime.utcnow()
+            job.job_output = "ERROR - Temporary file went missing, couldn't properly finish job"
+            job.save()
+            return
+
+        with open(tmp_file_path, 'rb+') as f:
+            file = FileStorage(f)
+            jtype = job.job_type
+            text = ""
+            if jtype in [1, 12]:
+                text, _ = txt_utils.extract_text_prepResp(file)
+            elif jtype in [3, 32]:
+                text, _ = txt_utils.ocr_text_prepResp(file)
+
+            if jtype in [1, 3]:
+                job.job_output = text
+                job.finished_on = datetime.datetime.utcnow()
+            elif jtype in [12, 32]:
+                job.job_input = text
+                job.job_type = 2
+                job.input_size = len(text)
+                del_file = True
+            job.save()
+        if del_file:
+            try:
+                os.remove(tmp_file_path)
+            except:
+                pass
+
+    except:
+        job.started_on = None
+        job.save()
+    finally:
+        doc2text_sem.release()
+
+
+def execute_classla_job(job: Job):
+    try:
+        classla_sem.acquire()
+        job.started_on = datetime.datetime.utcnow()
+        job.save()
+        conllu, _ = cl_utils.raw_text_to_conllu(job.job_input)
+        job.job_output = conllu
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+    finally:
+        classla_sem.release()
+
+
+def execute_ateapi_job(job: Job):
+    try:
+        ateapi_sem.acquire()
+        job.started_on = datetime.datetime.utcnow()
+        job.save()
+        info = json.loads(job.job_input)
+        ret_json, _ = do_izlusci(info['conllus'], info['prepovedane_besede'])
+        job.job_output = ret_json
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+    finally:
+        ateapi_sem.release()
+
+
+clear_up_unfinished_jobs()
+loop = asyncio.get_event_loop()
+
+
+def loop_in_thread(loop):
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(try_do_jobs())
+
+
+t = Thread(target=loop_in_thread, args=(loop,))
+t.start()

swagger_server/controllers/marktext_async_controller.py

@@ -0,0 +1,96 @@
+import connexion
+
+from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody  # noqa: E501
+from swagger_server.requets_db.models.vrsta import (JobManager)
+
+
+def get_text(body):  # noqa: E501
+    """Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
+
+     # noqa: E501
+
+    :param body:
+    :type body: dict | bytes
+
+    :rtype: str
+    """
+    if connexion.request.is_json:
+        body = OznaciBesediloAsyncBody.from_dict(connexion.request.get_json())  # noqa: E501
+    else:
+        return "Request in wrong format", 400
+    # conllu = cl_utils.raw_text_to_conllu(body.besedilo)
+    # return conllu
+    job, is_old_job = JobManager.create_job(2, body.besedilo)
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+
+    return ret, 200  # Todo: Update swagger to the newest response template later
+
+
+def get_conllu_from_file_async(file=None):  # noqa: E501
+    """Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    job, is_old_job = JobManager.create_job(12, file)
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+    return ret, 200
+
+
+def get_conllu_from_file_ocr_async(file=None):  # noqa: E501
+    """Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    job, is_old_job = JobManager.create_job(32, file)
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+    return ret, 200
+
+
+def get_text_from_doc_async(file=None):  # noqa: E501
+    """Pretvori datoteko v besedilo, vrača tekst
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    job, is_old_job = JobManager.create_job(1, file)
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+    return ret, 200
+
+
+def get_text_from_file_ocr_async(file=None):  # noqa: E501
+    """Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    job, is_old_job = JobManager.create_job(3, file)
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+    return ret, 200

swagger_server/controllers/marktext_controller.py

@@ -1,19 +0,0 @@
-import connexion
-import six
-
-from swagger_server import util
-
-
-def get_text(body):  # noqa: E501
-    """Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
-
-     # noqa: E501
-
-    :param body: 
-    :type body: dict | bytes
-
-    :rtype: str
-    """
-    if connexion.request.is_json:
-        body = str.from_dict(connexion.request.get_json())  # noqa: E501
-    return 'do some magic!'

swagger_server/controllers/marktext_sync_controller.py

@@ -0,0 +1,81 @@
+from swagger_server.utils import cl_utils
+from swagger_server.utils import txt_utils
+
+
+def datoteka_v_besedilo_in_classla(file=None):  # noqa: E501
+    """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    if not cl_utils.nlp_loaded:
+        return "NLP Models still loading up since server restart, please try again later.", 500
+    if file is None:
+        return "No file provided", 400
+    try:
+        txt, _ = txt_utils.extract_text_prepResp(file)
+        return cl_utils.raw_text_to_conllu(txt)
+    except Exception as e:
+        return str(e), 500
+
+
+def datoteka_v_besedilo_sync_post(file=None):  # noqa: E501
+    """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    if file is None:
+        return "No file provided", 400
+    try:
+        return txt_utils.extract_text_prepResp(file)
+    except Exception as e:
+        print(e)
+        return str(e), 500
+
+
+def get_conllu_ocr(file=None):  # noqa: E501
+    """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    if not cl_utils.nlp_loaded:
+        return "NLP Models still loading up since server restart, please try again later.", 500
+    if file is None:
+        return "No file provided", 400
+    try:
+        txt, _ = txt_utils.ocr_text_prepResp(file)
+        return cl_utils.raw_text_to_conllu(txt)
+    except Exception as e:
+        return str(e), 500
+
+
+def get_text_ocr(file=None):  # noqa: E501
+    """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
+
+     # noqa: E501
+
+    :param file:
+    :type file: strstr
+
+    :rtype: str
+    """
+    if file is None:
+        return "No file provided", 400
+    try:
+        return txt_utils.ocr_text_prepResp(file)
+    except Exception as e:
+        return str(e), 500

swagger_server/controllers/oss_controller.py

@@ -1,21 +1,6 @@
-import connexion
+from swagger_server.utils import db_utils
-import six
-
-from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat  # noqa: E501
 from swagger_server import util
-
+from flask import send_file
-
-def get_conllu(id):  # noqa: E501
-    """Vrne CoNNL-U po id-ju datoteke
-
-     # noqa: E501
-
-    :param id: 
-    :type id: int
-
-    :rtype: str
-    """
-    return 'do some magic!'
 
 
 def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
@@ -34,10 +19,16 @@ def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
 
     :rtype: List[str]
     """
-    return 'do some magic!'
+    if not kljucnebesede:
+        return "Manjkajo kljucne besede", 400
+    #zaenkrat ne potrebujemo te storitve
+    files = db_utils.get_files_by_udc(kljucnebesede)
+    if not files:
+        return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
+    return ' '.join(files), 200
 
 
-def get_extracted_words(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
+def get_extracted_words(leta=None, vrste=None, kljucnebesede=None, udk=None):  # noqa: E501
     """Vrne terminloške kandidate glede na 
 
      # noqa: E501
@@ -53,21 +44,10 @@ def get_extracted_words(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E50
 
     :rtype: List[TerminoloskiKandidat]
     """
-    return 'do some magic!'
+    files = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk)
+    return files, 200
 
 
-def get_file(id):  # noqa: E501
-    """Vrne binarni zapis v originalnem formatu po id-ju datoteke
-
-     # noqa: E501
-
-    :param id: 
-    :type id: int
-
-    :rtype: List[bytearray]
-    """
-    return 'do some magic!'
-
 
 def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
     """Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje
@@ -85,10 +65,16 @@ def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
 
     :rtype: List[List[bytearray]]
     """
-    return 'do some magic!'
+    if not kljucnebesede:
+        return "Manjkajo kljucne besede", 400
+    #zaenkrat ne potrebujemo te storitve
+    files = db_utils.get_files_by_udc(kljucnebesede)
+    if not files:
+        return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
+    return ' '.join(files), 200
 
 
-def get_number_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
+def get_number_texts(leta=None, vrste=None, kljucnebesede=None, udk=None):  # noqa: E501
     """Vrne število besedil glede na iskalne pogoje
 
      # noqa: E501
@@ -100,11 +86,15 @@ def get_number_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
     :param kljucnebesede: 
     :type kljucnebesede: List[str]
     :param cerifpodrocja: 
-    :type cerifpodrocja: List[int]
+    :type udc: List[int]
 
     :rtype: int
     """
-    return 'do some magic!'
+    #if not kljucnebesede:
+    #    return "Manjkajo kljucne besede", 400
+
+    files = db_utils.vrni_oss_dokumente(leta, vrste, kljucnebesede, udk)
+    return len(files), 200
 
 
 def get_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
@@ -123,17 +113,60 @@ def get_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
 
     :rtype: List[str]
     """
-    return 'do some magic!'
+    if not kljucnebesede:
+        return "Manjkajo kljucne besede", 400
+    #zaenkrat ne potrebujemo te storitve
+    files = db_utils.get_files_by_udc(kljucnebesede)
+    if not files:
+        return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
+    return ' '.join(files), 200
 
 
-def oss_besedilo_po_id_get(id):  # noqa: E501
+def get_conllu(file_id):  # noqa: E501
+    """Vrne CoNNL-U po id-ju datoteke
+
+     # noqa: E501
+
+    :param file_id:
+    :type file_id: int
+
+    :rtype: str
+    """
+    try:
+        return send_file(util.get_conllu_file_path_by_id(file_id), download_name=f'{file_id}.conllu')
+    except FileNotFoundError as e:
+        return "The conllu with this ID doesn't exist.", 404
+
+
+def get_file(file_id):  # noqa: E501
+    """Vrne binarni zapis v originalnem formatu po id-ju datoteke
+
+     # noqa: E501
+
+    :param file_id:
+    :type file_id: int
+
+    :rtype: List[bytearray]
+    """
+    try:
+        return send_file(util.get_original_file_path_by_id(file_id), download_name=f'{file_id}.xml')
+    except FileNotFoundError as e:
+        return "The file with this ID doesn't exist.", 404
+
+
+def oss_besedilo_po_id_get(file_id):  # noqa: E501
     """Vrne besedilo po id-ju datoteke
 
      # noqa: E501
 
-    :param id: 
+    :param file_id:
-    :type id: int
+    :type file_id: int
 
     :rtype: str
     """
-    return 'do some magic!'
+    try:
+        f = util.get_original_file_path_by_id(file_id)
+        print(f) # for debugging purposes on the server, delete this later
+        return send_file(util.get_original_file_path_by_id(file_id), download_name=f'{file_id}.xml')
+    except FileNotFoundError as e:
+        return "The file with this ID doesn't exist.", 404

swagger_server/models/__init__.py

@@ -3,7 +3,16 @@
 # flake8: noqa
 from __future__ import absolute_import
 # import models into model package
-from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody
+from swagger_server.models.datoteka_v_besedilo_async_body import DatotekaVBesediloAsyncBody
-from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody
+from swagger_server.models.datoteka_v_besedilo_async_ocr_body import DatotekaVBesediloAsyncOcrBody
-from swagger_server.models.izlusci_body import IzlusciBody
+from swagger_server.models.datoteka_v_besedilo_sync_body import DatotekaVBesediloSyncBody
+from swagger_server.models.datoteka_v_besedilo_sync_ocr_body import DatotekaVBesediloSyncOcrBody
+from swagger_server.models.datoteka_v_conllu_async_body import DatotekaVConlluAsyncBody
+from swagger_server.models.datoteka_v_conllu_async_ocr_body import DatotekaVConlluAsyncOcrBody
+from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody
+from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody
+from swagger_server.models.job_response import JobResponse
+from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody
 from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
+from swagger_server.models.izlusci_async_body import IzlusciAsyncBody
+from swagger_server.models.izlusci_sync_body import IzlusciSyncBody

swagger_server/models/datoteka_v_besedilo_async_body.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class DatotekaVBesediloAsyncBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, file: str=None):  # noqa: E501
+        """DatotekaVBesediloAsyncBody - a model defined in Swagger
+
+        :param file: The file of this DatotekaVBesediloAsyncBody.  # noqa: E501
+        :type file: str
+        """
+        self.swagger_types = {
+            'file': str
+        }
+
+        self.attribute_map = {
+            'file': 'file'
+        }
+        self._file = file
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'DatotekaVBesediloAsyncBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The datotekaVBesediloAsync_body of this DatotekaVBesediloAsyncBody.  # noqa: E501
+        :rtype: DatotekaVBesediloAsyncBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def file(self) -> str:
+        """Gets the file of this DatotekaVBesediloAsyncBody.
+
+
+        :return: The file of this DatotekaVBesediloAsyncBody.
+        :rtype: str
+        """
+        return self._file
+
+    @file.setter
+    def file(self, file: str):
+        """Sets the file of this DatotekaVBesediloAsyncBody.
+
+
+        :param file: The file of this DatotekaVBesediloAsyncBody.
+        :type file: str
+        """
+        if file is None:
+            raise ValueError("Invalid value for `file`, must not be `None`")  # noqa: E501
+
+        self._file = file

swagger_server/models/datoteka_v_besedilo_async_ocr_body.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class DatotekaVBesediloAsyncOcrBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, file: str=None):  # noqa: E501
+        """DatotekaVBesediloAsyncOcrBody - a model defined in Swagger
+
+        :param file: The file of this DatotekaVBesediloAsyncOcrBody.  # noqa: E501
+        :type file: str
+        """
+        self.swagger_types = {
+            'file': str
+        }
+
+        self.attribute_map = {
+            'file': 'file'
+        }
+        self._file = file
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'DatotekaVBesediloAsyncOcrBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The datotekaVBesediloAsync_ocr_body of this DatotekaVBesediloAsyncOcrBody.  # noqa: E501
+        :rtype: DatotekaVBesediloAsyncOcrBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def file(self) -> str:
+        """Gets the file of this DatotekaVBesediloAsyncOcrBody.
+
+
+        :return: The file of this DatotekaVBesediloAsyncOcrBody.
+        :rtype: str
+        """
+        return self._file
+
+    @file.setter
+    def file(self, file: str):
+        """Sets the file of this DatotekaVBesediloAsyncOcrBody.
+
+
+        :param file: The file of this DatotekaVBesediloAsyncOcrBody.
+        :type file: str
+        """
+        if file is None:
+            raise ValueError("Invalid value for `file`, must not be `None`")  # noqa: E501
+
+        self._file = file

swagger_server/models/datoteka_v_besedilo_sync_body.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class DatotekaVBesediloSyncBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, file: str=None):  # noqa: E501
+        """DatotekaVBesediloSyncBody - a model defined in Swagger
+
+        :param file: The file of this DatotekaVBesediloSyncBody.  # noqa: E501
+        :type file: str
+        """
+        self.swagger_types = {
+            'file': str
+        }
+
+        self.attribute_map = {
+            'file': 'file'
+        }
+        self._file = file
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'DatotekaVBesediloSyncBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The datotekaVBesediloSync_body of this DatotekaVBesediloSyncBody.  # noqa: E501
+        :rtype: DatotekaVBesediloSyncBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def file(self) -> str:
+        """Gets the file of this DatotekaVBesediloSyncBody.
+
+
+        :return: The file of this DatotekaVBesediloSyncBody.
+        :rtype: str
+        """
+        return self._file
+
+    @file.setter
+    def file(self, file: str):
+        """Sets the file of this DatotekaVBesediloSyncBody.
+
+
+        :param file: The file of this DatotekaVBesediloSyncBody.
+        :type file: str
+        """
+        if file is None:
+            raise ValueError("Invalid value for `file`, must not be `None`")  # noqa: E501
+
+        self._file = file

swagger_server/models/datoteka_v_besedilo_sync_ocr_body.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class DatotekaVBesediloSyncOcrBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, file: str=None):  # noqa: E501
+        """DatotekaVBesediloSyncOcrBody - a model defined in Swagger
+
+        :param file: The file of this DatotekaVBesediloSyncOcrBody.  # noqa: E501
+        :type file: str
+        """
+        self.swagger_types = {
+            'file': str
+        }
+
+        self.attribute_map = {
+            'file': 'file'
+        }
+        self._file = file
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'DatotekaVBesediloSyncOcrBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The datotekaVBesediloSync_ocr_body of this DatotekaVBesediloSyncOcrBody.  # noqa: E501
+        :rtype: DatotekaVBesediloSyncOcrBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def file(self) -> str:
+        """Gets the file of this DatotekaVBesediloSyncOcrBody.
+
+
+        :return: The file of this DatotekaVBesediloSyncOcrBody.
+        :rtype: str
+        """
+        return self._file
+
+    @file.setter
+    def file(self, file: str):
+        """Sets the file of this DatotekaVBesediloSyncOcrBody.
+
+
+        :param file: The file of this DatotekaVBesediloSyncOcrBody.
+        :type file: str
+        """
+        if file is None:
+            raise ValueError("Invalid value for `file`, must not be `None`")  # noqa: E501
+
+        self._file = file

swagger_server/models/datoteka_v_conllu_async_body.py

@@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model
 from swagger_server import util
 
 
-class DatotekaVBesediloOcrBody(Model):
+class DatotekaVConlluAsyncBody(Model):
     """NOTE: This class is auto generated by the swagger code generator program.
 
     Do not edit the class manually.
     """
     def __init__(self, file: str=None):  # noqa: E501
-        """DatotekaVBesediloOcrBody - a model defined in Swagger
+        """DatotekaVConlluAsyncBody - a model defined in Swagger
 
-        :param file: The file of this DatotekaVBesediloOcrBody.  # noqa: E501
+        :param file: The file of this DatotekaVConlluAsyncBody.  # noqa: E501
         :type file: str
         """
         self.swagger_types = {
@@ -30,32 +30,32 @@ class DatotekaVBesediloOcrBody(Model):
         self._file = file
 
     @classmethod
-    def from_dict(cls, dikt) -> 'DatotekaVBesediloOcrBody':
+    def from_dict(cls, dikt) -> 'DatotekaVConlluAsyncBody':
         """Returns the dict as a model
 
         :param dikt: A dict.
         :type: dict
-        :return: The datotekaVBesedilo_ocr_body of this DatotekaVBesediloOcrBody.  # noqa: E501
+        :return: The datotekaVConlluAsync_body of this DatotekaVConlluAsyncBody.  # noqa: E501
-        :rtype: DatotekaVBesediloOcrBody
+        :rtype: DatotekaVConlluAsyncBody
         """
         return util.deserialize_model(dikt, cls)
 
     @property
     def file(self) -> str:
-        """Gets the file of this DatotekaVBesediloOcrBody.
+        """Gets the file of this DatotekaVConlluAsyncBody.
 
 
-        :return: The file of this DatotekaVBesediloOcrBody.
+        :return: The file of this DatotekaVConlluAsyncBody.
         :rtype: str
         """
         return self._file
 
     @file.setter
     def file(self, file: str):
-        """Sets the file of this DatotekaVBesediloOcrBody.
+        """Sets the file of this DatotekaVConlluAsyncBody.
 
 
-        :param file: The file of this DatotekaVBesediloOcrBody.
+        :param file: The file of this DatotekaVConlluAsyncBody.
         :type file: str
         """
         if file is None:

swagger_server/models/datoteka_v_conllu_async_ocr_body.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class DatotekaVConlluAsyncOcrBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, file: str=None):  # noqa: E501
+        """DatotekaVConlluAsyncOcrBody - a model defined in Swagger
+
+        :param file: The file of this DatotekaVConlluAsyncOcrBody.  # noqa: E501
+        :type file: str
+        """
+        self.swagger_types = {
+            'file': str
+        }
+
+        self.attribute_map = {
+            'file': 'file'
+        }
+        self._file = file
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'DatotekaVConlluAsyncOcrBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The datotekaVConlluAsync_ocr_body of this DatotekaVConlluAsyncOcrBody.  # noqa: E501
+        :rtype: DatotekaVConlluAsyncOcrBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def file(self) -> str:
+        """Gets the file of this DatotekaVConlluAsyncOcrBody.
+
+
+        :return: The file of this DatotekaVConlluAsyncOcrBody.
+        :rtype: str
+        """
+        return self._file
+
+    @file.setter
+    def file(self, file: str):
+        """Sets the file of this DatotekaVConlluAsyncOcrBody.
+
+
+        :param file: The file of this DatotekaVConlluAsyncOcrBody.
+        :type file: str
+        """
+        if file is None:
+            raise ValueError("Invalid value for `file`, must not be `None`")  # noqa: E501
+
+        self._file = file

swagger_server/models/datoteka_v_conllu_sync_body.py

@@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model
 from swagger_server import util
 
 
-class DatotekaVBesediloBody(Model):
+class DatotekaVConlluSyncBody(Model):
     """NOTE: This class is auto generated by the swagger code generator program.
 
     Do not edit the class manually.
     """
     def __init__(self, file: str=None):  # noqa: E501
-        """DatotekaVBesediloBody - a model defined in Swagger
+        """DatotekaVConlluSyncBody - a model defined in Swagger
 
-        :param file: The file of this DatotekaVBesediloBody.  # noqa: E501
+        :param file: The file of this DatotekaVConlluSyncBody.  # noqa: E501
         :type file: str
         """
         self.swagger_types = {
@@ -30,32 +30,32 @@ class DatotekaVBesediloBody(Model):
         self._file = file
 
     @classmethod
-    def from_dict(cls, dikt) -> 'DatotekaVBesediloBody':
+    def from_dict(cls, dikt) -> 'DatotekaVConlluSyncBody':
         """Returns the dict as a model
 
         :param dikt: A dict.
         :type: dict
-        :return: The datotekaVBesedilo_body of this DatotekaVBesediloBody.  # noqa: E501
+        :return: The datotekaVConlluSync_body of this DatotekaVConlluSyncBody.  # noqa: E501
-        :rtype: DatotekaVBesediloBody
+        :rtype: DatotekaVConlluSyncBody
         """
         return util.deserialize_model(dikt, cls)
 
     @property
     def file(self) -> str:
-        """Gets the file of this DatotekaVBesediloBody.
+        """Gets the file of this DatotekaVConlluSyncBody.
 
 
-        :return: The file of this DatotekaVBesediloBody.
+        :return: The file of this DatotekaVConlluSyncBody.
         :rtype: str
         """
         return self._file
 
     @file.setter
     def file(self, file: str):
-        """Sets the file of this DatotekaVBesediloBody.
+        """Sets the file of this DatotekaVConlluSyncBody.
 
 
-        :param file: The file of this DatotekaVBesediloBody.
+        :param file: The file of this DatotekaVConlluSyncBody.
         :type file: str
         """
         if file is None:

swagger_server/models/datoteka_v_conllu_sync_ocr_body.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class DatotekaVConlluSyncOcrBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, file: str=None):  # noqa: E501
+        """DatotekaVConlluSyncOcrBody - a model defined in Swagger
+
+        :param file: The file of this DatotekaVConlluSyncOcrBody.  # noqa: E501
+        :type file: str
+        """
+        self.swagger_types = {
+            'file': str
+        }
+
+        self.attribute_map = {
+            'file': 'file'
+        }
+        self._file = file
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'DatotekaVConlluSyncOcrBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The datotekaVConlluSync_ocr_body of this DatotekaVConlluSyncOcrBody.  # noqa: E501
+        :rtype: DatotekaVConlluSyncOcrBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def file(self) -> str:
+        """Gets the file of this DatotekaVConlluSyncOcrBody.
+
+
+        :return: The file of this DatotekaVConlluSyncOcrBody.
+        :rtype: str
+        """
+        return self._file
+
+    @file.setter
+    def file(self, file: str):
+        """Sets the file of this DatotekaVConlluSyncOcrBody.
+
+
+        :param file: The file of this DatotekaVConlluSyncOcrBody.
+        :type file: str
+        """
+        if file is None:
+            raise ValueError("Invalid value for `file`, must not be `None`")  # noqa: E501
+
+        self._file = file

swagger_server/models/izlusci_async_body.py

@@ -0,0 +1,88 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class IzlusciAsyncBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
+        """IzlusciAsyncBody - a model defined in Swagger
+
+        :param conllus: The conllus of this IzlusciAsyncBody.  # noqa: E501
+        :type conllus: List[str]
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody.  # noqa: E501
+        :type prepovedane_besede: List[str]
+        """
+        self.swagger_types = {
+            'conllus': List[str],
+            'prepovedane_besede': List[str]
+        }
+
+        self.attribute_map = {
+            'conllus': 'conllus',
+            'prepovedane_besede': 'prepovedaneBesede'
+        }
+        self._conllus = conllus
+        self._prepovedane_besede = prepovedane_besede
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'IzlusciAsyncBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The izlusciAsync_body of this IzlusciAsyncBody.  # noqa: E501
+        :rtype: IzlusciAsyncBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def conllus(self) -> List[str]:
+        """Gets the conllus of this IzlusciAsyncBody.
+
+
+        :return: The conllus of this IzlusciAsyncBody.
+        :rtype: List[str]
+        """
+        return self._conllus
+
+    @conllus.setter
+    def conllus(self, conllus: List[str]):
+        """Sets the conllus of this IzlusciAsyncBody.
+
+
+        :param conllus: The conllus of this IzlusciAsyncBody.
+        :type conllus: List[str]
+        """
+
+        self._conllus = conllus
+
+    @property
+    def prepovedane_besede(self) -> List[str]:
+        """Gets the prepovedane_besede of this IzlusciAsyncBody.
+
+
+        :return: The prepovedane_besede of this IzlusciAsyncBody.
+        :rtype: List[str]
+        """
+        return self._prepovedane_besede
+
+    @prepovedane_besede.setter
+    def prepovedane_besede(self, prepovedane_besede: List[str]):
+        """Sets the prepovedane_besede of this IzlusciAsyncBody.
+
+
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciAsyncBody.
+        :type prepovedane_besede: List[str]
+        """
+
+        self._prepovedane_besede = prepovedane_besede

swagger_server/models/izlusci_sync_body.py

@@ -9,17 +9,17 @@ from swagger_server.models.base_model_ import Model
 from swagger_server import util
 
 
-class IzlusciBody(Model):
+class IzlusciSyncBody(Model):
     """NOTE: This class is auto generated by the swagger code generator program.
 
     Do not edit the class manually.
     """
     def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
-        """IzlusciBody - a model defined in Swagger
+        """IzlusciSyncBody - a model defined in Swagger
 
-        :param conllus: The conllus of this IzlusciBody.  # noqa: E501
+        :param conllus: The conllus of this IzlusciSyncBody.  # noqa: E501
         :type conllus: List[str]
-        :param prepovedane_besede: The prepovedane_besede of this IzlusciBody.  # noqa: E501
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody.  # noqa: E501
         :type prepovedane_besede: List[str]
         """
         self.swagger_types = {
@@ -35,32 +35,32 @@ class IzlusciBody(Model):
         self._prepovedane_besede = prepovedane_besede
 
     @classmethod
-    def from_dict(cls, dikt) -> 'IzlusciBody':
+    def from_dict(cls, dikt) -> 'IzlusciSyncBody':
         """Returns the dict as a model
 
         :param dikt: A dict.
         :type: dict
-        :return: The izlusci_body of this IzlusciBody.  # noqa: E501
+        :return: The izlusciSync_body of this IzlusciSyncBody.  # noqa: E501
-        :rtype: IzlusciBody
+        :rtype: IzlusciSyncBody
         """
         return util.deserialize_model(dikt, cls)
 
     @property
     def conllus(self) -> List[str]:
-        """Gets the conllus of this IzlusciBody.
+        """Gets the conllus of this IzlusciSyncBody.
 
 
-        :return: The conllus of this IzlusciBody.
+        :return: The conllus of this IzlusciSyncBody.
         :rtype: List[str]
         """
         return self._conllus
 
     @conllus.setter
     def conllus(self, conllus: List[str]):
-        """Sets the conllus of this IzlusciBody.
+        """Sets the conllus of this IzlusciSyncBody.
 
 
-        :param conllus: The conllus of this IzlusciBody.
+        :param conllus: The conllus of this IzlusciSyncBody.
         :type conllus: List[str]
         """
 
@@ -68,20 +68,20 @@ class IzlusciBody(Model):
 
     @property
     def prepovedane_besede(self) -> List[str]:
-        """Gets the prepovedane_besede of this IzlusciBody.
+        """Gets the prepovedane_besede of this IzlusciSyncBody.
 
 
-        :return: The prepovedane_besede of this IzlusciBody.
+        :return: The prepovedane_besede of this IzlusciSyncBody.
         :rtype: List[str]
         """
         return self._prepovedane_besede
 
     @prepovedane_besede.setter
     def prepovedane_besede(self, prepovedane_besede: List[str]):
-        """Sets the prepovedane_besede of this IzlusciBody.
+        """Sets the prepovedane_besede of this IzlusciSyncBody.
 
 
-        :param prepovedane_besede: The prepovedane_besede of this IzlusciBody.
+        :param prepovedane_besede: The prepovedane_besede of this IzlusciSyncBody.
         :type prepovedane_besede: List[str]
         """
 

swagger_server/models/job_response.py

@@ -0,0 +1,172 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class JobResponse(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, job_status: str=None, finished_on: datetime=None, started_on: datetime=None, created_on: datetime=None, job_result: str=None):  # noqa: E501
+        """JobResponse - a model defined in Swagger
+
+        :param job_status: The job_status of this JobResponse.  # noqa: E501
+        :type job_status: str
+        :param finished_on: The finished_on of this JobResponse.  # noqa: E501
+        :type finished_on: datetime
+        :param started_on: The started_on of this JobResponse.  # noqa: E501
+        :type started_on: datetime
+        :param created_on: The created_on of this JobResponse.  # noqa: E501
+        :type created_on: datetime
+        :param job_result: The job_result of this JobResponse.  # noqa: E501
+        :type job_result: str
+        """
+        self.swagger_types = {
+            'job_status': str,
+            'finished_on': datetime,
+            'started_on': datetime,
+            'created_on': datetime,
+            'job_result': str
+        }
+
+        self.attribute_map = {
+            'job_status': 'job_status',
+            'finished_on': 'finished_on',
+            'started_on': 'started_on',
+            'created_on': 'created_on',
+            'job_result': 'job_result'
+        }
+        self._job_status = job_status
+        self._finished_on = finished_on
+        self._started_on = started_on
+        self._created_on = created_on
+        self._job_result = job_result
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'JobResponse':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The JobResponse of this JobResponse.  # noqa: E501
+        :rtype: JobResponse
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def job_status(self) -> str:
+        """Gets the job_status of this JobResponse.
+
+
+        :return: The job_status of this JobResponse.
+        :rtype: str
+        """
+        return self._job_status
+
+    @job_status.setter
+    def job_status(self, job_status: str):
+        """Sets the job_status of this JobResponse.
+
+
+        :param job_status: The job_status of this JobResponse.
+        :type job_status: str
+        """
+        allowed_values = ["waiting in que", "currently processing", "finished processing"]  # noqa: E501
+        if job_status not in allowed_values:
+            raise ValueError(
+                "Invalid value for `job_status` ({0}), must be one of {1}"
+                .format(job_status, allowed_values)
+            )
+
+        self._job_status = job_status
+
+    @property
+    def finished_on(self) -> datetime:
+        """Gets the finished_on of this JobResponse.
+
+
+        :return: The finished_on of this JobResponse.
+        :rtype: datetime
+        """
+        return self._finished_on
+
+    @finished_on.setter
+    def finished_on(self, finished_on: datetime):
+        """Sets the finished_on of this JobResponse.
+
+
+        :param finished_on: The finished_on of this JobResponse.
+        :type finished_on: datetime
+        """
+
+        self._finished_on = finished_on
+
+    @property
+    def started_on(self) -> datetime:
+        """Gets the started_on of this JobResponse.
+
+
+        :return: The started_on of this JobResponse.
+        :rtype: datetime
+        """
+        return self._started_on
+
+    @started_on.setter
+    def started_on(self, started_on: datetime):
+        """Sets the started_on of this JobResponse.
+
+
+        :param started_on: The started_on of this JobResponse.
+        :type started_on: datetime
+        """
+
+        self._started_on = started_on
+
+    @property
+    def created_on(self) -> datetime:
+        """Gets the created_on of this JobResponse.
+
+
+        :return: The created_on of this JobResponse.
+        :rtype: datetime
+        """
+        return self._created_on
+
+    @created_on.setter
+    def created_on(self, created_on: datetime):
+        """Sets the created_on of this JobResponse.
+
+
+        :param created_on: The created_on of this JobResponse.
+        :type created_on: datetime
+        """
+
+        self._created_on = created_on
+
+    @property
+    def job_result(self) -> str:
+        """Gets the job_result of this JobResponse.
+
+
+        :return: The job_result of this JobResponse.
+        :rtype: str
+        """
+        return self._job_result
+
+    @job_result.setter
+    def job_result(self, job_result: str):
+        """Sets the job_result of this JobResponse.
+
+
+        :param job_result: The job_result of this JobResponse.
+        :type job_result: str
+        """
+
+        self._job_result = job_result

swagger_server/models/oznaci_besedilo_async_body.py

@@ -0,0 +1,62 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from swagger_server.models.base_model_ import Model
+from swagger_server import util
+
+
+class OznaciBesediloAsyncBody(Model):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+    def __init__(self, besedilo: str=None):  # noqa: E501
+        """OznaciBesediloAsyncBody - a model defined in Swagger
+
+        :param besedilo: The besedilo of this OznaciBesediloAsyncBody.  # noqa: E501
+        :type besedilo: str
+        """
+        self.swagger_types = {
+            'besedilo': str
+        }
+
+        self.attribute_map = {
+            'besedilo': 'besedilo'
+        }
+        self._besedilo = besedilo
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'OznaciBesediloAsyncBody':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The oznaciBesediloAsync_body of this OznaciBesediloAsyncBody.  # noqa: E501
+        :rtype: OznaciBesediloAsyncBody
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def besedilo(self) -> str:
+        """Gets the besedilo of this OznaciBesediloAsyncBody.
+
+
+        :return: The besedilo of this OznaciBesediloAsyncBody.
+        :rtype: str
+        """
+        return self._besedilo
+
+    @besedilo.setter
+    def besedilo(self, besedilo: str):
+        """Sets the besedilo of this OznaciBesediloAsyncBody.
+
+
+        :param besedilo: The besedilo of this OznaciBesediloAsyncBody.
+        :type besedilo: str
+        """
+
+        self._besedilo = besedilo

swagger_server/requets_db/models/vrsta.py

@@ -0,0 +1,90 @@
+import pathlib
+
+import werkzeug.datastructures
+from peewee import *
+from datetime import datetime
+import os
+from swagger_server.util import get_random_filename
+from werkzeug.utils import secure_filename
+
+LOCAL_PATH = "requets_db/dbs"
+SERVER_PATH = "swagger_server/requets_db/dbs"
+
+if not os.path.exists(LOCAL_PATH):
+    if os.path.exists('requets_db'):
+        os.makedirs(LOCAL_PATH, exist_ok=True)
+    elif os.path.exists('swagger_server/requets_db'):
+        os.makedirs(SERVER_PATH, exist_ok=True)
+
+DB = f'{LOCAL_PATH}/jobs.db'
+if not os.path.exists(LOCAL_PATH):
+    DB = f'{SERVER_PATH}/jobs.db'
+db = SqliteDatabase(DB, pragmas={
+    # 'journal_mode': 'wal',
+    'cache_size': -1 * 128 * 1024,  # 128MB
+    'foreign_keys': 1
+})
+
+
+class BaseModel(Model):
+    class Meta:
+        database = db
+
+
+class Job(BaseModel):
+    id = AutoField(index=True)
+    job_type = IntegerField()
+    job_input = TextField(index=True, null=True)
+    job_output = TextField(null=True)
+    created_on = DateTimeField(default=datetime.utcnow)
+    finished_on = DateTimeField(null=True)
+    started_on = DateTimeField(null=True)
+    input_size = IntegerField()
+    input_file = TextField(index=True, null=True)
+
+
+db.drop_tables([Job])  # TODO: After pushing this, comment it and push again
+db.create_tables([Job])
+
+
+class JobManager:
+    @staticmethod
+    def create_job(job_type, job_input) -> Tuple(Job, bool):
+        """
+        :param: job_type
+        :possibilities:
+        # 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 12 = oboje
+        # 3 = pretvori dat v besedilo OCR, 2 = oznaci besedilo, 32 = oboje
+        # 4 = izlusci async
+
+        :return: Job object, Did already exist boolean
+        """
+        try:
+            if job_type in [2, 4]:
+                job, is_new = Job.get_or_create(job_type=job_type, job_input=job_input, input_size=len(job_input))
+            elif job_type in [1, 3, 12, 32]:
+                tmp_file = ""
+                while True:
+                    # just in case a VERY rare chance of a same generate name happens
+                    tmp_file = "tmp/" + secure_filename(get_random_filename() + "_" + job_input.filename)
+                    if not os.path.exists(tmp_file):
+                        break
+                pathlib.Path('tmp').mkdir(exist_ok=True)
+                job_input: werkzeug.datastructures.FileStorage
+                job_input.save(tmp_file)
+                job, is_new = Job.get_or_create(job_type=job_type, input_file=tmp_file, input_size=-1)
+            return job, is_new
+
+        except Exception as e:
+            print(f'Exception at creating a job: {e}')
+            return None, False
+
+        # try:
+        #     job = Job.get_or_none(Job.job_input == job_input, Job.job_type == job_type)
+        #     if job:
+        #         return job, True
+        #     job = Job.create(job_type=job_type, job_input=job_input, input_size=len(job_input))
+        #     return job, False
+        # except Exception as e:
+        #     print(f'Exception at creating a job: {e}')
+        #     return None, False

swagger_server/swagger/swagger.yaml

@@ -6,18 +6,64 @@ servers:
 - url: http://localhost:8089
   description: Generated server url
 paths:
-  /oznaciBesedilo:
+  /job/{job_id}:
+    get:
+      tags:
+      - jobs
+      summary: Vrne status
+      operationId: get_job_status
+      parameters:
+      - name: job_id
+        in: path
+        required: true
+        style: simple
+        explode: false
+        schema:
+          type: integer
+          format: int64
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/JobResponse'
+      x-openapi-router-controller: swagger_server.controllers.jobs_controller
+    delete:
+      tags:
+      - jobs
+      summary: Izbriše job
+      operationId: delete_job
+      parameters:
+      - name: job_id
+        in: path
+        required: true
+        style: simple
+        explode: false
+        schema:
+          type: integer
+          format: int64
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.jobs_controller
+  /oznaciBesediloAsync:
     post:
       tags:
-      - marktext
+      - marktext-async
-      summary: Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne
+      summary: Označi surovo (angl. raw) besedilo s classlo/stanzo z uporabo slovenskih
-        conll-u format
+        modelov ter vrne conll-u format
       operationId: get_text
       requestBody:
         content:
           application/json:
             schema:
-              type: string
+              $ref: '#/components/schemas/oznaciBesediloAsync_body'
         required: true
       responses:
         "200":
@@ -27,18 +73,101 @@ paths:
               schema:
                 type: string
                 x-content-type: '*/*'
-      x-openapi-router-controller: swagger_server.controllers.marktext_controller
+      x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
-  /izlusci:
+  /datotekaVBesediloAsync:
+    post:
+      tags:
+      - marktext-async
+      summary: "Pretvori datoteko v besedilo, vrača tekst"
+      operationId: get_text_from_doc_async
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/datotekaVBesediloAsync_body'
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
+  /datotekaVConlluAsync:
+    post:
+      tags:
+      - marktext-async
+      summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih
+        modelov ter vrne conll-u format
+      operationId: get_conllu_from_file_async
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/datotekaVConlluAsync_body'
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
+  /datotekaVConlluAsync/ocr:
+    post:
+      tags:
+      - marktext-async
+      summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih
+        modelov ter vrne conll-u format
+      operationId: get_conllu_from_file_ocr_async
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/datotekaVConlluAsync_ocr_body'
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
+  /datotekaVBesediloAsync/ocr:
+    post:
+      tags:
+      - marktext-async
+      summary: "Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst"
+      operationId: get_text_from_file_ocr_async
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/datotekaVBesediloAsync_ocr_body'
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
+  /izlusciSync:
     post:
       tags:
       - extract
-      summary: Izlusci terminološke kandidate iz seznama besedil v conllu obliki
+      summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\
-      operationId: get_candidates
+        \ [sihrono, rezultat v sami zahtevi]"
+      operationId: get_candidates_sync
       requestBody:
         content:
           application/json:
             schema:
-              $ref: '#/components/schemas/izlusci_body'
+              $ref: '#/components/schemas/izlusciSync_body'
         required: true
       responses:
         "200":
@@ -51,10 +180,52 @@ paths:
                   $ref: '#/components/schemas/TerminoloskiKandidat'
                 x-content-type: '*/*'
       x-openapi-router-controller: swagger_server.controllers.extract_controller
-  /datotekaVBesedilo/ocr:
+  /izlusciAsync:
     post:
       tags:
-      - doc-2text
+      - extract
+      summary: "Izlusci terminološke kandidate iz seznama besedil v conllu obliki\
+        \ [asinhrono, ustvari novi job]"
+      operationId: get_candidates_async
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/izlusciAsync_body'
+        required: true
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.extract_controller
+  /datotekaVBesediloSync:
+    post:
+      tags:
+      - marktext-sync
+      summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo"
+      operationId: datoteka_v_besedilo_sync_post
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/datotekaVBesediloSync_body'
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
+  /datotekaVBesediloSync/ocr:
+    post:
+      tags:
+      - marktext-sync
       summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo\
         \ s pomočjo ocr razpoznavanja"
       operationId: get_text_ocr
@@ -62,7 +233,7 @@ paths:
         content:
           multipart/form-data:
             schema:
-              $ref: '#/components/schemas/datotekaVBesedilo_ocr_body'
+              $ref: '#/components/schemas/datotekaVBesediloSync_ocr_body'
       responses:
         "200":
           description: OK
@@ -71,18 +242,18 @@ paths:
               schema:
                 type: string
                 x-content-type: '*/*'
-      x-openapi-router-controller: swagger_server.controllers.doc2text_controller
+      x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
-  /datotekaVBesedilo/:
+  /datotekaVConlluSync:
     post:
       tags:
-      - doc-2text
+      - marktext-sync
-      summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo"
+      summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu"
-      operationId: datoteka_v_besedilo_post
+      operationId: datoteka_v_besedilo_in_classla
       requestBody:
         content:
           multipart/form-data:
             schema:
-              $ref: '#/components/schemas/datotekaVBesedilo_body'
+              $ref: '#/components/schemas/datotekaVConlluSync_body'
       responses:
         "200":
           description: OK
@@ -91,7 +262,28 @@ paths:
               schema:
                 type: string
                 x-content-type: '*/*'
-      x-openapi-router-controller: swagger_server.controllers.doc2text_controller
+      x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
+  /datotekaVConlluSync/ocr:
+    post:
+      tags:
+      - marktext-sync
+      summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s\
+        \ pomočjo ocr razpoznavanja"
+      operationId: get_conllu_ocr
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/datotekaVConlluSync_ocr_body'
+      responses:
+        "200":
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+                x-content-type: '*/*'
+      x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
   /oss/steviloBesedilPoIskanju:
     get:
       tags:
@@ -101,7 +293,7 @@ paths:
       parameters:
       - name: leta
         in: query
-        required: true
+        required: false
         style: form
         explode: true
         schema:
@@ -111,25 +303,7 @@ paths:
             format: int64
       - name: vrste
         in: query
-        required: true
+        required: false
-        style: form
-        explode: true
-        schema:
-          type: array
-          items:
-            type: string
-      - name: kljucnebesede
-        in: query
-        required: true
-        style: form
-        explode: true
-        schema:
-          type: array
-          items:
-            type: string
-      - name: cerifpodrocja
-        in: query
-        required: true
         style: form
         explode: true
         schema:
@@ -137,6 +311,24 @@ paths:
           items:
             type: integer
             format: int64
+      - name: kljucnebesede
+        in: query
+        required: false
+        style: form
+        explode: true
+        schema:
+          type: array
+          items:
+            type: string
+      - name: udk
+        in: query
+        required: false
+        style: form
+        explode: true
+        schema:
+          type: array
+          items:
+            type: string
       responses:
         "200":
           description: OK
@@ -156,7 +348,7 @@ paths:
       parameters:
       - name: leta
         in: query
-        required: true
+        required: false
         style: form
         explode: true
         schema:
@@ -166,25 +358,7 @@ paths:
             format: int64
       - name: vrste
         in: query
-        required: true
+        required: false
-        style: form
-        explode: true
-        schema:
-          type: array
-          items:
-            type: string
-      - name: kljucnebesede
-        in: query
-        required: true
-        style: form
-        explode: true
-        schema:
-          type: array
-          items:
-            type: string
-      - name: cerifpodrocja
-        in: query
-        required: true
         style: form
         explode: true
         schema:
@@ -192,6 +366,24 @@ paths:
           items:
             type: integer
             format: int64
+      - name: kljucnebesede
+        in: query
+        required: false
+        style: form
+        explode: true
+        schema:
+          type: array
+          items:
+            type: string
+      - name: udk
+        in: query
+        required: false
+        style: form
+        explode: true
+        schema:
+          type: array
+          items:
+            type: string
       responses:
         "200":
           description: OK
@@ -270,7 +462,7 @@ paths:
       summary: Vrne binarni zapis v originalnem formatu po id-ju datoteke
       operationId: get_file
       parameters:
-      - name: id
+      - name: file_id
         in: query
         required: true
         style: form
@@ -353,7 +545,7 @@ paths:
       summary: Vrne CoNNL-U po id-ju datoteke
       operationId: get_conllu
       parameters:
-      - name: id
+      - name: file_id
         in: query
         required: true
         style: form
@@ -377,7 +569,7 @@ paths:
       summary: Vrne besedilo po id-ju datoteke
       operationId: oss_besedilo_po_id_get
       parameters:
-      - name: id
+      - name: file_id
         in: query
         required: true
         style: form
@@ -485,7 +677,72 @@ components:
         kanonicnaoblika: kanonicnaoblika
         nosilnautez: 0.8008282
         kandidat: kandidat
-    izlusci_body:
+    JobResponse:
+      required:
+      - finished_job
+      type: object
+      properties:
+        job_status:
+          type: string
+          enum:
+          - waiting in que
+          - currently processing
+          - finished processing
+        finished_on:
+          type: string
+          format: date-time
+        started_on:
+          type: string
+          format: date-time
+        created_on:
+          type: string
+          format: date-time
+        job_result:
+          type: string
+      example:
+        job_status: waiting in que
+        started_on: 2000-01-23T04:56:07.000+00:00
+        created_on: 2000-01-23T04:56:07.000+00:00
+        finished_on: 2000-01-23T04:56:07.000+00:00
+        job_result: job_result
+    oznaciBesediloAsync_body:
+      type: object
+      properties:
+        besedilo:
+          type: string
+    datotekaVBesediloAsync_body:
+      required:
+      - file
+      type: object
+      properties:
+        file:
+          type: string
+          format: binary
+    datotekaVConlluAsync_body:
+      required:
+      - file
+      type: object
+      properties:
+        file:
+          type: string
+          format: binary
+    datotekaVConlluAsync_ocr_body:
+      required:
+      - file
+      type: object
+      properties:
+        file:
+          type: string
+          format: binary
+    datotekaVBesediloAsync_ocr_body:
+      required:
+      - file
+      type: object
+      properties:
+        file:
+          type: string
+          format: binary
+    izlusciSync_body:
       type: object
       properties:
         conllus:
@@ -496,7 +753,18 @@ components:
           type: array
           items:
             type: string
-    datotekaVBesedilo_ocr_body:
+    izlusciAsync_body:
+      type: object
+      properties:
+        conllus:
+          type: array
+          items:
+            type: string
+        prepovedaneBesede:
+          type: array
+          items:
+            type: string
+    datotekaVBesediloSync_body:
       required:
       - file
       type: object
@@ -504,7 +772,23 @@ components:
         file:
           type: string
           format: binary
-    datotekaVBesedilo_body:
+    datotekaVBesediloSync_ocr_body:
+      required:
+      - file
+      type: object
+      properties:
+        file:
+          type: string
+          format: binary
+    datotekaVConlluSync_body:
+      required:
+      - file
+      type: object
+      properties:
+        file:
+          type: string
+          format: binary
+    datotekaVConlluSync_ocr_body:
       required:
       - file
       type: object

swagger_server/test/__init__.py

@@ -1,16 +0,0 @@
-import logging
-
-import connexion
-from flask_testing import TestCase
-
-from swagger_server.encoder import JSONEncoder
-
-
-class BaseTestCase(TestCase):
-
-    def create_app(self):
-        logging.getLogger('connexion.operation').setLevel('ERROR')
-        app = connexion.App(__name__, specification_dir='../swagger/')
-        app.app.json_encoder = JSONEncoder
-        app.add_api('swagger.yaml')
-        return app.app

swagger_server/test/__main__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+import connexion
+
+from swagger_server import encoder
+
+
+def main():
+    app = connexion.App(__name__, specification_dir='./swagger/')
+    app.app.json_encoder = encoder.JSONEncoder
+    app.add_api('swagger.yaml', arguments={'title': 'OpenAPI definition'}, pythonic_params=True)
+    app.run(port=8080)
+
+
+if __name__ == '__main__':
+    main()

swagger_server/test/encoder.py

@@ -0,0 +1,20 @@
+from connexion.apps.flask_app import FlaskJSONEncoder
+import six
+
+from swagger_server.models.base_model_ import Model
+
+
+class JSONEncoder(FlaskJSONEncoder):
+    include_nulls = False
+
+    def default(self, o):
+        if isinstance(o, Model):
+            dikt = {}
+            for attr, _ in six.iteritems(o.swagger_types):
+                value = getattr(o, attr)
+                if value is None and not self.include_nulls:
+                    continue
+                attr = o.attribute_map[attr]
+                dikt[attr] = value
+            return dikt
+        return FlaskJSONEncoder.default(self, o)

swagger_server/test/test_doc2text_controller.py

@@ -1,45 +0,0 @@
-# coding: utf-8
-
-from __future__ import absolute_import
-
-from flask import json
-from six import BytesIO
-
-from swagger_server.test import BaseTestCase
-
-
-class TestDoc2textController(BaseTestCase):
-    """Doc2textController integration test stubs"""
-
-    def test_datoteka_v_besedilo_post(self):
-        """Test case for datoteka_v_besedilo_post
-
-        Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo
-        """
-        data = dict(file='file_example')
-        response = self.client.open(
-            '/datotekaVBesedilo/',
-            method='POST',
-            data=data,
-            content_type='multipart/form-data')
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_get_text_ocr(self):
-        """Test case for get_text_ocr
-
-        Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
-        """
-        data = dict(file='file_example')
-        response = self.client.open(
-            '/datotekaVBesedilo/ocr',
-            method='POST',
-            data=data,
-            content_type='multipart/form-data')
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()

swagger_server/test/test_extract_controller.py

@@ -1,33 +0,0 @@
-# coding: utf-8
-
-from __future__ import absolute_import
-
-from flask import json
-from six import BytesIO
-
-from swagger_server.models.izlusci_body import IzlusciBody  # noqa: E501
-from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat  # noqa: E501
-from swagger_server.test import BaseTestCase
-
-
-class TestExtractController(BaseTestCase):
-    """ExtractController integration test stubs"""
-
-    def test_get_candidates(self):
-        """Test case for get_candidates
-
-        Izlusci terminološke kandidate iz seznama besedil v conllu obliki
-        """
-        body = IzlusciBody()
-        response = self.client.open(
-            '/izlusci',
-            method='POST',
-            data=json.dumps(body),
-            content_type='application/json')
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()

swagger_server/test/test_marktext_controller.py

@@ -1,31 +0,0 @@
-# coding: utf-8
-
-from __future__ import absolute_import
-
-from flask import json
-from six import BytesIO
-
-from swagger_server.test import BaseTestCase
-
-
-class TestMarktextController(BaseTestCase):
-    """MarktextController integration test stubs"""
-
-    def test_get_text(self):
-        """Test case for get_text
-
-        Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
-        """
-        body = 'body_example'
-        response = self.client.open(
-            '/oznaciBesedilo',
-            method='POST',
-            data=json.dumps(body),
-            content_type='application/json')
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()

swagger_server/test/test_oss_controller.py

@@ -1,137 +0,0 @@
-# coding: utf-8
-
-from __future__ import absolute_import
-
-from flask import json
-from six import BytesIO
-
-from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat  # noqa: E501
-from swagger_server.test import BaseTestCase
-
-
-class TestOssController(BaseTestCase):
-    """OssController integration test stubs"""
-
-    def test_get_conllu(self):
-        """Test case for get_conllu
-
-        Vrne CoNNL-U po id-ju datoteke
-        """
-        query_string = [('id', 789)]
-        response = self.client.open(
-            '/oss/conlluPoId',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_get_conllus(self):
-        """Test case for get_conllus
-
-        Vrne seznam CoNNL-U-jev glede na iskalne pogoje
-        """
-        query_string = [('leta', 56),
-                        ('vrste', 'vrste_example'),
-                        ('kljucnebesede', 'kljucnebesede_example'),
-                        ('cerifpodrocja', 56)]
-        response = self.client.open(
-            '/oss/conlluPoIskanju',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_get_extracted_words(self):
-        """Test case for get_extracted_words
-
-        Vrne terminloške kandidate glede na 
-        """
-        query_string = [('leta', 56),
-                        ('vrste', 'vrste_example'),
-                        ('kljucnebesede', 'kljucnebesede_example'),
-                        ('cerifpodrocja', 56)]
-        response = self.client.open(
-            '/oss/izlusciPoIskanju',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_get_file(self):
-        """Test case for get_file
-
-        Vrne binarni zapis v originalnem formatu po id-ju datoteke
-        """
-        query_string = [('id', 789)]
-        response = self.client.open(
-            '/oss/datotekaPoId',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_get_files(self):
-        """Test case for get_files
-
-        Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje
-        """
-        query_string = [('leta', 56),
-                        ('vrste', 'vrste_example'),
-                        ('kljucnebesede', 'kljucnebesede_example'),
-                        ('cerifpodrocja', 56)]
-        response = self.client.open(
-            '/oss/datotekePoIskanju',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_get_number_texts(self):
-        """Test case for get_number_texts
-
-        Vrne število besedil glede na iskalne pogoje
-        """
-        query_string = [('leta', 56),
-                        ('vrste', 'vrste_example'),
-                        ('kljucnebesede', 'kljucnebesede_example'),
-                        ('cerifpodrocja', 56)]
-        response = self.client.open(
-            '/oss/steviloBesedilPoIskanju',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_get_texts(self):
-        """Test case for get_texts
-
-        Vrne seznam besedil glede na iskalne pogoje
-        """
-        query_string = [('leta', 56),
-                        ('vrste', 'vrste_example'),
-                        ('kljucnebesede', 'kljucnebesede_example'),
-                        ('cerifpodrocja', 56)]
-        response = self.client.open(
-            '/oss/besedilaPoIskanju',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-    def test_oss_besedilo_po_id_get(self):
-        """Test case for oss_besedilo_po_id_get
-
-        Vrne besedilo po id-ju datoteke
-        """
-        query_string = [('id', 789)]
-        response = self.client.open(
-            '/oss/besediloPoId',
-            method='GET',
-            query_string=query_string)
-        self.assert200(response,
-                       'Response body is : ' + response.data.decode('utf-8'))
-
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()

swagger_server/test/type_util.py

@@ -0,0 +1,32 @@
+# coding: utf-8
+
+import sys
+
+if sys.version_info < (3, 7):
+    import typing
+
+    def is_generic(klass):
+        """ Determine whether klass is a generic class """
+        return type(klass) == typing.GenericMeta
+
+    def is_dict(klass):
+        """ Determine whether klass is a Dict """
+        return klass.__extra__ == dict
+
+    def is_list(klass):
+        """ Determine whether klass is a List """
+        return klass.__extra__ == list
+
+else:
+
+    def is_generic(klass):
+        """ Determine whether klass is a generic class """
+        return hasattr(klass, '__origin__')
+
+    def is_dict(klass):
+        """ Determine whether klass is a Dict """
+        return klass.__origin__ == dict
+
+    def is_list(klass):
+        """ Determine whether klass is a List """
+        return klass.__origin__ == list

swagger_server/test/util.py

@@ -0,0 +1,142 @@
+import datetime
+
+import six
+import typing
+from swagger_server import type_util
+
+
+def _deserialize(data, klass):
+    """Deserializes dict, list, str into an object.
+
+    :param data: dict, list or str.
+    :param klass: class literal, or string of class name.
+
+    :return: object.
+    """
+    if data is None:
+        return None
+
+    if klass in six.integer_types or klass in (float, str, bool, bytearray):
+        return _deserialize_primitive(data, klass)
+    elif klass == object:
+        return _deserialize_object(data)
+    elif klass == datetime.date:
+        return deserialize_date(data)
+    elif klass == datetime.datetime:
+        return deserialize_datetime(data)
+    elif type_util.is_generic(klass):
+        if type_util.is_list(klass):
+            return _deserialize_list(data, klass.__args__[0])
+        if type_util.is_dict(klass):
+            return _deserialize_dict(data, klass.__args__[1])
+    else:
+        return deserialize_model(data, klass)
+
+
+def _deserialize_primitive(data, klass):
+    """Deserializes to primitive type.
+
+    :param data: data to deserialize.
+    :param klass: class literal.
+
+    :return: int, long, float, str, bool.
+    :rtype: int | long | float | str | bool
+    """
+    try:
+        value = klass(data)
+    except UnicodeEncodeError:
+        value = six.u(data)
+    except TypeError:
+        value = data
+    return value
+
+
+def _deserialize_object(value):
+    """Return an original value.
+
+    :return: object.
+    """
+    return value
+
+
+def deserialize_date(string):
+    """Deserializes string to date.
+
+    :param string: str.
+    :type string: str
+    :return: date.
+    :rtype: date
+    """
+    try:
+        from dateutil.parser import parse
+        return parse(string).date()
+    except ImportError:
+        return string
+
+
+def deserialize_datetime(string):
+    """Deserializes string to datetime.
+
+    The string should be in iso8601 datetime format.
+
+    :param string: str.
+    :type string: str
+    :return: datetime.
+    :rtype: datetime
+    """
+    try:
+        from dateutil.parser import parse
+        return parse(string)
+    except ImportError:
+        return string
+
+
+def deserialize_model(data, klass):
+    """Deserializes list or dict to model.
+
+    :param data: dict, list.
+    :type data: dict | list
+    :param klass: class literal.
+    :return: model object.
+    """
+    instance = klass()
+
+    if not instance.swagger_types:
+        return data
+
+    for attr, attr_type in six.iteritems(instance.swagger_types):
+        if data is not None \
+                and instance.attribute_map[attr] in data \
+                and isinstance(data, (list, dict)):
+            value = data[instance.attribute_map[attr]]
+            setattr(instance, attr, _deserialize(value, attr_type))
+
+    return instance
+
+
+def _deserialize_list(data, boxed_type):
+    """Deserializes a list and its elements.
+
+    :param data: list to deserialize.
+    :type data: list
+    :param boxed_type: class literal.
+
+    :return: deserialized list.
+    :rtype: list
+    """
+    return [_deserialize(sub_data, boxed_type)
+            for sub_data in data]
+
+
+def _deserialize_dict(data, boxed_type):
+    """Deserializes a dict and its elements.
+
+    :param data: dict to deserialize.
+    :type data: dict
+    :param boxed_type: class literal.
+
+    :return: deserialized dict.
+    :rtype: dict
+    """
+    return {k: _deserialize(v, boxed_type)
+            for k, v in six.iteritems(data)}

swagger_server/util.py

@@ -1,8 +1,18 @@
+import codecs
 import datetime
+import os.path
+import pathlib
 
 import six
 import typing
+
+import werkzeug.datastructures
+from werkzeug.utils import secure_filename
+
 from swagger_server import type_util
+import pandas as pd
+import string
+import random
 
 
 def _deserialize(data, klass):
@@ -140,3 +150,66 @@ def _deserialize_dict(data, boxed_type):
     """
     return {k: _deserialize(v, boxed_type)
             for k, v in six.iteritems(data)}
+
+
+def is_docker() -> bool:
+    # todo: better way of checking if we're on docker or if we're in the develoment enviroment
+    return not os.path.exists('.env')
+
+
+def get_conllu_file_path_by_id(file_id):
+    r = f'classla_OS2022/conll/rsdo_doc-{file_id}.plainText.conllu'
+    if is_docker():
+        return f'/usr/src/app/{r}'
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_original_file_path_by_id(file_id):
+    r = f'classla_OS2022/besedila/rsdo_doc-{file_id}.xml'
+    if is_docker():
+        return f'/usr/src/app/{r}'
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_tei_file_path_by_id(file_id):
+    r = f'classla_OS2022/tei/rsdo_doc-{file_id}.plainText.tei.xml'
+    if is_docker():
+        return f'/usr/src/app/{r}'
+    return f'../mnt/ssd/ds_ftp/{r}'
+
+
+def get_files_by_keywords(kljucnebesede):
+    ret = []
+    kljucnebesede = [k.lower() for k in kljucnebesede]
+    # Temporary solution until connection with mariadb is fixed
+    ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/"
+    print("Looping trough ngrams")
+    for path, dirs, files, in os.walk(ngrams_path):
+        for i, _file in enumerate(files[:100]):
+            if i % 500 == 0: print(f"{i}/{len(files)}")
+            file = f'{ngrams_path}{_file}'
+            data = pd.read_csv(file, sep='\t')
+            amount = len(data[data['ngram_len'] == 1 & data['gram_text'].str.lower().isin(kljucnebesede)])
+            # this should be 1
+            if amount >= 1:
+                ret.append(_file[9:][:-12])
+    return ret
+
+
+def get_random_filename():
+    ts = str(int(datetime.datetime.now().timestamp()))
+    extra = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
+    return f'{ts}_{extra}'
+
+
+def create_random_file_in_tmp_folder(fill_content, extension=""):
+    pathlib.Path('tmp').mkdir(exist_ok=True)
+    tmp_file = ""
+    while True:
+        # just in case a VERY rare chance of a same generate name happens
+        tmp_file = "tmp/" + secure_filename(get_random_filename() + extension)
+        if not os.path.exists(tmp_file):
+            break
+    with codecs.open(tmp_file, 'w', 'utf-8') as f:
+        f.write(fill_content)
+    return tmp_file

swagger_server/utils/cl_utils.py

@@ -0,0 +1,47 @@
+import classla
+import time
+from swagger_server import util
+from pathlib import Path
+import re
+
+nlp_loaded = False
+nlpSlo = classla.Pipeline('sl', processors='tokenize,ner,pos,lemma,depparse')
+nlp_loaded = True
+
+sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL)
+
+def raw_text_to_conllu(text):
+    try:
+        docall = nlpSlo(text)
+        docallconllu = docall.to_conll()
+
+        return docallconllu, 200
+    except Exception as e:
+        return e, 400
+
+
+def multipla_conllus_to_one_from_file_ids(list_file_ids):
+    sent_cnt = 1
+    ret = "# newpar id = 1\n"
+    files = [f'{util.get_conllu_file_path_by_id(i)}' for i in list_file_ids]
+    for file in files:
+        txt = Path(file).read_text('utf-8')
+        matches = sent_extractor.finditer(txt)
+        for match in matches:
+            ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
+            sent_cnt += 1
+
+    return ret
+
+
+def multipla_conllus_to_one_from_conllus_arr(list_conllus):
+    sent_cnt = 1
+    ret = "# newpar id = 1\n"
+    for conllu in list_conllus:
+        conllu = conllu.replace('\r\n', '\n')
+        matches = sent_extractor.finditer(conllu)
+        for match in matches:
+            ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
+            sent_cnt += 1
+
+    return ret

swagger_server/utils/db_utils.py

@@ -0,0 +1,225 @@
+import mariadb
+import os
+import sys
+import requests
+import json
+
+
+database_info = {
+    'database': os.getenv("MDB_DATABASE", "oss"),
+    'host': os.getenv("MDB_HOST", "localhost"),
+    'port': int(os.getenv("PORT", 3306)) ,
+    'user': os.getenv("MDB_USER", "root"),
+    'password': os.getenv("MDB_PASSWORD", "root"),
+}
+
+canonapi_endpoint = "http://canonizer:5000/rest_api/canonize"
+
+cur = None
+# Connect to MariaDB Platform
+
+def get_files_by_udc(udc):
+    ret = []
+
+    try:
+        print(database_info)
+        conn = mariadb.connect(**database_info)
+        cur = conn.cursor()
+        where_in = ','.join(['%s'] * len(udc))
+        print(where_in)
+        sql = "select distinct xml_id from metadata_udc where udk IN (%s)" % (where_in)
+        print(sql)
+        cur.execute(sql,udc)
+        #cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
+        ret = list(cur)
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB Platform: {e}")
+    
+
+    return ret
+
+def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
+    ret = []
+  
+    try:
+        print(database_info)
+        conn = mariadb.connect(**database_info)
+        cur = conn.cursor()
+        
+        
+        
+        sql = "select distinct document_id from metadata"
+        where=""
+        params=[]
+        if (udk):
+            where_in_udk = ','.join(['%s'] * len(udk))
+            where=" udk IN (%s) " % (where_in_udk)
+            params=udk
+        
+        if (leta):
+
+            where_in_leta = ','.join(['%s'] * len(leta))
+            if (where):
+                where=where+ " AND "
+            where=where + " leto IN (%s) " % (where_in_leta)
+            params=params+leta
+        
+        if (vrste):
+            if (where):
+                where=where+ " AND "
+            where_in_vrste = ','.join(['%s'] * len(vrste))
+            where=where + " tipologija IN (%s) " % (where_in_vrste)
+            params=params+vrste
+
+        if (kljucnebesede):
+            if (where):
+                where=where+ " AND "
+            where_in_kb = ','.join(['%s'] * len(kljucnebesede))
+            where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
+            params=params+kljucnebesede
+
+        if(where):
+            sql=sql+" where " + where + ";"
+        
+        print(sql)
+        print(params)
+
+
+        
+        cur.execute(sql,params)
+        
+        ret = list(cur)
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB Platform: {e}")
+    
+
+    return ret
+
+
+def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
+    ret = []
+  
+    try:
+        print(database_info)
+        conn = mariadb.connect(**database_info)
+        cur = conn.cursor()
+        
+        
+        
+        sql = "select distinct document_id from metadata"
+        where=""
+        params=[]
+        if (udk):
+            where_in_udk = ','.join(['%s'] * len(udk))
+            where=" udk IN (%s) " % (where_in_udk)
+            params=udk
+        
+        if (leta):
+
+            where_in_leta = ','.join(['%s'] * len(leta))
+            if (where):
+                where=where+ " AND "
+            where=where + " leto IN (%s) " % (where_in_leta)
+            params=params+leta
+        
+        if (vrste):
+            if (where):
+                where=where+ " AND "
+            where_in_vrste = ','.join(['%s'] * len(vrste))
+            where=where + " tipologija IN (%s) " % (where_in_vrste)
+            params=params+vrste
+
+        if (kljucnebesede):
+            if (where):
+                where=where+ " AND "
+            where_in_kb = ','.join(['%s'] * len(kljucnebesede))
+            where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
+            params=params+kljucnebesede
+
+        if(where):
+            sql=sql+" where " + where
+        
+        print(sql)
+        print(params)
+
+        sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf, sum(tf) as tf from (
+                    SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf
+                    FROM ngrams_upos_tf tf, documents d,
+                        (
+                        Select ngram, upos, count(*) as df from ngrams_upos_tf TF
+                        where document_id in
+                    ({sql})
+                    group by TF.ngram, TF.upos
+                    ) dff, ngrams_upos_df df
+                    where
+                    tf.document_id=d.document_id and
+                    df.ngram=tf.ngram AND df.upos=tf.upos and
+                    dff.ngram=tf.ngram AND dff.upos=tf.upos
+                    ) X
+                    group by ngram,upos
+                    order by tfidf desc
+                    limit 1000;"""
+                    #
+        print (sqltk)
+        
+        cur.execute(sqltk,params)
+        terms=cur.fetchall()
+        #ret = list(cur)
+        can = {'forms':[
+            ngram 
+            for ngram in terms
+            ]
+        }
+        res = requests.post(ATEapi_endpoint, json=can)
+        data = res.json().canonical_forms
+
+        ret = {'terminoloski_kandidati': [
+            {
+                'POSoznake': x.upos,
+                'kandidat': x.ngram,  # more to bit lemma al terms?
+                'kanonicnaoblika': d,
+                'ranking': x.tfidf,
+                'podporneutezi': [
+                    0.0,  # ????????
+                    0.0  # ??????
+                ],
+                'pogostostpojavljanja': [tf, 0]  # ???????
+            }
+            for d,x in zip(data,cur) 
+        ]}
+            
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB Platform: {e}")
+    
+
+    return ret
+
+# class BaseModel(Model):
+#     class Meta:
+#         database = db
+#
+#
+# class os2022_ngrams(BaseModel):
+#     file_id = IntegerField()
+#     sent_id = FloatField()
+#     ngram_len = IntegerField()
+#     frequency_g_t = IntegerField()
+#     gram_text = TextField()
+#     lemma_text = TextField()
+#     xpos_text = TextField()
+#     upos_text = TextField()
+#
+# db.connect()
+
+
+#class Ngrams_Manager:
+    #@staticmethod
+    #def get_by_file_id(file_id):
+        #try:
+            # cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}')
+            # cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
+            # return list(cur)
+         #   return 1
+        #except Exception as e:
+            #print(e, 'EXC')
+        #return 0

swagger_server/utils/txt_utils.py

@@ -0,0 +1,117 @@
+import os.path
+
+import pytesseract
+import requests
+import docx
+import xml.etree.ElementTree as ET
+from PyPDF2 import PdfReader
+from swagger_server.utils import cl_utils
+import cv2
+import numpy as np
+import magic
+import re
+#to še mora v env 
+tika_server = "http://tika2:9999/tika"
+
+# endpoint below to be used only for development purposes (don't need to run docker)
+# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
+
+
+def extract_text_prepResp(file, content_type=""):
+    content_type = file.content_type
+    if content_type is None:
+        content_type = magic.from_file(file.stream.name, mime=True)
+
+    content = ""
+    if tika_responding():
+        try:
+            response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
+            content = response.text
+            #preveri če je pretvorba uspešna
+            
+            # original string
+            res = re.findall(r'\w+', content)
+
+            #preveri, če imamo vsaj 10 besed in če je povprečna dolžina >3 in < 12
+            #če to drži, idi v ocr
+            reslen=map(lambda n:len(n),res)
+            print(f"Število besed je {len(res)}")
+
+            if len(res)>0 :
+                avglen=sum(reslen)/len(res)
+            else:
+                avglen=0
+            
+            print(f"Povprečna dolžina besede je {avglen}")
+            
+            if(len(res)<10 or avglen<4 or avglen>11):
+                print("Besedilo je sumljivo, gremo v OCR in damo file na začetek!")
+                file.seek(0)
+                response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
+                                             "Accept": "text/plain; charset=UTF-8"})
+                content = response.text
+
+            #odstranim še vse prelome vrstic, ker imamo s tem probleme
+            content=' '.join(content.splitlines())
+        except:
+            content = "ERROR - something went wrong when reading file with tika"
+
+    #if content == "":
+    #    if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
+    #        content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
+    #    elif "application/pdf" in content_type:
+    #        reader = PdfReader(file)
+    #        content = '\n'.join([p.extract_text() for p in reader.pages])
+    #       content = content
+    #    elif "text/xml" in content_type:
+    #        root = ET.parse(file).getroot()
+    #        plainText = root.findall('PlainText')
+    #        if len(plainText) == 0:
+    #            return "Didn't find anything in PlainText", 400
+    #        content = '\n'.join([pt.text for pt in plainText])
+    #   # elif "text/plain" in file.content_type:
+    #    else:
+    #       try:
+    #           content = file.read().decode('utf-8')
+    #       except:
+    #           content = "ERROR - something went wrong when reading file with not-tika method!"
+
+    return content, 200
+
+
+def ocr_text_prepResp(file):
+    content = ""
+    if tika_responding():
+        try:
+            response = requests.put(tika_server, data=file,
+                                    headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
+                                             "Accept": "text/plain; charset=UTF-8"})
+            content = response.text
+        except:
+            content = "ERROR - something went wrong when reading file with tika (OCR)"
+
+    if content == "":
+        try:
+            win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"
+            if os.path.exists(win_p):
+                pytesseract.pytesseract.tesseract_cmd = win_p
+
+            # convert string data to numpy array
+            file_bytes = np.fromstring(file.read(), np.uint8)
+            # convert numpy array to image
+            img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
+
+            conf = '-l eng+slv'
+            content = pytesseract.image_to_string(img, config=conf)
+        except:
+            content = "ERROR - something went wrong when reading file with not-tika method! (OCR)"
+
+    return content, 200
+
+
+def tika_responding():
+    try:
+        ret = requests.get(tika_server)
+        return ret.status_code == 200
+    except:
+        return False