webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working fix pre merge

Browse Source
This commit is contained in:
marko.ferme
2023-01-10 11:20:14 +01:00
parent 1cd7663d49
commit 69a92a3420
32 changed files with 1457 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
definition_extraction/Dockerfile
+21
View File
@@ -0,0 +1,21 @@
# pull official base image
FROM python:3.8
# set work directory
WORKDIR /usr/src/app
# set environment variables
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONUNBUFFERED 1
# install dependencies
RUN pip install --upgrade pip
COPY ./requirements.txt /usr/src/app/requirements.txt
RUN pip install -r requirements.txt
#RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
# copy project
COPY . /usr/src/app/
# run entrypoint.sh
ENTRYPOINT ["/usr/src/app/entrypoint.sh"]
definition_extraction/Dockerfile.prod
+58
View File
@@ -0,0 +1,58 @@
###########
# BUILDER #
###########
# pull official base image
FROM python:3.8 as builder
# set work directory
WORKDIR /usr/src/app
# set environment variables
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONUNBUFFERED 1
# install python dependencies
COPY ./requirements.txt .
RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
#########
# FINAL #
#########
# pull official base image
FROM python:3.8
# create directory for the app user
RUN mkdir -p /home/app
# create the app user
RUN addgroup --system app && adduser --system --group app
# create the appropriate directories
ENV HOME=/home/app
ENV APP_HOME=/home/app/web
RUN mkdir $APP_HOME
WORKDIR $APP_HOME
COPY --from=builder /usr/src/app/wheels /wheels
COPY --from=builder /usr/src/app/requirements.txt .
RUN pip install --upgrade pip
RUN pip install --no-cache /wheels/*
COPY ./entrypoint.prod.sh $APP_HOME
# copy project
COPY . $APP_HOME
RUN chmod u+x $APP_HOME/entrypoint.prod.sh
# chown all the files to the app user
RUN chown -R app:app $APP_HOME
# change to the app user
USER app
# run entrypoint.prod.sh
ENTRYPOINT ["/home/app/web/entrypoint.prod.sh"]
definition_extraction/JeStaSoPatterns_finishOK.txt
+9
View File
@@ -0,0 +1,9 @@
{{{\S+_____\S+_____N\S+\s+\S+_____\S+_____Va-r3[psd]-n\s+\S+_____\S+_____N\S+\s*}}} #N je/sta/so N
{{{\S+_____\S+_____N...ny?n?\s+\S+_____\S+_____Va-r3[psd]-n\s+\S+_____\S+_____N...ny?n?\s*}}} #Sam-nom je/sta/so Sam-nom
{{{(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s*)}}}
{{{(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s+)}}}
{{{^(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s*)}}}
{{{^(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s+)}}}
{{{(\S+_____\S+_____N...ny?n?\s+)((\S+_____\S+_____A\S+\s+)*((\S+_____\S+_____A\S+\s+)+((in)|(ali))_____\S+_____Cc\s*(\S+_____\S+_____A\S+\s+))*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*((in)|(ali)_____\S+_____C\S*\s*\S+_____\S+_____A\S+\s*)*(\S+_____\S+_____N\S+\s+)+)*(angl?\._____\S+_____\S+\s+(\S+_____\S+_____[^V]\S*\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*((\S+_____\S+_____A\S+\s+)+((in)|(ali))_____\S+_____Cc\s*(\S+_____\S+_____A\S+\s+))*\S+_____\S+_____N...ny?n?\s*}}}
definition_extraction/SentEx_patternsF.pl
+284
View File
@@ -0,0 +1,284 @@
#!/usr/bin/perl - w
#script to extract occurrences of definitions in a File
#output: definition candidates sorted by patterns
#usage: perl SentEx_patternsF.pl Korpus Patternlist
use utf8;
binmode STDIN, 'utf8';
binmode STDOUT, 'utf8';
$file = shift(@ARGV);
$pat = shift(@ARGV);
if ($file=~/\//){
($filestem) = $file=~ /.*\/(.+)\..../;
}
else {
($filestem) = $file=~ /(.+)\..../;
}
print STDERR "F:$filestem\n";
print STDERR "\n\nDo you want to evaluate against recall test set? If yes enter recall test set file else press return!\n";
# $recalltestset=<>;
$recalltestset="";
&recalltestset;
open(PAT, "<:utf8", "$pat") || die "Cannot open file:$!";
open(OUT, ">:utf8", "ALLPATTERNEVALUATION_$filestem"."\.txt") || die "Cannot open outfile:$!";
push @predefinedpatterns, "0"; #zato da se potem zacne z 1 ne 0
while (<PAT>){
chomp;
if (length>0){
if (m/^\#/){}
else {
print STDERR "[$_]\n\n";
s/{{{(.*)}}}/$1/;
push @predefinedpatterns, "$1";
}
}
}
$k=$#predefinedpatterns;
for ($i=1;$i<=$k; $i++){
$count=0;
$yes=0;
$no=0;
$undef=0;
$allpozit=0;
$allneg=0;
$stop=0;
$eval=0;
@comments=();
print STDERR "PATLIST$i:". $predefinedpatterns[$i]."\n";
#open(OUT1, ">:utf8", "PAT_lema".$i."_$filestem."."xml") || die "Cannot open outfile1:$!";
open(OUT2, ">:utf8", "PAT_".$i."_$filestem."."txt") || die "Cannot open outfile2:$!";
open(FILE, "<:utf8", "$file") || die "Cannot open file:$!";
@positivesbyeachpattern=();
$/ = "\/>\n";#"\n\n";#"<S/>";
while (<FILE>) {
$s = $_;
if ($s=~/defvalue=\"Y/){
$allpozit++;
}
if ($s=~/defvalue=\"N/){
$allneg++;
}
if ($s =~/.*?(<id_sp.*?\>).*?\n/){
($ids)= $s =~/.*?(<id_sp.*?\>).*?\n/g;
}
$s =~s/.*?<id_sp.*?\>.*?\n//gs;
($sentmeta)= $s =~ /(<S sid_sp.*)/;
#print STDERR "$sentmeta";
@wordforms = $s =~ /(.*?)\t.*?\n/gs;
$string = join(" ", @wordforms);
@lines = split(/\n/, $s);
@msds = ();
@token_type_msd = ();
foreach $line (@lines) {
if ($line =~ /((.*?\t+TOK\t)|(angl?\.?\t+TOK_ABBR\t))/){
($token,$type,$msd) = $line =~ /(.+?)\s*\t+.+\t+(.+?)\s*\t+(.+?)\s*\t*\s*$/;
#$msd=~s/(.*?)\s+/$1/;
$token_type_msd=$token."_____".$type."_____".$msd;
#print STDERR $token_type_msd."\n";
#print "$token_type_msd"."\n";
push (@token_type_msd, $token_type_msd);
}
}
$msd_string = join(" ", @token_type_msd);
#print STDERR $msd_string."]\n";
# print $msd_string."\n\n\n";
#print "PAT$i:","$predefinedpatterns[$i]"."\n";
if ($msd_string =~ /$predefinedpatterns[$i]/){ #v resnici token type msd
#print STDERR $msd_string."]\n\n";
#print STDERR "\nYES PREVIOUS MATCHED\n";
if ($stop==0){#da ga samo enkrat sprinta
print OUT2 "\n>>>>".$i.":\t"."{{{$predefinedpatterns[$i]}}}"."\n";
$stop=1;
}
#print OUT1 "$s"."\n";
print OUT2 "$string###$sentmeta"."\n";
$sentmeta=~/S sid_sp=\"(\d+)\"/;
$idnb=$1;
$union{$idnb}="$string###$sentmeta"."\n";
#print OUT2 "MSD string: $msd_string"."\n";
if ($sentmeta=~/defvalue=\"Y/){
$yes++;
$unionYES{$idnb}++;
push @positivesbyeachpattern, $idnb;
}
elsif ($sentmeta=~/defvalue=\"N/){
$no++;
$unionNO{$idnb}++;
}
else {
$undef++;
$unionUNDEF{$idnb}++;
}
$count++;
}
}
$eval=$count-$undef;
if ($count==0) {
$count=0.000001;
push @comments, "\$undefWas0";
}
if ($allpozit==0) {
$allpozit=0.000001;
push @comments, "\$allpozitWas0";
}
if ($eval==0) {
###print OUT "EVAL IS ".$eval." and changed to=0.00001 \n";
$eval=0.00001;
###print OUT "nEWEVAL IS ".$eval.":::\n";
push @comments, "\$evalWas0";
}
print OUT "PATTERN$i:\t{{{$predefinedpatterns[$i]}}}\n";
print OUT "TOTAL EXAMPLES: ".$count."\n";
print OUT "TOTAL EVALUATED: ".$eval."\n";
print OUT "POZITIVES:". $yes."\n";
print OUT "NEGATIVES:". $no."\n";
print OUT "UNDEFINED:". $undef."\n";
print OUT "ALLPOZ:".$allpozit."\n";
print OUT "ALLNEG:".$allneg."\n";
print OUT "PRECISION_eval:". $yes/$eval."\n";
#print OUT "PRECISION_noneval:". $yes/$count."\n";
#print OUT "RECALL:".$yes/$allpozit."\n";
#print OUT "POZ//UNDEF:\t"."$yes//".$undef."\n";
#print OUT "PREC//RECALL:".$yes/$eval."//".$yes/$allpozit;
#print OUT "POZITIVESBYPATTER>$i: "."@positivesbyeachpattern";
#presek @recalltestsetids in @positivesbyeachpattern
foreach $element (@positivesbyeachpattern, @recalltestsetids) { $count{$element}++ }
foreach $element (keys %count) {
push @{ $count{$element} > 1 ? \@intersection : \@difference }, $element;
}
print OUT "INTERSECTION:"."number elements:[".@intersection."]:"."@intersection"."\n";
&computerecall;
print OUT "RECALL ON RECALL TEST SET:".@intersection."/". @recalltestsetids."=$divided\n";
print OUT "(recall test set is:$recalltestset)\n";
#print OUT "Ints reason : "."PozByPatt:"."@positivesbyeachpattern"."recallset"." @recalltestsetids"."\n";
#$ints=@intersection."\n";
#print STDERR "\n\nRECALL: $ints"."\\"."$number_recall"."\n";
#print OUT "WorstPrec (if all non eval were ng:".$yes/$count;
print OUT "\n\n======\n\n";
#print OUT "COMMENTS:"."@commetns";
%count=();
@intersection=();
}
open (OUTUNION, ">:utf8", "outunion.txt");
foreach $key (sort sortAsc keys %union){
print OUTUNION "$union{$key}";
}
@unionpositivesbyeachpattern= keys %unionYES;
foreach $element (@unionpositivesbyeachpattern, @recalltestsetids) { $count4union{$element}++ }
foreach $element (keys %count4union) {
push @{ $count4union{$element} > 1 ? \@intersection4union : \@difference4union }, $element;
}
&computerecall;
print OUT "\n\n\nALL PATTERNS, i.e. UNION\n";
print OUT "TOTAL EXAMPLES in UNION: ";
print OUT scalar keys %union; $scalarall= scalar keys %union;
print OUT "\n";
print OUT "POZITIVES:";
print OUT scalar keys %unionYES; $scalaryes= scalar keys %unionYES;
print OUT "\n";
print OUT "NEGATIVES:";
print OUT scalar keys %unionNO;
print OUT "\n";
print OUT "UNDEFINED:";
print OUT scalar keys %unionUNDEF; $scalarundefined= scalar keys %unionUNDEF;
print OUT "\n";
$scalardefined=$scalarall-$scalarundefined;
# print OUT "PRECISION_eval:". $scalaryes/ $scalardefined."\n";
print OUT "INTERSECTION:"."number elements4union:[".@intersection4union."]:"."@intersection4union"."\n";
if ($chosenrecall>0){
$iu=@intersection4union;
$rt=@recalltestsetids;
$dividedunion=$iu/$rt;
print OUT "RECALL ON RECALL TEST SET:".@intersection4union."/". @recalltestsetids."=$dividedunion\n";
print OUT "(recall test set is:$recalltestset)\n";
}
sub sortAsc {
$a<=>$b;
}
sub recalltestset {
print STDERR "you entered:[".$recalltestset."]\n";
# chomp $recalltestset;
$recalltestset=~s/\n*\r*$//g;
print STDERR "CHOMPED:: [".$recalltestset."]\n";
if ($recalltestset=~/.../){
$chosenrecall=1;
open(RCSET, "<:utf8", "$recalltestset") || die "Cannot open RCSET file:$!";
while (<RCSET>){
if (m/S sid_sp=\"(\d+)\"/){
push @recalltestsetids, $1;
}
}
}
else {print STDERR "\nok no test set for recall!\n";}
}
sub computerecall {
if ($chosenrecall>0){
$is=@intersection;
$rs=@recalltestsetids;
$divided=$is/$rs;
}
}
print STDERR "total number of recall testset: ".@recalltestsetids."\n";
definition_extraction/__init__.py
+1
View File
@@ -0,0 +1 @@
definition_extraction/app.py
+84
View File
@@ -0,0 +1,84 @@
import os
import json
import tempfile
from flask import Flask
from flask_restx import Api, Resource, fields
from werkzeug.middleware.proxy_fix import ProxyFix
from werkzeug.datastructures import FileStorage
from werkzeug.exceptions import BadRequest
try:
from . import extract_defsent as extractor
except ImportError:
import extract_defsent as extractor
UPLOAD_FOLDER = 'uploads'
MAX_TERMS_STRLEN = 100000
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
api = Api(app, version='1.0', title='Definition sentence extraction',
description='''A simple API for the extraction of definition sentence candidates from a given CoNLL-U file. A list of candidate terms (can also be empty) is used to filter input sentences.
**NOTE**: Because the service accepts a file and an additional parameter (terms), the request's _Content-Type_ cannot be ```application/json``` but ```multipart/form-data```.
As a result, the terms parameter must be a string in the POST request form data and must contain valid JSON.
''')
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
ns = api.namespace('DefExAPI', description='Definition extraction API namespace')
def terms_as_json(value):
'''Parses and validates terms stored in JSON string'''
if len(value) > MAX_TERMS_STRLEN:
raise ValueError(f'Security error: string too long! Must be of length <= {MAX_TERMS_STRLEN}.')
try:
lem_terms = json.loads(value)
if 'lemmatized_terms' not in lem_terms or not isinstance(lem_terms['lemmatized_terms'], list):
raise Exception('''Invalid JSON format for terms, must be like {"lemmatized_terms": ["first term", "second term", ...]}.''')
tlist = [str(x) for x in lem_terms['lemmatized_terms']]
except Exception as e:
raise ValueError(f'Error while parsing terms JSON string: {str(e)}')
else:
return tlist
# Swagger documentation
terms_as_json.__schema__ = {'type': str, 'format': 'JSON'}
parser = api.parser()
parser.add_argument('terms', type=terms_as_json, location='form', required=False)
parser.add_argument('conllu_file', type=FileStorage, location='files', required=True)
#@api.route('/with-parser/', endpoint='with-parser')
@ns.route('/definition_sentence_extraction')
@ns.expect(parser, validate=True)
@ns.doc(params={'terms': f'''This is a string contaning an _optional_ list of **lemmatized terms** for which we want to extract potential definition sentence candidates.
This string should contain a valid JSON where terms are under key **lemmatized_terms**, e.g.:
```{{"lemmatized_terms": ["first term", "second term", ...]}}```
**NOTE**: For security reasons, the length of the string is limited to {MAX_TERMS_STRLEN} characters.''',
'conllu_file': '''This is a mandatory parameter containing a valid CoNLL-U file.'''
})
class DefinitionSentenceExtractionService(Resource):
def post(self):
args = parser.parse_args()
terms = args['terms'] if args['terms'] is not None else []
conllu_filestorage = args['conllu_file']
tempdir = tempfile.TemporaryDirectory()
tmpfile = os.path.join(tempdir.name, 'input.conllu')
with open(tmpfile, 'w') as fp:
fp.write(conllu_filestorage.read().decode())
try:
sentences = extractor.mp_extract(tmpfile, terms)
except Exception as e:
raise BadRequest(str(e))
tempdir.cleanup()
return {'definition_candidates': sentences}
#if __name__ == '__main__':
#app.run(debug=True)
definition_extraction/entrypoint.prod.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
exec "$@"
definition_extraction/entrypoint.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
exec "$@"
definition_extraction/extract_defsent.py
+205
View File
@@ -0,0 +1,205 @@
import sys
import os
import shutil
import atexit
import tempfile
import subprocess
import argparse
import json
import time
import re
import multiprocessing as mp
import conllu
termex_perl = 'SentEx_patternsF.pl'
termex_pattern1 = 'JeStaSoPatterns_finishOK.txt'
input_fname = 'input.lines'
output_union = 'outunion.txt'
#def cleanup(tempdir):
#if isinstance(tempdir, tempfile.TemporaryDirectory):
#print(f'INFO: Removing temporary folder: {tempdir.name}', file=sys.stderr)
#tempdir.cleanup()
def words_search(words_string, sentence_string):
#words_string = " " + words_string.strip() + " "
#sentence_string = " " + sentence_string.strip() + " "
words_string = " " + words_string + " "
sentence_string = " " + sentence_string + " "
return sentence_string.find(words_string)
# this search works for searching single whitespace joined words in other single whitespace joined words
# See here (https://docs.python.org/3/library/re.html) for the definition of \b
# NOTE: this is slow!
def find_words_in_string(words, string):
if re.search(r"\b" + re.escape(words) + r"\b", string):
return True
return False
# returns all non-overlapping starting sublist positions
def sublist(lst, sub):
elt = sub[0]
pos = 0
allpos = []
while lst != []:
if elt in lst:
idx = lst.index(elt)
pos += idx
if lst[idx: idx + len(sub)] == sub:
allpos.append(pos)
lst = lst[idx + len(sub):]
pos += len(sub)
# return pos
else:
lst = lst[idx + 1:]
pos += 1
else:
break
# return -1
return allpos
# return -1
# returns first sublist position or -1
def is_sublist(lst, sub):
elt = sub[0]
pos = 0
while lst != []:
if elt in lst:
idx = lst.index(elt)
pos += idx
if lst[idx: idx + len(sub)] == sub:
return pos
else:
lst = lst[idx + 1:]
pos += 1
else:
return -1
return -1
def split_conllu_file(conllu_file, n):
tempdir = tempfile.TemporaryDirectory()
#print('-->', tempdir.name)
parts = [[] for i in range(n)]
with open(conllu_file, 'r', encoding="utf-8") as ifp:
for i, sent in enumerate(conllu.parse_incr(ifp)):
parts[i%n].append(sent)
outfiles = [os.path.join(tempdir.name, f'{i}.conllu') for i in range(n)]
for fname, sentences in zip(outfiles, parts):
with open(fname, 'w') as fp:
for s in sentences:
fp.write(s.serialize())
return tempdir, outfiles
def read_terms_json_file(fname):
with open(fname) as fp:
lem_terms = json.load(fp)
if 'lemmatized_terms' not in lem_terms or not isinstance(lem_terms['lemmatized_terms'], list):
raise Exception('''Invalid JSON format for terms, must be like {"lemmatized_terms": ["first term", "second term", ...]}.''')
return [str(x).strip() for x in lem_terms['lemmatized_terms']]
def extract_definition_sentences(conllu_file, terms=[]):
# initialize temp directory
tempdir = tempfile.TemporaryDirectory()
st = time.time()
# prepare input data in correct format
with open(conllu_file, 'r', encoding="utf-8") as ifp:
with open(os.path.join(tempdir.name, input_fname), 'w') as ofp:
for i, sent in enumerate(conllu.parse_incr(ifp)):
if terms != []:
lem_sent = ' '.join([tok['lemma'] for tok in sent])
for term in terms:
if term in lem_sent: # first test: string search, can lead to false results (substrings, not whole words)
term_tokens = term.split()
lem_tokens = [tok['lemma'] for tok in sent]
if is_sublist(lem_tokens, term_tokens) != -1: #slower exact testing
# if words_search(term, lem_sent) != -1:
# if is_sublist(lem_tokens, term) != -1:
# if find_words_in_string(term, lem_sent):
# if term in lem_sent:
#if len(term)>1:
# print('--->', term, lem_tokens)
for tok in sent:
print(f"{tok['form']}\tTOK\t{tok['lemma']}\t{tok['xpos']}", file=ofp)
sid = sent.metadata.get('sent_id', i)
print(f'''\t\t\t<S sid_sp="{sid}" aid_sp="{conllu_file}" defvalue="" term="{term}"/>\n''', file=ofp)
else:
for tok in sent:
print(f"{tok['form']}\tTOK\t{tok['lemma']}\t{tok['xpos']}", file=ofp)
sid = sent.metadata.get('sent_id', i)
print(f'''\t\t\t<S sid_sp="{sid}" aid_sp="{conllu_file}" defvalue=""/>\n''', file=ofp)
#print('Preparation and filtering time: ', time.time() - st)
st = time.time()
for fn in [termex_perl, termex_pattern1]:
shutil.copyfile(fn, os.path.join(tempdir.name, fn))
# run extractor
p = subprocess.run(['perl', termex_perl, input_fname, termex_pattern1],
stderr=subprocess.DEVNULL,
cwd=tempdir.name)
if p.returncode != 0:
raise IOError('Term extraction process failed, check its perl script.')
resultfile = os.path.join(tempdir.name, output_union)
if not os.path.exists(resultfile):
raise IOError('Output file with union of results does not exist: {resultfile}')
#print('Extraction time: ', time.time() - st)
lines = open(resultfile).read()
print(lines)
return [{
"definicija":line.split('###')[0].strip(),
"term": re.search('term="(.*)"', line).group(1)
} for line in lines.split('\n') if line.strip()]
def mp_extract(conllu, terms, ncores=os.cpu_count()):
defs = []
conllu_tempdir, conllu_part_files = split_conllu_file(conllu, ncores)
with mp.Pool(ncores) as pool:
params = zip(conllu_part_files, [terms] * len(conllu_part_files))
for i, result in enumerate(pool.starmap(extract_definition_sentences, params)):
print(result)
defs.extend(result)
#return sorted(list(set(defs)))
return defs;
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('CoNLLU_file', help='Input file in CoNLL-U format')
parser.add_argument('-n', '--ncpu', type=int, help='Number of CPU cores to use. Leave empty to use all cores.')
parser.add_argument('-t', '--terms', help='Optional input file with lemmatized terms for filtering the input file, formatted like this: {"lemmatized_terms": ["first term", "second term", ...]} ')
args = parser.parse_args()
#sents = extract_definition_sentences(args.CoNLLU_file)
#print('\n'.join(list(set(sents))))
ncores = args.ncpu if args.ncpu is not None else os.cpu_count()
lem_terms = read_terms_json_file(args.terms) if args.terms else []
defs = mp_extract(args.CoNLLU_file, lem_terms, ncores)
for s in defs:
print(s)
definition_extraction/requirements.txt
+6
View File
@@ -0,0 +1,6 @@
conllu==4.5.*
gunicorn==20.1.*
werkzeug==2.1.2
flask==2.1.3
flask-restx==0.5.1