webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

limit ngrams to 200 because this temporary implementation is kind of slow

Browse Source
This commit is contained in:
Kikimanox
2022-08-29 23:35:22 +02:00
parent 246c60de02
commit 5bdba94035
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swagger_server/util.py
+1 -1
View File
@@ -177,7 +177,7 @@ def get_files_by_keywords(kljucnebesede):
ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/" ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/"
print("Looping trough ngrams") print("Looping trough ngrams")
for path, dirs, files, in os.walk(ngrams_path): for path, dirs, files, in os.walk(ngrams_path):
for i, _file in enumerate(files): for i, _file in enumerate(files[:200]):
if i % 200 == 0: print(f"{i}/{len(files)}") if i % 200 == 0: print(f"{i}/{len(files)}")
file = f'{ngrams_path}{_file}' file = f'{ngrams_path}{_file}'
data = pd.read_csv(file, sep='\t') data = pd.read_csv(file, sep='\t')