Fix (pogostostpojavljanja is now array) for importing term lists, new version.

README.md

@@ -1 +1,53 @@
 # RSDO (Razvoj Slovenščine v Digitalnem Okolju) - Concordancer
+This repository contains the source code of the concordancer which is a part of the [Terminology Portal](https://github.com/clarinsi/rsdo_term_portal).
+
+There are two main parts:
+- Web application with backend (API interface) written in ASP.NET Core 6 and frontend written in React
+- System Manager which is written in .NET 6 and is used to initially set up the system
+
+## Prerequisites
+Web application requires following external dependencies
+- OpenSearch v2.3.0 (use docker image: `opensearchproject/opensearch:2.3.0`)
+- PostgreSQL (use docker image: `postgres`)
+
+## Building source code
+### Frontend (React application)
+To build a frontend application successfully, first, [Node.js](https://nodejs.org/en/) must be installed (at least version 16).
+Use `npm install` to install all required dependencies, and `npm run build` to build the application.
+For more info see the [Readme](src/apps/Rsdo.Concordancer.Api/WebApp/README.md) inside the frontend [root](src/apps/Rsdo.Concordancer.Api/WebApp) folder.
+
+### Backend (ASP.NET Core 6) and System Manager (.NET 6)
+Open [Rsdo.Concordancer.sln](src/Rsdo.Concordancer.sln) inside of your IDE and build the solution, or use `dotnet build` command to build the solution. All required dependencies will be installed via NuGet packages.
+
+## Creating docker images
+There are two Dockerfile's in the source repository for each part. Both images should be compiled within the context of the project root folder.
+
+### Web application
+[Dockerfile](src/apps/Rsdo.Concordancer.Api/Dockerfile) for Web Application accepts additional optional argument (WEBAPP_PUBLIC_URL) which is mapped to React [PUBLIC_URL](https://create-react-app.dev/docs/using-the-public-folder/) variable. If the variable is omited, then application should be deployed on a domain root.
+
+`docker build -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .`
+
+`docker build --build-arg WEBAPP_PUBLIC_URL=/corpus -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .`
+
+### System Manager
+Use following [Dockerfile](src/apps/Rsdo.Concordancer.SystemManager/Dockerfile) to build System Manager.
+
+`docker build -t rsdo-concordancer-systemmanager -f src/apps/Rsdo.Concordancer.SystemManager/Dockerfile .`
+
+## Environment variables
+
+### Web application
+Web application must be configured with the following Environment variables and values:
+- `RSDO:Database:MasterConnectionString` holds connection string to the main database; e.g.: `rsdo_db;Username=rsdo;Password=strongpassword;Database=Concordancer`
+- `RSDO:Elastic:ConnectionString` holds connection string to OpenSearch server; e.g.: `http://rsdo_opensearch:9200`
+- `RSDO:Tokenizer:ClasslaUrl` holds and url to external service which is used for tokenizing search strings; e.g.: `https://orodja.cjvt.si/oznacevalnik/ajax_api/v1/slv/process`
+
+### System Manager
+System manager must be configured with the following Environment variables and values:
+- `RSDO:Database:MasterConnectionString` holds connection string to the main database; e.g.: `rsdo_db;Username=rsdo;Password=strongpassword;Database=Concordancer`
+- `RSDO:Elastic:ConnectionString` holds connection string to OpenSearch server; e.g.: `http://rsdo_opensearch:9200`
+
+## Using System Manager
+To initialy setup the system, you have to use System Manager and run following commands:
+1. `dotnet Rsdo.Concordancer.SystemManager.dll createMasterDb --connectionString="Host=rsdo_db;Username=rsdo;Password=strongpassword;Database=postgres"` which will create master database with the name Concordancer. You can select different name with the `--name` option
+2. `dotnet Rsdo.Concordancer.SystemManager.dll importSloleks --sourceFile=/data/Sloleks2.0.LMF/sloleks_clarin_2.0.xml` which will import Sloleks (which is used for better searching). XML file is not part of this repository, but it can be downloaded from the following [URL](https://www.clarin.si/repository/xmlui/bitstream/handle/11356/1230/Sloleks2.0.LMF.zip?sequence=3&isAllowed=y)

src/apps/Rsdo.Concordancer.Api/Controllers/WebAppController.cs

@@ -1,42 +0,0 @@
-using System;
-using System.IO;
-using System.Linq;
-using System.Net.Mime;
-using HtmlAgilityPack;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Configuration;
-using Rsdo.Concordancer.Core.Constants;
-
-namespace Rsdo.Concordancer.Api.Controllers
-{
-    public class WebAppController : Controller
-    {
-        private readonly IConfiguration configuration;
-        private readonly IWebHostEnvironment webHostEnvironment;
-
-        public WebAppController(IConfiguration configuration, IWebHostEnvironment webHostEnvironment)
-        {
-            this.configuration = configuration;
-            this.webHostEnvironment = webHostEnvironment;
-        }
-
-        public IActionResult Index()
-        {
-            var indexFilePath = webHostEnvironment.WebRootFileProvider.GetFileInfo("index.html").PhysicalPath;
-            var html = new HtmlDocument();
-            html.Load(indexFilePath);
-
-            var baseAppPath = configuration[ConfigurationKey.Web.BaseAppPath] ?? string.Empty;
-            var scriptNode = HtmlNode.CreateNode($"<script>window.baseAppPath = '{baseAppPath}';</script>");
-            var headNode = html.DocumentNode.SelectSingleNode("//head");
-            headNode.ChildNodes.Add(scriptNode);
-
-            var indexStream = new MemoryStream();
-            html.Save(indexStream);
-            indexStream.Position = 0;
-
-            return File(indexStream, MediaTypeNames.Text.Html);
-        }
-    }
-}

src/apps/Rsdo.Concordancer.Api/Dockerfile

@@ -4,6 +4,12 @@ EXPOSE 80
 
 FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build
 
+# INSTALL NODE.JS
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
+    && apt-get install -y \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
 # COPY LOCAL NUGET PACKAGES
 WORKDIR /nuget
 COPY local-packages/. .
@@ -36,8 +42,14 @@ COPY src/core/Rsdo.Concordancer.Infrastructure/. Rsdo.Concordancer.Infrastructur
 COPY src/core/Rsdo.Concordancer.ServiceModel/. Rsdo.Concordancer.ServiceModel/
 COPY src/core/Rsdo.Concordancer.Services/. Rsdo.Concordancer.Services/
 
-# BUILD
+# BUILD WEBAPP AND AFTER THAT API, WHICH INCLUDES WEBAPP
 FROM build AS publish
+
+WORKDIR /src/apps/Rsdo.Concordancer.Api/WebApp
+ARG WEBAPP_PUBLIC_URL=''
+ENV PUBLIC_URL=${WEBAPP_PUBLIC_URL}
+RUN npm install && npm run build
+
 WORKDIR /src/apps/Rsdo.Concordancer.Api
 RUN dotnet publish "Rsdo.Concordancer.Api.csproj" -c Release -o /app/publish
 

src/apps/Rsdo.Concordancer.Api/Rsdo.Concordancer.Api.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.2</Version>
     </PropertyGroup>
 
     <ItemGroup>
@@ -12,7 +12,6 @@
         <PackageReference Include="Autofac.Extensions.DependencyInjection" Version="8.0.0" />
         <PackageReference Include="Hangfire" Version="1.7.31" />
         <PackageReference Include="Hangfire.PostgreSql" Version="1.9.9" />
-        <PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
         <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.10" />
         <PackageReference Include="Rsdo.StyleCop" Version="1.0.0" />
         <PackageReference Include="Serilog.AspNetCore" Version="6.0.1" />

src/apps/Rsdo.Concordancer.Api/Startup.cs

@@ -144,7 +144,7 @@ public class Startup
             {
                 endpoints.MapControllers();
                 endpoints.MapHangfireDashboard("/hangfire");
-                endpoints.MapFallbackToController("Index", "WebApp");
+                endpoints.MapFallbackToFile("index.html");
             });
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "concordancer-web",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "concordancer-web",
-      "version": "0.1.0",
+      "version": "1.0.0",
       "dependencies": {
         "@testing-library/jest-dom": "^5.16.5",
         "@testing-library/react": "^13.4.0",

src/apps/Rsdo.Concordancer.Api/WebApp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "concordancer-web",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "private": true,
   "dependencies": {
     "@testing-library/jest-dom": "^5.16.5",

src/apps/Rsdo.Concordancer.Api/WebApp/public/index.html

@@ -5,10 +5,7 @@
     <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="theme-color" content="#000000" />
-    <meta
+    <base href="%PUBLIC_URL%/" />
-      name="description"
-      content="Web site created using create-react-app"
-    />
     <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
     <!--
       manifest.json provides metadata used when your web app is installed on a

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetails.js

@@ -20,7 +20,7 @@ const ConcordanceDetails = (props) => {
     const search = useSearch();
     const body = search.getDetailsRequest(props.paragraphId, props.tokenOrder);
 
-    const fetchDetails = () => fetch(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/concordances/details`, {
+    const fetchDetails = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/concordances/details`, {
         method: 'POST',
         body: JSON.stringify(body),
         headers: {

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/index/IndexStatistics.js

@@ -9,7 +9,7 @@ const IndexStatistics = () => {
     const search = useSearch();
     const corpusId = search.corpusId;
 
-    const fetchStatistics = () => fetch(`${window.baseAppPath}/concordancer/corpus/${corpusId}/stats`, {
+    const fetchStatistics = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${corpusId}/stats`, {
         method: 'GET',
         headers: {
             'Content-Type': 'application/json'

src/apps/Rsdo.Concordancer.Api/WebApp/src/index.js

@@ -13,10 +13,11 @@ if (language) {
   i18n.changeLanguage(language);
 }
 
+const baseUrl = document.getElementsByTagName('base')[0].getAttribute('href');
 const root = ReactDOM.createRoot(document.getElementById('root'));
 root.render(
   <React.StrictMode>
-    <BrowserRouter basename={window.baseAppPath}>
+    <BrowserRouter basename={baseUrl}>
       <App />
     </BrowserRouter>
   </React.StrictMode>

src/apps/Rsdo.Concordancer.Api/WebApp/src/pages/concordance/ConcordanceResults.js

@@ -21,7 +21,7 @@ const ConcordanceResults = () => {
 
     const body = search.getSearchRequest();
 
-    const fetchConcordances = () => fetch(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/concordances/search`, {
+    const fetchConcordances = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/concordances/search`, {
         method: 'POST',
         body: JSON.stringify(body),
         headers: {
@@ -34,7 +34,7 @@ const ConcordanceResults = () => {
 
     const { isLoading, isSuccess, isError, data, error } = useQuery(['concordances', body], fetchConcordances, { keepPreviousData: true, refetchOnWindowFocus: false });
     const [isVisible, setIsVisible] = useState(false);
-    const download = useDownload(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/concordances/export`);
+    const download = useDownload(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/concordances/export`);
 
     const onExportClose = () => {
         setIsVisible(false);

src/apps/Rsdo.Concordancer.Api/WebApp/src/pages/list/ListResults.js

@@ -20,7 +20,7 @@ const ListResults = () => {
     const history = useHistory();
     const body = search.getSearchRequest();
 
-    const fetchList = () => fetch(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/list/search`, {
+    const fetchList = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/list/search`, {
         method: 'POST',
         body: JSON.stringify(body),
         headers: {
@@ -33,7 +33,7 @@ const ListResults = () => {
 
     const { isLoading, isSuccess, isError, data, error } = useQuery(['list', body], fetchList, { keepPreviousData: true, refetchOnWindowFocus: false });
     const [isVisible, setIsVisible] = useState(false);
-    const download = useDownload(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/list/export`);
+    const download = useDownload(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/list/export`);
 
     const onExportClose = () => {
         setIsVisible(false);

src/apps/Rsdo.Concordancer.SystemManager/Rsdo.Concordancer.SystemManager.csproj

@@ -5,7 +5,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.2</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Core/Constants/ConfigurationKey.cs

@@ -16,9 +16,4 @@ public static class ConfigurationKey
     {
         public const string ClasslaTokenizerUrl = "RSDO:Tokenizer:ClasslaUrl";
     }
-
-    public static class Web
-    {
-        public const string BaseAppPath = "RSDO:Web:BaseAppPath";
-    }
 }

src/core/Rsdo.Concordancer.Core/Rsdo.Concordancer.Core.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.2</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Data/Rsdo.Concordancer.Data.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.2</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Infrastructure/Rsdo.Concordancer.Infrastructure.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.2</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Infrastructure/Search/AddRecordsHandlers/BaseAddRecordsHandler.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using OpenSearch.Client;
 using Rsdo.Concordancer.Infrastructure.Search.Extensions;
@@ -27,6 +28,9 @@ public abstract class BaseAddRecordsHandler<TEntity, TElasticEntity> : IAddRecor
         // Get index name
         var indexName = indexProvider.IndexName;
 
+        // Batch entities by 1000
+        foreach (var batch in entities.Chunk(1000))
+        {
             // Create request
             var request = new BulkRequest(indexName)
             {
@@ -35,7 +39,7 @@ public abstract class BaseAddRecordsHandler<TEntity, TElasticEntity> : IAddRecor
             };
 
             // Convert entities to elastic entities
-        foreach (var entity in entities)
+            foreach (var entity in batch)
             {
                 var elasticEntity = ConvertEntity(entity);
                 request.Operations.Add(new BulkIndexOperation<TElasticEntity>(elasticEntity));
@@ -45,6 +49,7 @@ public abstract class BaseAddRecordsHandler<TEntity, TElasticEntity> : IAddRecor
             var response = await client.BulkAsync(request);
             response.ThrowIfInvalid();
         }
+    }
 
     protected abstract TElasticEntity ConvertEntity(TEntity entity);
 }

src/core/Rsdo.Concordancer.ServiceModel/Rsdo.Concordancer.ServiceModel.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.2</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Services/RequestHandlers/TermLists/ImportTermListHandler.cs

@@ -109,7 +109,7 @@ public class ImportTermListHandler : IRequestHandler<ImportTermList, ExecutionRe
                         Lemma = termObj["kandidat"].Value<string>(),
                         Msd = termObj["POSoznake"].Value<string>(),
                         Weight = termObj["ranking"].Value<decimal>(),
-                        Frequency = termObj["pogostostpojavljanja"].Value<int>(),
+                        Frequency = termObj["pogostostpojavljanja"].Values<int>().First(),
                         TermListAutoId = termList.AutoId,
                     }.ApplyCreateValues();
 

src/core/Rsdo.Concordancer.Services/RequestHandlers/Texts/IndexTextHandler.cs

@@ -90,39 +90,35 @@ public class IndexTextHandler : IRequestHandler<IndexText, ExecutionResult>
 
     private static List<Concordance> GetConcordances(Text text, Paragraph paragraph, List<Token> tokens)
     {
-        // Get window size (max 10)
-        var windowSize = Math.Min(tokens.Count - 1, 10);
-
-        // Create list of tokens with positions
-        var tokensIdx = tokens.Select((t, i) => new KeyValuePair<int, Token>(i, t)).ToList();
-
-        // Get tokens which will appear in current concordance
-        List<KeyValuePair<int, Token>> concordanceTokens;
-
         var concordances = new List<Concordance>();
-        while ((concordanceTokens = tokensIdx.Where(x => x.Key >= -windowSize && x.Key <= windowSize).ToList().OrderBy(x => x.Key).ToList()).Any())
+        for (var i = 0; i < tokens.Count; i++)
         {
-            if (concordanceTokens.Last().Key < 0)
-            {
-                break;
-            }
-
             var concordance = new Concordance()
             {
                 ParagraphId = paragraph.Id,
                 TextId = text.Id,
             };
 
-            // Loop through tokens and set it in the position
+            // set center token
-            foreach (var concordanceToken in concordanceTokens)
+            concordance.SetToken(tokens[i], 0);
+
+            // set left and right context tokens
+            for (var c = 1; c <= 10; c++)
             {
-                concordance.SetToken(concordanceToken.Value, concordanceToken.Key);
+                // left context
+                if (i - c >= 0)
+                {
+                    concordance.SetToken(tokens[i - c], -c);
+                }
+
+                // right context
+                if (i + c < tokens.Count)
+                {
+                    concordance.SetToken(tokens[i + c], c);
+                }
             }
 
             concordances.Add(concordance);
-
-            // Decrease indexes of tokens (shift tokens to left, relative to window size)
-            tokensIdx = tokensIdx.Select(x => new KeyValuePair<int, Token>(x.Key - 1, x.Value)).ToList();
         }
 
         return concordances;

src/core/Rsdo.Concordancer.Services/Rsdo.Concordancer.Services.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.2</Version>
     </PropertyGroup>
 
     <ItemGroup>