Bumping up version for Docker containers.

README.md

@@ -1 +1,53 @@
 # RSDO (Razvoj Slovenščine v Digitalnem Okolju) - Concordancer
+This repository contains the source code of the concordancer which is a part of the [Terminology Portal](https://github.com/clarinsi/rsdo_term_portal).
+
+There are two main parts:
+- Web application with backend (API interface) written in ASP.NET Core 6 and frontend written in React
+- System Manager which is written in .NET 6 and is used to initially set up the system
+
+## Prerequisites
+Web application requires following external dependencies
+- OpenSearch v2.3.0 (use docker image: `opensearchproject/opensearch:2.3.0`)
+- PostgreSQL (use docker image: `postgres`)
+
+## Building source code
+### Frontend (React application)
+To build a frontend application successfully, first, [Node.js](https://nodejs.org/en/) must be installed (at least version 16).
+Use `npm install` to install all required dependencies, and `npm run build` to build the application.
+For more info see the [Readme](src/apps/Rsdo.Concordancer.Api/WebApp/README.md) inside the frontend [root](src/apps/Rsdo.Concordancer.Api/WebApp) folder.
+
+### Backend (ASP.NET Core 6) and System Manager (.NET 6)
+Open [Rsdo.Concordancer.sln](src/Rsdo.Concordancer.sln) inside of your IDE and build the solution, or use `dotnet build` command to build the solution. All required dependencies will be installed via NuGet packages.
+
+## Creating docker images
+There are two Dockerfile's in the source repository for each part. Both images should be compiled within the context of the project root folder.
+
+### Web application
+[Dockerfile](src/apps/Rsdo.Concordancer.Api/Dockerfile) for Web Application accepts additional optional argument (WEBAPP_PUBLIC_URL) which is mapped to React [PUBLIC_URL](https://create-react-app.dev/docs/using-the-public-folder/) variable. If the variable is omited, then application should be deployed on a domain root.
+
+`docker build -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .`
+
+`docker build --build-arg WEBAPP_PUBLIC_URL=/corpus -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .`
+
+### System Manager
+Use following [Dockerfile](src/apps/Rsdo.Concordancer.SystemManager/Dockerfile) to build System Manager.
+
+`docker build -t rsdo-concordancer-systemmanager -f src/apps/Rsdo.Concordancer.SystemManager/Dockerfile .`
+
+## Environment variables
+
+### Web application
+Web application must be configured with the following Environment variables and values:
+- `RSDO:Database:MasterConnectionString` holds connection string to the main database; e.g.: `rsdo_db;Username=rsdo;Password=strongpassword;Database=Concordancer`
+- `RSDO:Elastic:ConnectionString` holds connection string to OpenSearch server; e.g.: `http://rsdo_opensearch:9200`
+- `RSDO:Tokenizer:ClasslaUrl` holds and url to external service which is used for tokenizing search strings; e.g.: `https://orodja.cjvt.si/oznacevalnik/ajax_api/v1/slv/process`
+
+### System Manager
+System manager must be configured with the following Environment variables and values:
+- `RSDO:Database:MasterConnectionString` holds connection string to the main database; e.g.: `rsdo_db;Username=rsdo;Password=strongpassword;Database=Concordancer`
+- `RSDO:Elastic:ConnectionString` holds connection string to OpenSearch server; e.g.: `http://rsdo_opensearch:9200`
+
+## Using System Manager
+To initialy setup the system, you have to use System Manager and run following commands:
+1. `dotnet Rsdo.Concordancer.SystemManager.dll createMasterDb --connectionString="Host=rsdo_db;Username=rsdo;Password=strongpassword;Database=postgres"` which will create master database with the name Concordancer. You can select different name with the `--name` option
+2. `dotnet Rsdo.Concordancer.SystemManager.dll importSloleks --sourceFile=/data/Sloleks2.0.LMF/sloleks_clarin_2.0.xml` which will import Sloleks (which is used for better searching). XML file is not part of this repository, but it can be downloaded from the following [URL](https://www.clarin.si/repository/xmlui/bitstream/handle/11356/1230/Sloleks2.0.LMF.zip?sequence=3&isAllowed=y)

build-docker.bat

@@ -0,0 +1,3 @@
+docker build --no-cache -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .
+docker build --no-cache --build-arg WEBAPP_PUBLIC_URL=/korpus -t rsdo-concordancer-api-term-portal -f src/apps/Rsdo.Concordancer.Api/Dockerfile .
+docker build --no-cache -t rsdo-concordancer-systemmanager -f src/apps/Rsdo.Concordancer.SystemManager/Dockerfile .

build-publish-docker.bat

@@ -0,0 +1,24 @@
+docker build --no-cache -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .
+docker build --no-cache --build-arg WEBAPP_PUBLIC_URL=/korpus -t rsdo-concordancer-api-term-portal -f src/apps/Rsdo.Concordancer.Api/Dockerfile .
+docker build --no-cache -t rsdo-concordancer-systemmanager -f src/apps/Rsdo.Concordancer.SystemManager/Dockerfile .
+
+docker login ghcr.io
+
+docker tag rsdo-concordancer-api ghcr.io/clarinsi/rsdo-concordancer-api:latest
+docker tag rsdo-concordancer-api ghcr.io/clarinsi/rsdo-concordancer-api:v1.0.4
+docker tag rsdo-concordancer-api ghcr.io/clarinsi/rsdo-concordancer-api:v1.0
+docker tag rsdo-concordancer-api ghcr.io/clarinsi/rsdo-concordancer-api:v1
+
+docker tag rsdo-concordancer-api-term-portal ghcr.io/clarinsi/rsdo-concordancer-api-term-portal:latest
+docker tag rsdo-concordancer-api-term-portal ghcr.io/clarinsi/rsdo-concordancer-api-term-portal:v1.0.4
+docker tag rsdo-concordancer-api-term-portal ghcr.io/clarinsi/rsdo-concordancer-api-term-portal:v1.0
+docker tag rsdo-concordancer-api-term-portal ghcr.io/clarinsi/rsdo-concordancer-api-term-portal:v1
+
+docker tag rsdo-concordancer-systemmanager ghcr.io/clarinsi/rsdo-concordancer-systemmanager:latest
+docker tag rsdo-concordancer-systemmanager ghcr.io/clarinsi/rsdo-concordancer-systemmanager:v1.0.4
+docker tag rsdo-concordancer-systemmanager ghcr.io/clarinsi/rsdo-concordancer-systemmanager:v1.0
+docker tag rsdo-concordancer-systemmanager ghcr.io/clarinsi/rsdo-concordancer-systemmanager:v1
+
+docker push --all-tags ghcr.io/clarinsi/rsdo-concordancer-api
+docker push --all-tags ghcr.io/clarinsi/rsdo-concordancer-api-term-portal
+docker push --all-tags ghcr.io/clarinsi/rsdo-concordancer-systemmanager

src/apps/Rsdo.Concordancer.Api/Controllers/WebAppController.cs

@@ -1,42 +0,0 @@
-using System;
-using System.IO;
-using System.Linq;
-using System.Net.Mime;
-using HtmlAgilityPack;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Configuration;
-using Rsdo.Concordancer.Core.Constants;
-
-namespace Rsdo.Concordancer.Api.Controllers
-{
-    public class WebAppController : Controller
-    {
-        private readonly IConfiguration configuration;
-        private readonly IWebHostEnvironment webHostEnvironment;
-
-        public WebAppController(IConfiguration configuration, IWebHostEnvironment webHostEnvironment)
-        {
-            this.configuration = configuration;
-            this.webHostEnvironment = webHostEnvironment;
-        }
-
-        public IActionResult Index()
-        {
-            var indexFilePath = webHostEnvironment.WebRootFileProvider.GetFileInfo("index.html").PhysicalPath;
-            var html = new HtmlDocument();
-            html.Load(indexFilePath);
-
-            var baseAppPath = configuration[ConfigurationKey.Web.BaseAppPath] ?? string.Empty;
-            var scriptNode = HtmlNode.CreateNode($"<script>window.baseAppPath = '{baseAppPath}';</script>");
-            var headNode = html.DocumentNode.SelectSingleNode("//head");
-            headNode.ChildNodes.Add(scriptNode);
-
-            var indexStream = new MemoryStream();
-            html.Save(indexStream);
-            indexStream.Position = 0;
-
-            return File(indexStream, MediaTypeNames.Text.Html);
-        }
-    }
-}

src/apps/Rsdo.Concordancer.Api/Dockerfile

@@ -4,6 +4,12 @@ EXPOSE 80
 
 FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build
 
+# INSTALL NODE.JS
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
+    && apt-get install -y \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
 # COPY LOCAL NUGET PACKAGES
 WORKDIR /nuget
 COPY local-packages/. .
@@ -36,8 +42,14 @@ COPY src/core/Rsdo.Concordancer.Infrastructure/. Rsdo.Concordancer.Infrastructur
 COPY src/core/Rsdo.Concordancer.ServiceModel/. Rsdo.Concordancer.ServiceModel/
 COPY src/core/Rsdo.Concordancer.Services/. Rsdo.Concordancer.Services/
 
-# BUILD
+# BUILD WEBAPP AND AFTER THAT API, WHICH INCLUDES WEBAPP
 FROM build AS publish
+
+WORKDIR /src/apps/Rsdo.Concordancer.Api/WebApp
+ARG WEBAPP_PUBLIC_URL=''
+ENV PUBLIC_URL=${WEBAPP_PUBLIC_URL}
+RUN npm install && npm run build
+
 WORKDIR /src/apps/Rsdo.Concordancer.Api
 RUN dotnet publish "Rsdo.Concordancer.Api.csproj" -c Release -o /app/publish
 

src/apps/Rsdo.Concordancer.Api/Rsdo.Concordancer.Api.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.4</Version>
     </PropertyGroup>
 
     <ItemGroup>
@@ -12,7 +12,6 @@
         <PackageReference Include="Autofac.Extensions.DependencyInjection" Version="8.0.0" />
         <PackageReference Include="Hangfire" Version="1.7.31" />
         <PackageReference Include="Hangfire.PostgreSql" Version="1.9.9" />
-        <PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
         <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.10" />
         <PackageReference Include="Rsdo.StyleCop" Version="1.0.0" />
         <PackageReference Include="Serilog.AspNetCore" Version="6.0.1" />

src/apps/Rsdo.Concordancer.Api/Startup.cs

@@ -144,7 +144,7 @@ public class Startup
             {
                 endpoints.MapControllers();
                 endpoints.MapHangfireDashboard("/hangfire");
-                endpoints.MapFallbackToController("Index", "WebApp");
+                endpoints.MapFallbackToFile("index.html");
             });
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "concordancer-web",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "concordancer-web",
-      "version": "0.1.0",
+      "version": "1.0.0",
       "dependencies": {
         "@testing-library/jest-dom": "^5.16.5",
         "@testing-library/react": "^13.4.0",

src/apps/Rsdo.Concordancer.Api/WebApp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "concordancer-web",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "private": true,
   "dependencies": {
     "@testing-library/jest-dom": "^5.16.5",

src/apps/Rsdo.Concordancer.Api/WebApp/public/index.html

@@ -5,10 +5,7 @@
     <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="theme-color" content="#000000" />
-    <meta
+    <base href="%PUBLIC_URL%/" />
-      name="description"
-      content="Web site created using create-react-app"
-    />
     <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
     <!--
       manifest.json provides metadata used when your web app is installed on a

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAggregationItem.js

@@ -8,7 +8,7 @@ const ConcordanceAggregationItem = (props) => {
     const link = isSelected ? search.getClearFilterLink('TextIds', props.filterKey) : search.getFilterLink('TextIds', props.filterKey);
 
     return (
-        <li className={styles.aggregationItem}><Link to={link}>{isSelected && <span>X - </span>}{props.title}<span className='float-end'>{props.count}</span></Link></li>
+        <li className={styles.aggregationItem}><Link to={link}>{isSelected && <span className={styles.remove} >X</span>}<span className={styles.title}>{props.title}</span><span className={styles.count}>{props.count}</span></Link></li>
     );
 };
 

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAggregationItem.module.scss

@@ -7,6 +7,18 @@
 
     a {
         color: $text-color;
+        display: flex;
         text-decoration: none;
+
+        .remove {
+            margin-right: 0.8rem;
+        }
+
+        .title {
+            flex-grow: 1;
+            margin-right: 0.8rem;
+            overflow: hidden;
+            text-overflow: clip;
+        }
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAlternateSearchItem.js

@@ -6,7 +6,7 @@ const ConcordanceAlternateSearchItem = (props) => {
     const search = useSearch();
     const link = search.getAlternateSearchLink(props.search);
     return (
-        <li className={styles.alternateSearchItem}><Link to={link}>{props.title}<span className='float-end'>{props.count}</span></Link></li>
+        <li className={styles.alternateSearchItem}><Link to={link}><span className={styles.title}>{props.title}</span><span className={styles.count}>{props.count}</span></Link></li>
     );
 };
 

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAlternateSearchItem.module.scss

@@ -7,6 +7,14 @@
 
     a {
         color: $text-color;
+        display: flex;
         text-decoration: none;
+
+        .title {
+            flex-grow: 1;
+            margin-right: 0.8rem;
+            overflow: hidden;
+            text-overflow: clip;
+        }
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetails.js

@@ -20,7 +20,7 @@ const ConcordanceDetails = (props) => {
     const search = useSearch();
     const body = search.getDetailsRequest(props.paragraphId, props.tokenOrder);
 
-    const fetchDetails = () => fetch(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/concordances/details`, {
+    const fetchDetails = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/concordances/details`, {
         method: 'POST',
         body: JSON.stringify(body),
         headers: {

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetailsText.js

@@ -1,6 +1,6 @@
-import { useTranslation } from 'react-i18next';
+import ConcordanceDetailsTextInfo from './ConcordanceDetailsTextInfo';
 
-import sytles from './ConcordanceDetailsText.module.scss';
+import { useTranslation } from 'react-i18next';
 
 const ConcordanceDetailsText = (props) => {
     // Hooks
@@ -8,22 +8,10 @@ const ConcordanceDetailsText = (props) => {
 
     return (
         <>
-            <div className={sytles.textInfo}>
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextSource')} value={props.source} />
-                <h5>{t('concordance.detailsTextSource')}</h5>
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextYear')} value={props.year} />
-                <p>{props.source}</p>
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextTitle')} value={props.title} />
-            </div>
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextAuthor')} value={props.author} />
-            <div className={sytles.textInfo}>
-                <h5>{t('concordance.detailsTextYear')}</h5>
-                <p>{props.year}</p>
-            </div>
-            <div className={sytles.textInfo}>
-                <h5>{t('concordance.detailsTextTitle')}</h5>
-                <p>{props.title}</p>
-            </div>
-            <div className={sytles.textInfo}>
-                <h5>{t('concordance.detailsTextAuthor')}</h5>
-                <p>{props.author}</p>
-            </div>
         </>
     );
 };

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetailsTextInfo.js

@@ -0,0 +1,15 @@
+import styles from './ConcordanceDetailsTextInfo.module.scss';
+
+const ConcordanceDetailsTextInfo = (props) => {
+    var title = props.title;
+    var value = props.value;
+
+    if (value) {
+        return (<div className={styles.textInfo}>
+            <h5>{title}</h5>
+            <p>{value}</p>
+        </div>);
+    }
+};
+
+export default ConcordanceDetailsTextInfo;

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetailsTextInfo.module.scss

@@ -18,5 +18,7 @@
 
     p {
         margin: 0;
+        text-overflow: clip;
+        overflow: hidden;
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceListItem.js

@@ -6,8 +6,8 @@ const ConcordanceListItem = (props) => {
     return (
         <>
             <div className={`row ${styles.item}`} onClick={() => { props.toggleDetails(props.index) }}>
-                <div className='col text-end'><ConcordanceTokens tokens={props.item.leftContext} />&nbsp;</div>
+                <div className={`col ${styles.left}`}><span><ConcordanceTokens tokens={props.item.leftContext} />&nbsp;</span></div>
-                <div className='col'><ConcordanceTokens tokens={[props.item.centerContext]} /> <ConcordanceTokens tokens={props.item.rightContext} /></div>
+                <div className={`col ${styles.right}`}><ConcordanceTokens tokens={[props.item.centerContext]} /> <ConcordanceTokens tokens={props.item.rightContext} /></div>
             </div>
             {props.expanded && <ConcordanceDetails paragraphId={props.item.paragraphId} tokenOrder={props.item.centerContext.tokenOrder} onClose={() => { props.toggleDetails(null) }} />}
         </>

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceListItem.module.scss

@@ -5,4 +5,24 @@
     cursor: pointer;
     margin-bottom: 0.5rem;
     padding-bottom: 0.5rem;
+
+    :global(.col) {
+
+        &.left,
+        &.right {
+            overflow: hidden;
+            white-space: nowrap;
+            text-overflow: clip;
+        }
+
+        &.left {
+            span {
+                display: inline-block;
+                float: right;
+                min-width: 100%;
+                overflow: visible;
+                text-align: right;
+            }
+        }
+    }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/index/IndexStatistics.js

@@ -9,7 +9,7 @@ const IndexStatistics = () => {
     const search = useSearch();
     const corpusId = search.corpusId;
 
-    const fetchStatistics = () => fetch(`${window.baseAppPath}/concordancer/corpus/${corpusId}/stats`, {
+    const fetchStatistics = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${corpusId}/stats`, {
         method: 'GET',
         headers: {
             'Content-Type': 'application/json'

src/apps/Rsdo.Concordancer.Api/WebApp/src/hooks/use-download.js

@@ -12,7 +12,7 @@ const useDownload = (url) => {
                 var url = window.URL.createObjectURL(blob);
                 var a = document.createElement('a');
                 a.href = url;
-                a.download = "export.csv";
+                a.download = "export.txt";
                 document.body.appendChild(a); // we need to append the element to the dom -> otherwise it will not work in firefox
                 a.click();
                 a.remove();  //afterwards we remove the element again                       

src/apps/Rsdo.Concordancer.Api/WebApp/src/index.js

@@ -13,10 +13,11 @@ if (language) {
   i18n.changeLanguage(language);
 }
 
+const baseUrl = document.getElementsByTagName('base')[0].getAttribute('href');
 const root = ReactDOM.createRoot(document.getElementById('root'));
 root.render(
   <React.StrictMode>
-    <BrowserRouter basename={window.baseAppPath}>
+    <BrowserRouter basename={baseUrl}>
       <App />
     </BrowserRouter>
   </React.StrictMode>

src/apps/Rsdo.Concordancer.Api/WebApp/src/pages/concordance/ConcordanceResults.js

@@ -21,7 +21,7 @@ const ConcordanceResults = () => {
 
     const body = search.getSearchRequest();
 
-    const fetchConcordances = () => fetch(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/concordances/search`, {
+    const fetchConcordances = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/concordances/search`, {
         method: 'POST',
         body: JSON.stringify(body),
         headers: {
@@ -34,7 +34,7 @@ const ConcordanceResults = () => {
 
     const { isLoading, isSuccess, isError, data, error } = useQuery(['concordances', body], fetchConcordances, { keepPreviousData: true, refetchOnWindowFocus: false });
     const [isVisible, setIsVisible] = useState(false);
-    const download = useDownload(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/concordances/export`);
+    const download = useDownload(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/concordances/export`);
 
     const onExportClose = () => {
         setIsVisible(false);

src/apps/Rsdo.Concordancer.Api/WebApp/src/pages/list/ListResults.js

@@ -20,7 +20,7 @@ const ListResults = () => {
     const history = useHistory();
     const body = search.getSearchRequest();
 
-    const fetchList = () => fetch(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/list/search`, {
+    const fetchList = () => fetch(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/list/search`, {
         method: 'POST',
         body: JSON.stringify(body),
         headers: {
@@ -33,7 +33,7 @@ const ListResults = () => {
 
     const { isLoading, isSuccess, isError, data, error } = useQuery(['list', body], fetchList, { keepPreviousData: true, refetchOnWindowFocus: false });
     const [isVisible, setIsVisible] = useState(false);
-    const download = useDownload(`${window.baseAppPath}/concordancer/corpus/${search.corpusId}/list/export`);
+    const download = useDownload(`${process.env.PUBLIC_URL}/concordancer/corpus/${search.corpusId}/list/export`);
 
     const onExportClose = () => {
         setIsVisible(false);

src/apps/Rsdo.Concordancer.SystemManager/Rsdo.Concordancer.SystemManager.csproj

@@ -5,7 +5,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.4</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Core/Constants/ConfigurationKey.cs

@@ -16,9 +16,4 @@ public static class ConfigurationKey
     {
         public const string ClasslaTokenizerUrl = "RSDO:Tokenizer:ClasslaUrl";
     }
-
-    public static class Web
-    {
-        public const string BaseAppPath = "RSDO:Web:BaseAppPath";
-    }
 }

src/core/Rsdo.Concordancer.Core/Rsdo.Concordancer.Core.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.4</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Data/Migrations/v1_0/Corpus/Mig20230413111500_ChangeFormToText.cs

@@ -0,0 +1,22 @@
+using System;
+using FluentMigrator;
+using Rsdo.Concordancer.Core.Entities;
+using Rsdo.Concordancer.Data.Framework;
+
+namespace Rsdo.Concordancer.Data.Migrations.v1_0.Corpus;
+
+[CorpusDb]
+[Migration(20230413111500)]
+public class Mig20230413111500_ChangeFormToText : ForwardOnlyMigration
+{
+    public override void Up()
+    {
+        // Token
+        Alter.Column(nameof(Token.Form)).OnTable(nameof(Token)).AsString(int.MaxValue).Nullable();
+        Alter.Column(nameof(Token.Lemma)).OnTable(nameof(Token)).AsString(int.MaxValue).Nullable();
+
+        // Term
+        Alter.Column(nameof(Term.Form)).OnTable(nameof(Term)).AsString(int.MaxValue).NotNullable();
+        Alter.Column(nameof(Term.Lemma)).OnTable(nameof(Term)).AsString(int.MaxValue).NotNullable();
+    }
+}

src/core/Rsdo.Concordancer.Data/Rsdo.Concordancer.Data.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.4</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Infrastructure/Rsdo.Concordancer.Infrastructure.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.4</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Infrastructure/Search/AddRecordsHandlers/BaseAddRecordsHandler.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using OpenSearch.Client;
 using Rsdo.Concordancer.Infrastructure.Search.Extensions;
@@ -27,23 +28,27 @@ public abstract class BaseAddRecordsHandler<TEntity, TElasticEntity> : IAddRecor
         // Get index name
         var indexName = indexProvider.IndexName;
 
-        // Create request
+        // Batch entities by 1000
-        var request = new BulkRequest(indexName)
+        foreach (var batch in entities.Chunk(1000))
         {
-            Operations = new List<IBulkOperation>(),
+            // Create request
-            Timeout = TimeSpan.FromMinutes(5),
+            var request = new BulkRequest(indexName)
-        };
+            {
+                Operations = new List<IBulkOperation>(),
+                Timeout = TimeSpan.FromMinutes(5),
+            };
 
-        // Convert entities to elastic entities
+            // Convert entities to elastic entities
-        foreach (var entity in entities)
+            foreach (var entity in batch)
-        {
+            {
-            var elasticEntity = ConvertEntity(entity);
+                var elasticEntity = ConvertEntity(entity);
-            request.Operations.Add(new BulkIndexOperation<TElasticEntity>(elasticEntity));
+                request.Operations.Add(new BulkIndexOperation<TElasticEntity>(elasticEntity));
+            }
+
+            // Get response
+            var response = await client.BulkAsync(request);
+            response.ThrowIfInvalid();
         }
-
-        // Get response
-        var response = await client.BulkAsync(request);
-        response.ThrowIfInvalid();
     }
 
     protected abstract TElasticEntity ConvertEntity(TEntity entity);

src/core/Rsdo.Concordancer.ServiceModel/Rsdo.Concordancer.ServiceModel.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.4</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Services/Framework/DatabaseManager/PostgreSqlDatabaseManager.cs

@@ -4,6 +4,7 @@ using System.Threading.Tasks;
 using Microsoft.EntityFrameworkCore;
 using Npgsql;
 using Rsdo.Concordancer.Core.Interfaces;
+using Rsdo.Concordancer.ServiceModel.Types;
 using Rsdo.Concordancer.Services.Framework.DbContext;
 
 namespace Rsdo.Concordancer.Services.Framework.DatabaseManager;
@@ -57,7 +58,7 @@ public class PostgreSqlDatabaseManager : IDatabaseManager
     public async Task UpdateCorpusDatabases()
     {
         // Update all corpus databases
-        var corpusIds = await dbContext.Corpus.Select(c => c.Id).ToListAsync();
+        var corpusIds = await dbContext.Corpus.Where(s => s.Status == CorpusStatus.Active).Select(c => c.Id).ToListAsync();
         foreach (var corpusId in corpusIds)
         {
             var connectionString = connectionStringProvider.GetCorpusConnectionString(corpusId);

src/core/Rsdo.Concordancer.Services/RequestHandlers/Concordances/ExportConcordancesHandler.cs

@@ -1,8 +1,11 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
 using Rsdo.Concordancer.Core.Constants;
 using Rsdo.Concordancer.Core.Interfaces;
 using Rsdo.Concordancer.Core.Model;
@@ -11,6 +14,7 @@ using Rsdo.Concordancer.Core.Search.Queries.Concordances;
 using Rsdo.Concordancer.ServiceModel.Requests.Concordances;
 using Rsdo.Concordancer.ServiceModel.Shared;
 using Rsdo.Concordancer.ServiceModel.Types;
+using Rsdo.Concordancer.Services.Framework.DbContext;
 using Rsdo.Concordancer.Services.Search.QueryFactories;
 using Rsdo.Concordancer.Services.Services.ParagraphService;
 
@@ -18,15 +22,18 @@ namespace Rsdo.Concordancer.Services.RequestHandlers.Concordances;
 
 public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequestHandler<ExportConcordances, ExportConcordancesResponse>
 {
+    private readonly CorpusDbContext dbContext;
     private readonly IQueryFactory<ExportConcordances, ConcordancesQuery> queryFactory;
     private readonly ISearchEngine searchEngine;
 
     public ExportConcordancesHandler(
+        CorpusDbContext dbContext,
         IParagraphService paragraphService,
         IQueryFactory<ExportConcordances, ConcordancesQuery> queryFactory,
         ISearchEngine searchEngine)
         : base(paragraphService)
     {
+        this.dbContext = dbContext;
         this.queryFactory = queryFactory;
         this.searchEngine = searchEngine;
     }
@@ -42,6 +49,9 @@ public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequest
         // Get search items
         var items = await GetItems(query, result.EntityIds);
 
+        // Get sources
+        var sources = await GetSources(items);
+
         // Export
         var stream = new MemoryStream();
         await using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true))
@@ -52,7 +62,7 @@ public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequest
             // Write items
             foreach (var item in items)
             {
-                await WriteItem(writer, item);
+                await WriteItem(writer, item, sources[item.ParagraphId]);
             }
         }
 
@@ -91,16 +101,35 @@ public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequest
         await writer.WriteAsync("Iskani niz");
         await writer.WriteAsync("\t");
         await writer.WriteAsync("Desno besedilo");
+        await writer.WriteAsync("\t");
+        await writer.WriteAsync("Vir");
         await writer.WriteLineAsync();
     }
 
-    private static async Task WriteItem(StreamWriter writer, SearchConcordancesResponseItem item)
+    private static async Task WriteItem(StreamWriter writer, SearchConcordancesResponseItem item, string source)
     {
         await WriteContext(writer, item.LeftContext);
         await writer.WriteAsync("\t");
         await writer.WriteAsync(item.CenterContext.Form);
         await writer.WriteAsync("\t");
         await WriteContext(writer, item.RightContext);
+        await writer.WriteAsync("\t");
+        await writer.WriteAsync(source);
         await writer.WriteLineAsync();
     }
+
+    private async Task<Dictionary<Guid, string>> GetSources(List<SearchConcordancesResponseItem> items)
+    {
+        var paragraphIds = items.Select(i => i.ParagraphId).ToList();
+        var sources = await dbContext.Paragraph.Include(p => p.Text)
+            .Where(p => paragraphIds.Contains(p.Id))
+            .Select(
+                p => new
+                {
+                    p.Id,
+                    p.Text.SourceFile,
+                })
+            .ToListAsync();
+        return sources.ToDictionary(x => x.Id, x => Path.GetFileNameWithoutExtension(x.SourceFile));
+    }
 }

src/core/Rsdo.Concordancer.Services/RequestHandlers/TermLists/ImportTermListHandler.cs

@@ -109,7 +109,7 @@ public class ImportTermListHandler : IRequestHandler<ImportTermList, ExecutionRe
                         Lemma = termObj["kandidat"].Value<string>(),
                         Msd = termObj["POSoznake"].Value<string>(),
                         Weight = termObj["ranking"].Value<decimal>(),
-                        Frequency = termObj["pogostostpojavljanja"].Value<int>(),
+                        Frequency = termObj["pogostostpojavljanja"].Values<int>().First(),
                         TermListAutoId = termList.AutoId,
                     }.ApplyCreateValues();
 

src/core/Rsdo.Concordancer.Services/RequestHandlers/Texts/IndexTextHandler.cs

@@ -90,39 +90,35 @@ public class IndexTextHandler : IRequestHandler<IndexText, ExecutionResult>
 
     private static List<Concordance> GetConcordances(Text text, Paragraph paragraph, List<Token> tokens)
     {
-        // Get window size (max 10)
-        var windowSize = Math.Min(tokens.Count - 1, 10);
-
-        // Create list of tokens with positions
-        var tokensIdx = tokens.Select((t, i) => new KeyValuePair<int, Token>(i, t)).ToList();
-
-        // Get tokens which will appear in current concordance
-        List<KeyValuePair<int, Token>> concordanceTokens;
-
         var concordances = new List<Concordance>();
-        while ((concordanceTokens = tokensIdx.Where(x => x.Key >= -windowSize && x.Key <= windowSize).ToList().OrderBy(x => x.Key).ToList()).Any())
+        for (var i = 0; i < tokens.Count; i++)
         {
-            if (concordanceTokens.Last().Key < 0)
-            {
-                break;
-            }
-
             var concordance = new Concordance()
             {
                 ParagraphId = paragraph.Id,
                 TextId = text.Id,
             };
 
-            // Loop through tokens and set it in the position
+            // set center token
-            foreach (var concordanceToken in concordanceTokens)
+            concordance.SetToken(tokens[i], 0);
+
+            // set left and right context tokens
+            for (var c = 1; c <= 10; c++)
             {
-                concordance.SetToken(concordanceToken.Value, concordanceToken.Key);
+                // left context
+                if (i - c >= 0)
+                {
+                    concordance.SetToken(tokens[i - c], -c);
+                }
+
+                // right context
+                if (i + c < tokens.Count)
+                {
+                    concordance.SetToken(tokens[i + c], c);
+                }
             }
 
             concordances.Add(concordance);
-
-            // Decrease indexes of tokens (shift tokens to left, relative to window size)
-            tokensIdx = tokensIdx.Select(x => new KeyValuePair<int, Token>(x.Key - 1, x.Value)).ToList();
         }
 
         return concordances;

src/core/Rsdo.Concordancer.Services/Rsdo.Concordancer.Services.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.0</Version>
+        <Version>1.0.4</Version>
     </PropertyGroup>
 
     <ItemGroup>