Fix layout when source (file name) is long, bump version to 1.0.3

README.md

@@ -1 +1,53 @@
 # RSDO (Razvoj Slovenščine v Digitalnem Okolju) - Concordancer
+This repository contains the source code of the concordancer which is a part of the [Terminology Portal](https://github.com/clarinsi/rsdo_term_portal).
+
+There are two main parts:
+- Web application with backend (API interface) written in ASP.NET Core 6 and frontend written in React
+- System Manager which is written in .NET 6 and is used to initially set up the system
+
+## Prerequisites
+Web application requires following external dependencies
+- OpenSearch v2.3.0 (use docker image: `opensearchproject/opensearch:2.3.0`)
+- PostgreSQL (use docker image: `postgres`)
+
+## Building source code
+### Frontend (React application)
+To build a frontend application successfully, first, [Node.js](https://nodejs.org/en/) must be installed (at least version 16).
+Use `npm install` to install all required dependencies, and `npm run build` to build the application.
+For more info see the [Readme](src/apps/Rsdo.Concordancer.Api/WebApp/README.md) inside the frontend [root](src/apps/Rsdo.Concordancer.Api/WebApp) folder.
+
+### Backend (ASP.NET Core 6) and System Manager (.NET 6)
+Open [Rsdo.Concordancer.sln](src/Rsdo.Concordancer.sln) inside of your IDE and build the solution, or use `dotnet build` command to build the solution. All required dependencies will be installed via NuGet packages.
+
+## Creating docker images
+There are two Dockerfile's in the source repository for each part. Both images should be compiled within the context of the project root folder.
+
+### Web application
+[Dockerfile](src/apps/Rsdo.Concordancer.Api/Dockerfile) for Web Application accepts additional optional argument (WEBAPP_PUBLIC_URL) which is mapped to React [PUBLIC_URL](https://create-react-app.dev/docs/using-the-public-folder/) variable. If the variable is omited, then application should be deployed on a domain root.
+
+`docker build -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .`
+
+`docker build --build-arg WEBAPP_PUBLIC_URL=/corpus -t rsdo-concordancer-api -f src/apps/Rsdo.Concordancer.Api/Dockerfile .`
+
+### System Manager
+Use following [Dockerfile](src/apps/Rsdo.Concordancer.SystemManager/Dockerfile) to build System Manager.
+
+`docker build -t rsdo-concordancer-systemmanager -f src/apps/Rsdo.Concordancer.SystemManager/Dockerfile .`
+
+## Environment variables
+
+### Web application
+Web application must be configured with the following Environment variables and values:
+- `RSDO:Database:MasterConnectionString` holds connection string to the main database; e.g.: `rsdo_db;Username=rsdo;Password=strongpassword;Database=Concordancer`
+- `RSDO:Elastic:ConnectionString` holds connection string to OpenSearch server; e.g.: `http://rsdo_opensearch:9200`
+- `RSDO:Tokenizer:ClasslaUrl` holds and url to external service which is used for tokenizing search strings; e.g.: `https://orodja.cjvt.si/oznacevalnik/ajax_api/v1/slv/process`
+
+### System Manager
+System manager must be configured with the following Environment variables and values:
+- `RSDO:Database:MasterConnectionString` holds connection string to the main database; e.g.: `rsdo_db;Username=rsdo;Password=strongpassword;Database=Concordancer`
+- `RSDO:Elastic:ConnectionString` holds connection string to OpenSearch server; e.g.: `http://rsdo_opensearch:9200`
+
+## Using System Manager
+To initialy setup the system, you have to use System Manager and run following commands:
+1. `dotnet Rsdo.Concordancer.SystemManager.dll createMasterDb --connectionString="Host=rsdo_db;Username=rsdo;Password=strongpassword;Database=postgres"` which will create master database with the name Concordancer. You can select different name with the `--name` option
+2. `dotnet Rsdo.Concordancer.SystemManager.dll importSloleks --sourceFile=/data/Sloleks2.0.LMF/sloleks_clarin_2.0.xml` which will import Sloleks (which is used for better searching). XML file is not part of this repository, but it can be downloaded from the following [URL](https://www.clarin.si/repository/xmlui/bitstream/handle/11356/1230/Sloleks2.0.LMF.zip?sequence=3&isAllowed=y)

src/apps/Rsdo.Concordancer.Api/Rsdo.Concordancer.Api.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.1</Version>
+        <Version>1.0.3</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAggregationItem.js

@@ -8,7 +8,7 @@ const ConcordanceAggregationItem = (props) => {
     const link = isSelected ? search.getClearFilterLink('TextIds', props.filterKey) : search.getFilterLink('TextIds', props.filterKey);
 
     return (
-        <li className={styles.aggregationItem}><Link to={link}>{isSelected && <span>X - </span>}{props.title}<span className='float-end'>{props.count}</span></Link></li>
+        <li className={styles.aggregationItem}><Link to={link}>{isSelected && <span className={styles.remove} >X</span>}<span className={styles.title}>{props.title}</span><span className={styles.count}>{props.count}</span></Link></li>
     );
 };
 

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAggregationItem.module.scss

@@ -7,6 +7,18 @@
 
     a {
         color: $text-color;
+        display: flex;
         text-decoration: none;
+
+        .remove {
+            margin-right: 0.8rem;
+        }
+
+        .title {
+            flex-grow: 1;
+            margin-right: 0.8rem;
+            overflow: hidden;
+            text-overflow: clip;
+        }
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAlternateSearchItem.js

@@ -6,7 +6,7 @@ const ConcordanceAlternateSearchItem = (props) => {
     const search = useSearch();
     const link = search.getAlternateSearchLink(props.search);
     return (
-        <li className={styles.alternateSearchItem}><Link to={link}>{props.title}<span className='float-end'>{props.count}</span></Link></li>
+        <li className={styles.alternateSearchItem}><Link to={link}><span className={styles.title}>{props.title}</span><span className={styles.count}>{props.count}</span></Link></li>
     );
 };
 

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceAlternateSearchItem.module.scss

@@ -7,6 +7,14 @@
 
     a {
         color: $text-color;
+        display: flex;
         text-decoration: none;
+
+        .title {
+            flex-grow: 1;
+            margin-right: 0.8rem;
+            overflow: hidden;
+            text-overflow: clip;
+        }
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetailsText.js

@@ -1,6 +1,6 @@
-import { useTranslation } from 'react-i18next';
+import ConcordanceDetailsTextInfo from './ConcordanceDetailsTextInfo';
 
-import sytles from './ConcordanceDetailsText.module.scss';
+import { useTranslation } from 'react-i18next';
 
 const ConcordanceDetailsText = (props) => {
     // Hooks
@@ -8,22 +8,10 @@ const ConcordanceDetailsText = (props) => {
 
     return (
         <>
-            <div className={sytles.textInfo}>
-                <h5>{t('concordance.detailsTextSource')}</h5>
-                <p>{props.source}</p>
-            </div>
-            <div className={sytles.textInfo}>
-                <h5>{t('concordance.detailsTextYear')}</h5>
-                <p>{props.year}</p>
-            </div>
-            <div className={sytles.textInfo}>
-                <h5>{t('concordance.detailsTextTitle')}</h5>
-                <p>{props.title}</p>
-            </div>
-            <div className={sytles.textInfo}>
-                <h5>{t('concordance.detailsTextAuthor')}</h5>
-                <p>{props.author}</p>
-            </div>
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextSource')} value={props.source} />
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextYear')} value={props.year} />
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextTitle')} value={props.title} />
+            <ConcordanceDetailsTextInfo title={t('concordance.detailsTextAuthor')} value={props.author} />
         </>
     );
 };

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetailsTextInfo.js

@@ -0,0 +1,15 @@
+import styles from './ConcordanceDetailsTextInfo.module.scss';
+
+const ConcordanceDetailsTextInfo = (props) => {
+    var title = props.title;
+    var value = props.value;
+
+    if (value) {
+        return (<div className={styles.textInfo}>
+            <h5>{title}</h5>
+            <p>{value}</p>
+        </div>);
+    }
+};
+
+export default ConcordanceDetailsTextInfo;

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceDetailsTextInfo.module.scss

@@ -18,5 +18,7 @@
 
     p {
         margin: 0;
+        text-overflow: clip;
+        overflow: hidden;
     }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceListItem.js

@@ -6,8 +6,8 @@ const ConcordanceListItem = (props) => {
     return (
         <>
             <div className={`row ${styles.item}`} onClick={() => { props.toggleDetails(props.index) }}>
-                <div className='col text-end'><ConcordanceTokens tokens={props.item.leftContext} />&nbsp;</div>
-                <div className='col'><ConcordanceTokens tokens={[props.item.centerContext]} /> <ConcordanceTokens tokens={props.item.rightContext} /></div>
+                <div className={`col ${styles.left}`}><span><ConcordanceTokens tokens={props.item.leftContext} />&nbsp;</span></div>
+                <div className={`col ${styles.right}`}><ConcordanceTokens tokens={[props.item.centerContext]} /> <ConcordanceTokens tokens={props.item.rightContext} /></div>
             </div>
             {props.expanded && <ConcordanceDetails paragraphId={props.item.paragraphId} tokenOrder={props.item.centerContext.tokenOrder} onClose={() => { props.toggleDetails(null) }} />}
         </>

src/apps/Rsdo.Concordancer.Api/WebApp/src/components/concordance/ConcordanceListItem.module.scss

@@ -5,4 +5,24 @@
     cursor: pointer;
     margin-bottom: 0.5rem;
     padding-bottom: 0.5rem;
+
+    :global(.col) {
+
+        &.left,
+        &.right {
+            overflow: hidden;
+            white-space: nowrap;
+            text-overflow: clip;
+        }
+
+        &.left {
+            span {
+                display: inline-block;
+                float: right;
+                min-width: 100%;
+                overflow: visible;
+                text-align: right;
+            }
+        }
+    }
 }

src/apps/Rsdo.Concordancer.Api/WebApp/src/hooks/use-download.js

@@ -12,7 +12,7 @@ const useDownload = (url) => {
                 var url = window.URL.createObjectURL(blob);
                 var a = document.createElement('a');
                 a.href = url;
-                a.download = "export.csv";
+                a.download = "export.txt";
                 document.body.appendChild(a); // we need to append the element to the dom -> otherwise it will not work in firefox
                 a.click();
                 a.remove();  //afterwards we remove the element again                       

src/apps/Rsdo.Concordancer.SystemManager/Rsdo.Concordancer.SystemManager.csproj

@@ -5,7 +5,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.1</Version>
+        <Version>1.0.3</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Core/Rsdo.Concordancer.Core.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.1</Version>
+        <Version>1.0.3</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Data/Rsdo.Concordancer.Data.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.1</Version>
+        <Version>1.0.3</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Infrastructure/Rsdo.Concordancer.Infrastructure.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.1</Version>
+        <Version>1.0.3</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Infrastructure/Search/AddRecordsHandlers/BaseAddRecordsHandler.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using OpenSearch.Client;
 using Rsdo.Concordancer.Infrastructure.Search.Extensions;
@@ -27,23 +28,27 @@ public abstract class BaseAddRecordsHandler<TEntity, TElasticEntity> : IAddRecor
         // Get index name
         var indexName = indexProvider.IndexName;
 
-        // Create request
-        var request = new BulkRequest(indexName)
+        // Batch entities by 1000
+        foreach (var batch in entities.Chunk(1000))
         {
-            Operations = new List<IBulkOperation>(),
-            Timeout = TimeSpan.FromMinutes(5),
-        };
+            // Create request
+            var request = new BulkRequest(indexName)
+            {
+                Operations = new List<IBulkOperation>(),
+                Timeout = TimeSpan.FromMinutes(5),
+            };
 
-        // Convert entities to elastic entities
-        foreach (var entity in entities)
-        {
-            var elasticEntity = ConvertEntity(entity);
-            request.Operations.Add(new BulkIndexOperation<TElasticEntity>(elasticEntity));
+            // Convert entities to elastic entities
+            foreach (var entity in batch)
+            {
+                var elasticEntity = ConvertEntity(entity);
+                request.Operations.Add(new BulkIndexOperation<TElasticEntity>(elasticEntity));
+            }
+
+            // Get response
+            var response = await client.BulkAsync(request);
+            response.ThrowIfInvalid();
         }
-
-        // Get response
-        var response = await client.BulkAsync(request);
-        response.ThrowIfInvalid();
     }
 
     protected abstract TElasticEntity ConvertEntity(TEntity entity);

src/core/Rsdo.Concordancer.ServiceModel/Rsdo.Concordancer.ServiceModel.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.1</Version>
+        <Version>1.0.3</Version>
     </PropertyGroup>
 
     <ItemGroup>

src/core/Rsdo.Concordancer.Services/RequestHandlers/Concordances/ExportConcordancesHandler.cs

@@ -1,8 +1,11 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
 using Rsdo.Concordancer.Core.Constants;
 using Rsdo.Concordancer.Core.Interfaces;
 using Rsdo.Concordancer.Core.Model;
@@ -11,6 +14,7 @@ using Rsdo.Concordancer.Core.Search.Queries.Concordances;
 using Rsdo.Concordancer.ServiceModel.Requests.Concordances;
 using Rsdo.Concordancer.ServiceModel.Shared;
 using Rsdo.Concordancer.ServiceModel.Types;
+using Rsdo.Concordancer.Services.Framework.DbContext;
 using Rsdo.Concordancer.Services.Search.QueryFactories;
 using Rsdo.Concordancer.Services.Services.ParagraphService;
 
@@ -18,15 +22,18 @@ namespace Rsdo.Concordancer.Services.RequestHandlers.Concordances;
 
 public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequestHandler<ExportConcordances, ExportConcordancesResponse>
 {
+    private readonly CorpusDbContext dbContext;
     private readonly IQueryFactory<ExportConcordances, ConcordancesQuery> queryFactory;
     private readonly ISearchEngine searchEngine;
 
     public ExportConcordancesHandler(
+        CorpusDbContext dbContext,
         IParagraphService paragraphService,
         IQueryFactory<ExportConcordances, ConcordancesQuery> queryFactory,
         ISearchEngine searchEngine)
         : base(paragraphService)
     {
+        this.dbContext = dbContext;
         this.queryFactory = queryFactory;
         this.searchEngine = searchEngine;
     }
@@ -42,6 +49,9 @@ public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequest
         // Get search items
         var items = await GetItems(query, result.EntityIds);
 
+        // Get sources
+        var sources = await GetSources(items);
+
         // Export
         var stream = new MemoryStream();
         await using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true))
@@ -52,7 +62,7 @@ public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequest
             // Write items
             foreach (var item in items)
             {
-                await WriteItem(writer, item);
+                await WriteItem(writer, item, sources[item.ParagraphId]);
             }
         }
 
@@ -91,16 +101,35 @@ public class ExportConcordancesHandler : BaseSearchConcordancesHandler, IRequest
         await writer.WriteAsync("Iskani niz");
         await writer.WriteAsync("\t");
         await writer.WriteAsync("Desno besedilo");
+        await writer.WriteAsync("\t");
+        await writer.WriteAsync("Vir");
         await writer.WriteLineAsync();
     }
 
-    private static async Task WriteItem(StreamWriter writer, SearchConcordancesResponseItem item)
+    private static async Task WriteItem(StreamWriter writer, SearchConcordancesResponseItem item, string source)
     {
         await WriteContext(writer, item.LeftContext);
         await writer.WriteAsync("\t");
         await writer.WriteAsync(item.CenterContext.Form);
         await writer.WriteAsync("\t");
         await WriteContext(writer, item.RightContext);
+        await writer.WriteAsync("\t");
+        await writer.WriteAsync(source);
         await writer.WriteLineAsync();
     }
+
+    private async Task<Dictionary<Guid, string>> GetSources(List<SearchConcordancesResponseItem> items)
+    {
+        var paragraphIds = items.Select(i => i.ParagraphId).ToList();
+        var sources = await dbContext.Paragraph.Include(p => p.Text)
+            .Where(p => paragraphIds.Contains(p.Id))
+            .Select(
+                p => new
+                {
+                    p.Id,
+                    p.Text.SourceFile,
+                })
+            .ToListAsync();
+        return sources.ToDictionary(x => x.Id, x => Path.GetFileNameWithoutExtension(x.SourceFile));
+    }
 }

src/core/Rsdo.Concordancer.Services/RequestHandlers/TermLists/ImportTermListHandler.cs

@@ -109,7 +109,7 @@ public class ImportTermListHandler : IRequestHandler<ImportTermList, ExecutionRe
                         Lemma = termObj["kandidat"].Value<string>(),
                         Msd = termObj["POSoznake"].Value<string>(),
                         Weight = termObj["ranking"].Value<decimal>(),
-                        Frequency = termObj["pogostostpojavljanja"].Value<int>(),
+                        Frequency = termObj["pogostostpojavljanja"].Values<int>().First(),
                         TermListAutoId = termList.AutoId,
                     }.ApplyCreateValues();
 

src/core/Rsdo.Concordancer.Services/RequestHandlers/Texts/IndexTextHandler.cs

@@ -90,39 +90,35 @@ public class IndexTextHandler : IRequestHandler<IndexText, ExecutionResult>
 
     private static List<Concordance> GetConcordances(Text text, Paragraph paragraph, List<Token> tokens)
     {
-        // Get window size (max 10)
-        var windowSize = Math.Min(tokens.Count - 1, 10);
-
-        // Create list of tokens with positions
-        var tokensIdx = tokens.Select((t, i) => new KeyValuePair<int, Token>(i, t)).ToList();
-
-        // Get tokens which will appear in current concordance
-        List<KeyValuePair<int, Token>> concordanceTokens;
-
         var concordances = new List<Concordance>();
-        while ((concordanceTokens = tokensIdx.Where(x => x.Key >= -windowSize && x.Key <= windowSize).ToList().OrderBy(x => x.Key).ToList()).Any())
+        for (var i = 0; i < tokens.Count; i++)
         {
-            if (concordanceTokens.Last().Key < 0)
-            {
-                break;
-            }
-
             var concordance = new Concordance()
             {
                 ParagraphId = paragraph.Id,
                 TextId = text.Id,
             };
 
-            // Loop through tokens and set it in the position
-            foreach (var concordanceToken in concordanceTokens)
+            // set center token
+            concordance.SetToken(tokens[i], 0);
+
+            // set left and right context tokens
+            for (var c = 1; c <= 10; c++)
             {
-                concordance.SetToken(concordanceToken.Value, concordanceToken.Key);
+                // left context
+                if (i - c >= 0)
+                {
+                    concordance.SetToken(tokens[i - c], -c);
+                }
+
+                // right context
+                if (i + c < tokens.Count)
+                {
+                    concordance.SetToken(tokens[i + c], c);
+                }
             }
 
             concordances.Add(concordance);
-
-            // Decrease indexes of tokens (shift tokens to left, relative to window size)
-            tokensIdx = tokensIdx.Select(x => new KeyValuePair<int, Token>(x.Key - 1, x.Value)).ToList();
         }
 
         return concordances;

src/core/Rsdo.Concordancer.Services/Rsdo.Concordancer.Services.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <EnableNETAnalyzers>false</EnableNETAnalyzers>
-        <Version>1.0.1</Version>
+        <Version>1.0.3</Version>
     </PropertyGroup>
 
     <ItemGroup>