From d7170126f07c727478964b5ac6ecf36faa1fb3df Mon Sep 17 00:00:00 2001
From: Rok Rejc <rok@rejc.org>
Date: Wed, 11 Jan 2023 12:38:10 +0100
Subject: [PATCH] Performance improvements for indexing text.

---
 .../BaseAddRecordsHandler.cs                  | 33 +++++++++-------
 .../RequestHandlers/Texts/IndexTextHandler.cs | 38 +++++++++----------
 2 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/src/core/Rsdo.Concordancer.Infrastructure/Search/AddRecordsHandlers/BaseAddRecordsHandler.cs b/src/core/Rsdo.Concordancer.Infrastructure/Search/AddRecordsHandlers/BaseAddRecordsHandler.cs
index 4b3dc4c..f13fe8e 100644
--- a/src/core/Rsdo.Concordancer.Infrastructure/Search/AddRecordsHandlers/BaseAddRecordsHandler.cs
+++ b/src/core/Rsdo.Concordancer.Infrastructure/Search/AddRecordsHandlers/BaseAddRecordsHandler.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using OpenSearch.Client;
 using Rsdo.Concordancer.Infrastructure.Search.Extensions;
@@ -27,23 +28,27 @@ public abstract class BaseAddRecordsHandler<TEntity, TElasticEntity> : IAddRecor
         // Get index name
         var indexName = indexProvider.IndexName;
 
-        // Create request
-        var request = new BulkRequest(indexName)
+        // Batch entities by 1000
+        foreach (var batch in entities.Chunk(1000))
         {
-            Operations = new List<IBulkOperation>(),
-            Timeout = TimeSpan.FromMinutes(5),
-        };
+            // Create request
+            var request = new BulkRequest(indexName)
+            {
+                Operations = new List<IBulkOperation>(),
+                Timeout = TimeSpan.FromMinutes(5),
+            };
 
-        // Convert entities to elastic entities
-        foreach (var entity in entities)
-        {
-            var elasticEntity = ConvertEntity(entity);
-            request.Operations.Add(new BulkIndexOperation<TElasticEntity>(elasticEntity));
+            // Convert entities to elastic entities
+            foreach (var entity in batch)
+            {
+                var elasticEntity = ConvertEntity(entity);
+                request.Operations.Add(new BulkIndexOperation<TElasticEntity>(elasticEntity));
+            }
+
+            // Get response
+            var response = await client.BulkAsync(request);
+            response.ThrowIfInvalid();
         }
-
-        // Get response
-        var response = await client.BulkAsync(request);
-        response.ThrowIfInvalid();
     }
 
     protected abstract TElasticEntity ConvertEntity(TEntity entity);
diff --git a/src/core/Rsdo.Concordancer.Services/RequestHandlers/Texts/IndexTextHandler.cs b/src/core/Rsdo.Concordancer.Services/RequestHandlers/Texts/IndexTextHandler.cs
index 008ff82..443e59d 100644
--- a/src/core/Rsdo.Concordancer.Services/RequestHandlers/Texts/IndexTextHandler.cs
+++ b/src/core/Rsdo.Concordancer.Services/RequestHandlers/Texts/IndexTextHandler.cs
@@ -90,39 +90,35 @@ public class IndexTextHandler : IRequestHandler<IndexText, ExecutionResult>
 
     private static List<Concordance> GetConcordances(Text text, Paragraph paragraph, List<Token> tokens)
     {
-        // Get window size (max 10)
-        var windowSize = Math.Min(tokens.Count - 1, 10);
-
-        // Create list of tokens with positions
-        var tokensIdx = tokens.Select((t, i) => new KeyValuePair<int, Token>(i, t)).ToList();
-
-        // Get tokens which will appear in current concordance
-        List<KeyValuePair<int, Token>> concordanceTokens;
-
         var concordances = new List<Concordance>();
-        while ((concordanceTokens = tokensIdx.Where(x => x.Key >= -windowSize && x.Key <= windowSize).ToList().OrderBy(x => x.Key).ToList()).Any())
+        for (var i = 0; i < tokens.Count; i++)
         {
-            if (concordanceTokens.Last().Key < 0)
-            {
-                break;
-            }
-
             var concordance = new Concordance()
             {
                 ParagraphId = paragraph.Id,
                 TextId = text.Id,
             };
 
-            // Loop through tokens and set it in the position
-            foreach (var concordanceToken in concordanceTokens)
+            // set center token
+            concordance.SetToken(tokens[i], 0);
+
+            // set left and right context tokens
+            for (var c = 1; c <= 10; c++)
             {
-                concordance.SetToken(concordanceToken.Value, concordanceToken.Key);
+                // left context
+                if (i - c >= 0)
+                {
+                    concordance.SetToken(tokens[i - c], -c);
+                }
+
+                // right context
+                if (i + c < tokens.Count)
+                {
+                    concordance.SetToken(tokens[i + c], c);
+                }
             }
 
             concordances.Add(concordance);
-
-            // Decrease indexes of tokens (shift tokens to left, relative to window size)
-            tokensIdx = tokensIdx.Select(x => new KeyValuePair<int, Token>(x.Key - 1, x.Value)).ToList();
         }
 
         return concordances;