Initial commit

src/Gos.Tests.Unit/Services/Framework/Comparers/ConcordanceTokensComparer.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Generic;
+using Gos.Core.Model;
+
+namespace Gos.Tests.Unit.Services.Framework.Comparers;
+
+public class ConcordanceTokensComparer : IEqualityComparer<Concordance>
+{
+    public bool Equals(Concordance x, Concordance y)
+    {
+        if (ReferenceEquals(x, y))
+        {
+            return true;
+        }
+
+        if (ReferenceEquals(x, null))
+        {
+            return false;
+        }
+
+        if (ReferenceEquals(y, null))
+        {
+            return false;
+        }
+
+        if (x.GetType() != y.GetType())
+        {
+            return false;
+        }
+
+        return Equals(x.Token, y.Token) &&
+            Equals(x.TokenLeft1, y.TokenLeft1) &&
+            Equals(x.TokenLeft10, y.TokenLeft10) &&
+            Equals(x.TokenLeft2, y.TokenLeft2) &&
+            Equals(x.TokenLeft3, y.TokenLeft3) &&
+            Equals(x.TokenLeft4, y.TokenLeft4) &&
+            Equals(x.TokenLeft5, y.TokenLeft5) &&
+            Equals(x.TokenLeft6, y.TokenLeft6) &&
+            Equals(x.TokenLeft7, y.TokenLeft7) &&
+            Equals(x.TokenLeft8, y.TokenLeft8) &&
+            Equals(x.TokenLeft9, y.TokenLeft9) &&
+            Equals(x.TokenRight1, y.TokenRight1) &&
+            Equals(x.TokenRight10, y.TokenRight10) &&
+            Equals(x.TokenRight2, y.TokenRight2) &&
+            Equals(x.TokenRight3, y.TokenRight3) &&
+            Equals(x.TokenRight4, y.TokenRight4) &&
+            Equals(x.TokenRight5, y.TokenRight5) &&
+            Equals(x.TokenRight6, y.TokenRight6) &&
+            Equals(x.TokenRight7, y.TokenRight7) &&
+            Equals(x.TokenRight8, y.TokenRight8) &&
+            Equals(x.TokenRight9, y.TokenRight9);
+    }
+
+    public int GetHashCode(Concordance obj)
+    {
+        var hashCode = new HashCode();
+        hashCode.Add(obj.Token);
+        hashCode.Add(obj.TokenLeft1);
+        hashCode.Add(obj.TokenLeft10);
+        hashCode.Add(obj.TokenLeft2);
+        hashCode.Add(obj.TokenLeft3);
+        hashCode.Add(obj.TokenLeft4);
+        hashCode.Add(obj.TokenLeft5);
+        hashCode.Add(obj.TokenLeft6);
+        hashCode.Add(obj.TokenLeft7);
+        hashCode.Add(obj.TokenLeft8);
+        hashCode.Add(obj.TokenLeft9);
+        hashCode.Add(obj.TokenRight1);
+        hashCode.Add(obj.TokenRight10);
+        hashCode.Add(obj.TokenRight2);
+        hashCode.Add(obj.TokenRight3);
+        hashCode.Add(obj.TokenRight4);
+        hashCode.Add(obj.TokenRight5);
+        hashCode.Add(obj.TokenRight6);
+        hashCode.Add(obj.TokenRight7);
+        hashCode.Add(obj.TokenRight8);
+        hashCode.Add(obj.TokenRight9);
+        return hashCode.ToHashCode();
+    }
+}

src/Gos.Tests.Unit/Services/Framework/Comparers/FragmentParserTokenComparer.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using Gos.Core.Entities;
+
+namespace Gos.Tests.Unit.Services.Framework.Comparers;
+
+public class FragmentParserTokenComparer : IEqualityComparer<Token>
+{
+    public bool Equals(Token x, Token y)
+    {
+        if (ReferenceEquals(x, y))
+        {
+            return true;
+        }
+
+        if (ReferenceEquals(x, null))
+        {
+            return false;
+        }
+
+        if (ReferenceEquals(y, null))
+        {
+            return false;
+        }
+
+        if (x.GetType() != y.GetType())
+        {
+            return false;
+        }
+
+        return x.ConversationalForm == y.ConversationalForm && x.Lemma == y.Lemma && x.Msd == y.Msd && x.StandardForm == y.StandardForm && x.Type == y.Type;
+    }
+
+    public int GetHashCode(Token obj)
+    {
+        return HashCode.Combine(obj.ConversationalForm, obj.DiscourseOrder, obj.Lemma, obj.Msd, obj.Order, obj.Segment, obj.StandardForm, (int)obj.Type);
+    }
+}

src/Gos.Tests.Unit/Services/Framework/ConcordanceBuilderTests.cs

@@ -0,0 +1,160 @@
+using System.Collections.Generic;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.ConcordanceBuilder;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework
+{
+    public class ConcordanceBuilderTests
+    {
+        [Fact]
+        public void ReturnedConcordances()
+        {
+            // Arrange
+            var token1 = new Token(TokenType.Word).WithConversationForm("to").WithStandardForm("to").WithLemma("ta").WithMsd("mte:Pd-nsn");
+            var token2 = new Token(TokenType.Word).WithConversationForm("je").WithStandardForm("je").WithLemma("biti").WithMsd("mte:Va-r3s-n");
+            var token3 = new Token(TokenType.Word).WithConversationForm("zdej").WithStandardForm("zdaj").WithLemma("zdaj").WithMsd("mte:Rgp");
+            var token4 = new Token(TokenType.Word).WithConversationForm("to").WithStandardForm("to").WithLemma("ta").WithMsd("mte:Pd-nsa");
+            var token5 = new Token(TokenType.Word).WithConversationForm("kar").WithStandardForm("kar").WithLemma("kar").WithMsd("mte:Pr-nsa");
+            var token6 = new Token(TokenType.Word).WithConversationForm("bi").WithStandardForm("bi").WithLemma("biti").WithMsd("mte:Va-c");
+            var token7 = new Token(TokenType.Word).WithConversationForm("zmerl").WithStandardForm("izmerili").WithLemma("izmeriti").WithMsd("mte:Vmep-pm");
+            var token8 = new Token(TokenType.Word).WithConversationForm("na").WithStandardForm("na").WithLemma("na").WithMsd("mte:Sl");
+            var token9 = new Token(TokenType.Word).WithConversationForm("somi").WithStandardForm("somi").WithLemma("som").WithMsd("mte:Ncmpn");
+            var tokens = new List<Token>()
+            {
+                token1,
+                token2,
+                token3,
+                token4,
+                token5,
+                token6,
+                token7,
+                token8,
+                token9
+            };
+            var statement = new Statement();
+            var builder = new ConcordanceBuilder();
+
+            // Act
+            var concordances = builder.GetConcordances(statement, tokens);
+
+            // Assert
+            var expected = new List<Concordance>()
+            {
+                new()
+                {
+                    Token = token1,
+                    TokenRight1 = token2,
+                    TokenRight2 = token3,
+                    TokenRight3 = token4,
+                    TokenRight4 = token5,
+                    TokenRight5 = token6,
+                    TokenRight6 = token7,
+                    TokenRight7 = token8,
+                    TokenRight8 = token9
+                },
+                new()
+                {
+                    TokenLeft1 = token1,
+                    Token = token2,
+                    TokenRight1 = token3,
+                    TokenRight2 = token4,
+                    TokenRight3 = token5,
+                    TokenRight4 = token6,
+                    TokenRight5 = token7,
+                    TokenRight6 = token8,
+                    TokenRight7 = token9
+                },
+                new()
+                {
+                    TokenLeft2 = token1,
+                    TokenLeft1 = token2,
+                    Token = token3,
+                    TokenRight1 = token4,
+                    TokenRight2 = token5,
+                    TokenRight3 = token6,
+                    TokenRight4 = token7,
+                    TokenRight5 = token8,
+                    TokenRight6 = token9
+                },
+                new()
+                {
+                    TokenLeft3 = token1,
+                    TokenLeft2 = token2,
+                    TokenLeft1 = token3,
+                    Token = token4,
+                    TokenRight1 = token5,
+                    TokenRight2 = token6,
+                    TokenRight3 = token7,
+                    TokenRight4 = token8,
+                    TokenRight5 = token9
+                },
+                new()
+                {
+                    TokenLeft4 = token1,
+                    TokenLeft3 = token2,
+                    TokenLeft2 = token3,
+                    TokenLeft1 = token4,
+                    Token = token5,
+                    TokenRight1 = token6,
+                    TokenRight2 = token7,
+                    TokenRight3 = token8,
+                    TokenRight4 = token9
+                },
+                new()
+                {
+                    TokenLeft5 = token1,
+                    TokenLeft4 = token2,
+                    TokenLeft3 = token3,
+                    TokenLeft2 = token4,
+                    TokenLeft1 = token5,
+                    Token = token6,
+                    TokenRight1 = token7,
+                    TokenRight2 = token8,
+                    TokenRight3 = token9
+                },
+                new()
+                {
+                    TokenLeft6 = token1,
+                    TokenLeft5 = token2,
+                    TokenLeft4 = token3,
+                    TokenLeft3 = token4,
+                    TokenLeft2 = token5,
+                    TokenLeft1 = token6,
+                    Token = token7,
+                    TokenRight1 = token8,
+                    TokenRight2 = token9
+                },
+                new()
+                {
+                    TokenLeft7 = token1,
+                    TokenLeft6 = token2,
+                    TokenLeft5 = token3,
+                    TokenLeft4 = token4,
+                    TokenLeft3 = token5,
+                    TokenLeft2 = token6,
+                    TokenLeft1 = token7,
+                    Token = token8,
+                    TokenRight1 = token9
+                },
+                new()
+                {
+                    TokenLeft8 = token1,
+                    TokenLeft7 = token2,
+                    TokenLeft6 = token3,
+                    TokenLeft5 = token4,
+                    TokenLeft4 = token5,
+                    TokenLeft3 = token6,
+                    TokenLeft2 = token7,
+                    TokenLeft1 = token8,
+                    Token = token9
+                }
+            };
+
+            Assert.Equal(expected, concordances, new ConcordanceTokensComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/CharacterFragmentParserTests.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    public class CharacterFragmentParserTests
+    {
+        [Fact]
+        public void Character()
+        {
+            // Arrange
+            var xml = @"<pc xmlns=""http://www.tei-c.org/ns/1.0"" ana=""mte:Z"" msd=""UPosTag=PUNCT"" xml:id=""GosVL03_medit.tok349"">?</pc>";
+            var characterEl = XElement.Parse(xml);
+            var parser = new CharacterFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(characterEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Character).WithConversationForm("?").WithStandardForm("?").WithLemma("?").WithMsd("mte:Z")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/CorrectionFragmentParserTests.cs

@@ -0,0 +1,43 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Fixtures;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    [Collection(nameof(CorpusParserCollection))]
+    public class CorrectionFragmentParserTests
+    {
+        private readonly CorpusParserFixture corpusParserFixture;
+
+        public CorrectionFragmentParserTests(CorpusParserFixture corpusParserFixture)
+        {
+            this.corpusParserFixture = corpusParserFixture;
+        }
+
+        [Fact]
+        public void Correction()
+        {
+            // Arrange
+            var xml =
+                $@"<del xmlns=""http://www.tei-c.org/ns/1.0"" type=""falseStart""><w lemma=""p"" ana=""mte:Xt"" xml:id=""GosVL01_pravo.tok318"">p</w></del>";
+            var correctionEl = XElement.Parse(xml);
+            var parser = new CorrectionFragmentParser(corpusParserFixture.FragmentParserFactory);
+
+            // Act
+            var tokens = parser.GetTokens(correctionEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("p()").WithStandardForm("p()").WithLemma("p()").WithMsd("mte:Xt")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/ForeignFragmentParserTests.cs

@@ -0,0 +1,51 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Fixtures;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    [Collection(nameof(CorpusParserCollection))]
+    public class ForeignFragmentParserTests
+    {
+        private readonly CorpusParserFixture corpusParserFixture;
+
+        public ForeignFragmentParserTests(CorpusParserFixture corpusParserFixture)
+        {
+            this.corpusParserFixture = corpusParserFixture;
+        }
+
+        [Fact]
+        public void Foreign()
+        {
+            // Arrange
+            var xml = @"<foreign xmlns=""http://www.tei-c.org/ns/1.0"">
+                <w xml:id=""gos020.tok3452""
+            norm=""arbeit""
+            lemma=""arbeit""
+            ana=""mte:Xf"">arbajt</w>
+                <w xml:id=""gos020.tok3453"" norm=""macht"" lemma=""macht"" ana=""mte:Xf"">maht</w>
+                <w xml:id=""gos020.tok3454"" norm=""frei"" lemma=""frei"" ana=""mte:Xf"">fraj</w>
+                </foreign>";
+            var foreignEl = XElement.Parse(xml);
+            var parser = new ForeignFragmentParser(corpusParserFixture.FragmentParserFactory);
+
+            // Act
+            var tokens = parser.GetTokens(foreignEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("arbajt").WithStandardForm("arbeit").WithLemma("arbeit").WithMsd("mte:Xf"),
+                new Token(TokenType.Word).WithConversationForm("maht").WithStandardForm("macht").WithLemma("macht").WithMsd("mte:Xf"),
+                new Token(TokenType.Word).WithConversationForm("fraj").WithStandardForm("frei").WithLemma("frei").WithMsd("mte:Xf"),
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/GapFragmentParserTests.cs

@@ -0,0 +1,58 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    public class GapFragmentParserTests
+    {
+        [Fact]
+        public void Unclear()
+        {
+            // Arrange
+            var xml = @"<gap xmlns=""http://www.tei-c.org/ns/1.0"" reason=""unclear"" />";
+            var pauseEl = XElement.Parse(xml);
+            var parser = new GapFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(pauseEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("[nerazumljivo]")
+                    .WithStandardForm("[nerazumljivo]")
+                    .WithLemma("[nerazumljivo]")
+                    .WithMsd("mte:Cc"),
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+
+        [Fact]
+        public void Anon()
+        {
+            // Arrange
+            var xml = @"<gap xmlns=""http://www.tei-c.org/ns/1.0"" reason=""anon"" quantity=""1"" unit=""words"">
+                <desc>[ime]</desc>
+                </gap>";
+
+            var pauseEl = XElement.Parse(xml);
+            var parser = new GapFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(pauseEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("[ime]").WithStandardForm("[ime]").WithLemma("[ime]").WithMsd("mte:Xp"),
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/IncidentFragmentParserTests.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    public class IncidentFragmentParserTests
+    {
+        [Fact]
+        public void Sound()
+        {
+            // Arrange
+            var xml = @"<incident xmlns=""http://www.tei-c.org/ns/1.0"" type=""sound""/>";
+            var vocalEl = XElement.Parse(xml);
+            var parser = new IncidentFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(vocalEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("[zvok]").WithStandardForm("[zvok]").WithLemma("[zvok]").WithMsd("mte:Xt"),
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/NameFragmentParserTests.cs

@@ -0,0 +1,93 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Fixtures;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    [Collection(nameof(CorpusParserCollection))]
+    public class NameFragmentParserTests
+    {
+        private readonly CorpusParserFixture corpusParserFixture;
+
+        public NameFragmentParserTests(CorpusParserFixture corpusParserFixture)
+        {
+            this.corpusParserFixture = corpusParserFixture;
+        }
+
+        [Fact]
+        public void Name()
+        {
+            // Arrange
+            var xml = @"<name xmlns=""http://www.tei-c.org/ns/1.0"">
+                  <w norm=""HTTPS""
+            lemma=""HTTPS""
+            ana=""mte:Npmsn""
+            msd=""UPosTag=PROPN|Case=Nom|Gender=Masc|Number=Sing""
+            xml:id=""GosVL07_kungf.tok5881"">https</w>
+                </name>";
+            var nameEl = XElement.Parse(xml);
+            var parser = new NameFragmentParser(corpusParserFixture.FragmentParserFactory);
+
+            // Act
+            var tokens = parser.GetTokens(nameEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("https").WithStandardForm("HTTPS").WithLemma("HTTPS").WithMsd("mte:Npmsn"),
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+
+        [Fact]
+        public void AnonymizedName()
+        {
+            // Arrange
+            var xml = @"<name xmlns=""http://www.tei-c.org/ns/1.0"" type=""anon"">
+                <gap reason=""anon"" quantity=""1"" unit=""words"">
+                <desc>[ime]</desc>
+                </gap>
+                <gap reason=""anon"" quantity=""1"" unit=""words"">
+                <desc>[priimek]</desc>
+                </gap>
+                </name>";
+            var nameEl = XElement.Parse(xml);
+            var parser = new NameFragmentParser(corpusParserFixture.FragmentParserFactory);
+
+            // Act
+            var tokens = parser.GetTokens(nameEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("[ime]").WithStandardForm("[ime]").WithLemma("[ime]").WithMsd("mte:Xp"),
+                new Token(TokenType.Word).WithConversationForm("[priimek]").WithStandardForm("[priimek]").WithLemma("[priimek]").WithMsd("mte:Xp"),
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+
+        [Fact]
+        public void AnonymizedNameEmpty()
+        {
+            var xml = @"<name xmlns=""http://www.tei-c.org/ns/1.0"" type=""anon"">
+                <gap reason=""anon"" />
+                </name>";
+
+            var nameEl = XElement.Parse(xml);
+            var parser = new NameFragmentParser(corpusParserFixture.FragmentParserFactory);
+
+            // Act
+            var tokens = parser.GetTokens(nameEl);
+
+            // Assert
+            var expected = new List<Token>();
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/PauseFragmentParserTests.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    public class PauseFragmentParserTests
+    {
+        [Fact]
+        public void Pause()
+        {
+            // Arrange
+            var xml = @"<pause xmlns=""http://www.tei-c.org/ns/1.0"" />";
+            var pauseEl = XElement.Parse(xml);
+            var parser = new PauseFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(pauseEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("[pavza]").WithStandardForm("[pavza]").WithLemma("[pavza]").WithMsd("mte:Xt")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/SegmentFragmentParserTests.cs

@@ -0,0 +1,47 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Fixtures;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    [Collection(nameof(CorpusParserCollection))]
+    public class SegmentFragmentParserTests
+    {
+        private readonly CorpusParserFixture corpusParserFixture;
+
+        public SegmentFragmentParserTests(CorpusParserFixture corpusParserFixture)
+        {
+            this.corpusParserFixture = corpusParserFixture;
+        }
+
+        [Fact]
+        public void Segment()
+        {
+            // Arrange
+            var xml = @"<seg xmlns=""http://www.tei-c.org/ns/1.0"" xml:id=""GosVL02_kleme.s4"">
+                <w lemma=""in""
+            ana=""mte:Cc""
+            msd=""UPosTag=CCONJ""
+            xml:id=""GosVL02_kleme.tok92"">in</w>
+                </seg>";
+            var segmentEl = XElement.Parse(xml);
+            var parser = new SegmentFragmentParser(corpusParserFixture.FragmentParserFactory);
+
+            // Act
+            var tokens = parser.GetTokens(segmentEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("in").WithStandardForm("in").WithLemma("in").WithMsd("mte:Cc")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/VocalFragmentParserTests.cs

@@ -0,0 +1,54 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    public class VocalFragmentParserTests
+    {
+        [Fact]
+        public void EmptyVocal()
+        {
+            // Arrange
+            var xml = @"<vocal xmlns=""http://www.tei-c.org/ns/1.0"" type=""voice""/>";
+            var vocalEl = XElement.Parse(xml);
+            var parser = new VocalFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(vocalEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("[prazen]").WithStandardForm("[prazen]").WithLemma("[prazen]").WithMsd("mte:Xt")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+
+        [Fact]
+        public void Vocal()
+        {
+            // Arrange
+            var xml = @"<vocal xmlns=""http://www.tei-c.org/ns/1.0"" type=""pause"">
+                <desc>eee</desc>
+                </vocal>";
+            var vocalEl = XElement.Parse(xml);
+            var parser = new VocalFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(vocalEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("[eee]").WithStandardForm("[eee]").WithLemma("[eee]").WithMsd("mte:Xt")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}

src/Gos.Tests.Unit/Services/Framework/CorpusParser/WordFragmentParserTests.cs

@@ -0,0 +1,117 @@
+using System.Collections.Generic;
+using System.Xml.Linq;
+using Gos.Core.Entities;
+using Gos.Core.Extensions;
+using Gos.Core.Model;
+using Gos.Services.Framework.Fragments;
+using Gos.Tests.Unit.Services.Framework.Comparers;
+using Xunit;
+
+namespace Gos.Tests.Unit.Services.Framework.CorpusParser
+{
+    public class WordFragmentParserTests
+    {
+        [Fact]
+        public void ConversationalFormSameAsStandardForm()
+        {
+            // Arrange
+            var xml = @"<w xmlns=""http://www.tei-c.org/ns/1.0"" lemma=""med""
+            ana=""mte:Si""
+            msd=""UPosTag=ADP|Case=Ins""
+            xml:id=""GosVL01_pravo.tok44"">med</w>";
+            var wordEl = XElement.Parse(xml);
+            var parser = new WordFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(wordEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("med").WithStandardForm("med").WithLemma("med").WithMsd("mte:Si")
+            };
+
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+
+        [Fact]
+        public void ConversationalFormDifferentAsStandardForm()
+        {
+            // Arrange
+            var xml = @"<w xmlns=""http://www.tei-c.org/ns/1.0"" norm=""nekoliko""
+            lemma=""nekoliko""
+            ana=""mte:Rgp""
+            msd=""UPosTag=ADV|Degree=Pos""
+            xml:id=""GosVL01_pravo.tok47"">nekolk</w>";
+            var wordEl = XElement.Parse(xml);
+            var parser = new WordFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(wordEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("nekolk").WithStandardForm("nekoliko").WithLemma("nekoliko").WithMsd("mte:Rgp")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+
+        [Fact]
+        public void OneWordInConversationalFormMultipleWordsInStandardForm()
+        {
+            // Arrange
+            var xml = @"<w xmlns=""http://www.tei-c.org/ns/1.0"" xml:id=""GosVL07_kungf.tok252"">novte<w norm=""ne""
+            lemma=""ne""
+            ana=""mte:Q""
+            msd=""UPosTag=PART|Polarity=Neg""
+            xml:id=""GosVL07_kungf.tok252.n1""/>
+                <w norm=""boste""
+            lemma=""biti""
+            ana=""mte:Va-f2p-n""
+            msd=""UPosTag=AUX|Mood=Ind|Number=Plur|Person=2|Polarity=Pos|Tense=Fut|VerbForm=Fin""
+            xml:id=""GosVL07_kungf.tok252.n2""/>
+                </w>";
+            var wordEl = XElement.Parse(xml);
+            var parser = new WordFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(wordEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("novte").WithStandardForm("ne").WithLemma("ne").WithMsd("mte:Q"),
+                new Token(TokenType.Word).WithStandardForm("boste").WithLemma("biti").WithMsd("mte:Va-f2p-n")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+
+        [Fact]
+        public void MultipleWordsInConversationalFormOneWordInStandardForm()
+        {
+            // Arrange
+            var xml = @"<w xmlns=""http://www.tei-c.org/ns/1.0"" norm=""JJ""
+            lemma=""JJ""
+            ana=""mte:Npmsn""
+            msd=""UPosTag=PROPN|Case=Nom|Gender=Masc|Number=Sing""
+            xml:id=""GosVL07_kungf.tok926"">
+                <w xml:id=""GosVL07_kungf.tok926.o0"">Džej</w>
+                <w xml:id=""GosVL07_kungf.tok926.o1"">Džej</w>
+                </w>";
+            var wordEl = XElement.Parse(xml);
+            var parser = new WordFragmentParser();
+
+            // Act
+            var tokens = parser.GetTokens(wordEl);
+
+            // Assert
+            var expected = new List<Token>()
+            {
+                new Token(TokenType.Word).WithConversationForm("Džej").WithStandardForm("JJ").WithLemma("JJ").WithMsd("mte:Npmsn"),
+                new Token(TokenType.Word).WithConversationForm("Džej")
+            };
+            Assert.Equal(expected, tokens, new FragmentParserTokenComparer());
+        }
+    }
+}