First full release

express/models/comment.js

@@ -1,6 +1,10 @@
 const db = require('./db')
 const debug = require('debug')('termPortal:models/comment')
 const User = require('../models/user')
+const {
+  portalAdminInitialEmail,
+  portalAdminInitialPassword
+} = require('../config/keys')
 
 class Comment {
   // Deserialize flat data into an organized comment object.
@@ -311,24 +315,21 @@ async function seedMockUsersInDb() {
 }
 
 async function seedPortalAdmin() {
-  const MOCK_ADMIN_BASE = 'admin'
-
   const {
-    rows: [mockAdmin]
-  } = await db.query('SELECT id FROM "user" WHERE username = $1', [
-    MOCK_ADMIN_BASE
-  ])
+    rows: [{ exists }]
+  } = await db.query(
+    "SELECT EXISTS (SELECT 1 FROM user_role WHERE role_name = 'portal admin')"
+  )
 
-  if (mockAdmin) {
-    return `Portal admin already exists (username: ${MOCK_ADMIN_BASE}, password: ${MOCK_ADMIN_BASE})`
-  }
+  if (exists) return 'Skipping creation of portal admin (already exists)'
 
+  const MOCK_ADMIN_BASE = 'admin'
   const adminUser = {
     username: MOCK_ADMIN_BASE,
     firstName: MOCK_ADMIN_BASE,
     lastName: MOCK_ADMIN_BASE,
-    password: MOCK_ADMIN_BASE,
-    email: `${MOCK_ADMIN_BASE}@rsdo.com`
+    password: portalAdminInitialPassword,
+    email: portalAdminInitialEmail
   }
   const userId = await User.create(adminUser)
   const assignAdminRole = db.query(
@@ -345,7 +346,7 @@ async function seedPortalAdmin() {
     [adminUser.username]
   )
   await Promise.all([assignAdminRole, activateAdminUser])
-  return `Successfully seeded portal admin (username: ${MOCK_ADMIN_BASE}, password: ${MOCK_ADMIN_BASE})`
+  return `Successfully created portal admin (username: ${adminUser.username}, password: ${adminUser.password})`
 }
 
 async function seedConsultants() {

express/models/consultancy-entry.js

@@ -70,6 +70,7 @@ class ConsultancyEntry {
 
   // Fetch consultancy entry by ID
   // to_char(time_created,'HH24:MI:SS DD/MM/YYYY')
+  // TODO i18n date format
   static async fetchByIdWithFormattedTime(id) {
     const { rows: fetchedConsEntry } = await db.query(
       `
@@ -679,6 +680,7 @@ class ConsultancyEntry {
   }
 
   // (Re)index specific consultancy entry into consultancy search index.
+  // TODO i18n name_sl
   static async indexIntoSearchEngine(entryId, shouldWait) {
     const values = [entryId]
     const text = `

express/models/dictionary.js

@@ -1,8 +1,15 @@
+const { mkdir, open, unlink } = require('fs/promises')
 const Cursor = require('pg-cursor')
 const db = require('./db')
 const { intoDbArray, getInstanceSetting } = require('./helpers')
-const { deserialize, bulkIndex } = require('./helpers/dictionary')
+const {
+  deserialize,
+  bulkIndex,
+  getExportFilesPath
+} = require('./helpers/dictionary')
 const { readFileIntoDb } = require('./helpers/dictionary/import-file')
+const { transformAndAppend } = require('./helpers/dictionary/export-file')
+const { origin } = require('../config/keys')
 // const debug = require('debug')('termPortal:models/dictionary')
 
 class Dictionary {
@@ -25,6 +32,7 @@ class Dictionary {
   }
 
   // Fetch all dictionaries from DB.
+  // TODO i18n name_sl
   static async fetchAll() {
     // TODO Implement SQL stored procedures or functions.
     const { rows: fetchedDictionaries } = await db.query(`
@@ -65,6 +73,7 @@ class Dictionary {
   }
   */
 
+  // TODO i18n name_sl
   // Fetch all dictionaries from DB for which the user has at least one dictionary role.
   static async fetchAllByUser(userId) {
     // TODO Implement SQL stored procedures or functions.
@@ -75,7 +84,12 @@ class Dictionary {
         time_modified,
         status,
         count_entries,
-        count_comments
+        count_comments,
+        (
+          SELECT administration
+          FROM user_role
+          WHERE user_id = $1 and dictionary_id = id
+        ) as is_admin
       FROM dictionary
       WHERE id IN (
         SELECT dictionary_id
@@ -86,8 +100,8 @@ class Dictionary {
     `
     const values = [userId]
     const { rows: fetchedDictionaries } = await db.query(text, values)
-    const deserializedDictionaries = fetchedDictionaries.map(
-      dictionary => new this(dictionary)
+    const deserializedDictionaries = fetchedDictionaries.map(dictionary =>
+      deserialize.dictionary(dictionary)
     )
     return deserializedDictionaries
   }
@@ -138,13 +152,13 @@ class Dictionary {
   }
 
   // Fetch all languages from DB.
-  static async fetchAllLanguages(lang) {
+  static async fetchAllLanguages(lang, excludeSlovene) {
     // TODO Implement SQL stored procedures or functions.
     const { rows: fetchedLanguages } = await db.query(`
       SELECT
         id,
         ${lang}
-      FROM language
+      FROM language${excludeSlovene ? "\nWHERE code <> 'sl'" : ''}
       ORDER BY ${lang}`)
     const deserializedLanguages = fetchedLanguages.map(language =>
       deserialize.language(language)
@@ -413,15 +427,15 @@ class Dictionary {
       description,
       l.name_sl languageSl,
       ds.name_sl domainSecondarySl,
-      lp.name portalname,
+      lp.name portalnamesl,
       lp.code portalcode
     FROM
       dictionary d
-      INNER JOIN dictionary_language dl ON dl.dictionary_id = d.id
-      INNER JOIN language l ON dl.language_id = l.id
+      LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
+      LEFT JOIN language l ON dl.language_id = l.id
       LEFT JOIN dictionary_domain_secondary dds on dds.dictionary_id = d.id 
       LEFT JOIN domain_secondary ds ON dds.domain_secondary_id = ds.id
-      INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
+      LEFT JOIN domain_primary dp ON d.domain_primary_id = dp.id
       LEFT JOIN linked_dictionary ld ON  ld.target_dictionary_id = d.id
       LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
     WHERE d.id = $1`
@@ -447,7 +461,7 @@ class Dictionary {
       d.author,
       dp.name_sl domain_primary,
       description,
-      lp.name portalname,
+      lp.name portalnamesl,
       lp.code portalcode
     FROM
       dictionary d
@@ -509,7 +523,7 @@ class Dictionary {
       d.author,
       dp.name_sl domain_primary,
       description,
-      lp.name portalname,
+      lp.name portalnamesl,
       lp.code portalcode
     FROM
       dictionary d
@@ -577,12 +591,13 @@ class Dictionary {
       distinct (d.id),
       d.${lang} dictionarysl,
       d.count_entries,
+      d.count_comments,
       to_char(d.time_modified,'YYYY-MM-DD') time_modified,
       d.issn,
       d.author,
       dp.name_sl domain_primary,
       description,
-      lp.name portalname,
+      lp.name portalnamesl,
       lp.code portalcode
     FROM
       dictionary d
@@ -590,7 +605,7 @@ class Dictionary {
       INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
       LEFT JOIN linked_dictionary ld ON  ld.target_dictionary_id = d.id
       LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
-      WHERE d.name_sl LIKE '%' || $1 || '%' ${queryAppend}
+      WHERE d.status = 'published' AND LOWER(d.name_sl) LIKE '%' || LOWER($1) || '%' ${queryAppend}
       ORDER BY ${orderBy}
       LIMIT $2
       OFFSET $3`
@@ -643,7 +658,7 @@ class Dictionary {
       INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
       LEFT JOIN linked_dictionary ld ON  ld.target_dictionary_id = d.id
       LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
-      WHERE d.name_sl LIKE '%' || $1 || '%' ${queryAppend}`
+      WHERE d.status = 'published' AND d.name_sl LIKE '%' || $1 || '%' ${queryAppend}`
 
     const { rows } = await db.query(text, [searchQuery])
 
@@ -656,8 +671,8 @@ class Dictionary {
       count(d.id)
       FROM
       dictionary d
-      INNER JOIN dictionary_language dl ON dl.dictionary_id = d.id
-      INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
+      LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
+      LEFT JOIN domain_primary dp ON d.domain_primary_id = dp.id
       LEFT JOIN linked_dictionary ld ON  ld.target_dictionary_id = d.id
       LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id`
 
@@ -666,6 +681,23 @@ class Dictionary {
     return rows[0]
   }
 
+  static async fetchAllDictionariesPublishedCount() {
+    const text = `
+    SELECT
+      count(d.id)
+      FROM
+      dictionary d
+      LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
+      LEFT JOIN domain_primary dp ON d.domain_primary_id = dp.id
+      LEFT JOIN linked_dictionary ld ON  ld.target_dictionary_id = d.id
+      LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
+      WHERE d.status = 'published'`
+
+    const { rows } = await db.query(text)
+
+    return rows[0]
+  }
+
   // Fetch single dictionary data for editing structure from DB.
   static async fetchDictionaryWithEditStructure(dictionaryId) {
     const text = `
@@ -734,10 +766,11 @@ class Dictionary {
     SELECT
     d.id,
     ${lang} dictionarysl,
-    dp.name_sl domain_primary
+    dp.name_sl domain_primary,
+    d.count_comments
     FROM dictionary d
     INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
-    where d.time_published is not NULL
+    where d.time_published is not NULL and d.status = 'published'
     ORDER BY d.time_published desc
     limit 3`
 
@@ -984,15 +1017,62 @@ class Dictionary {
     }
   }
 
+  // Import entries from extraction into DB.
+  static async importFromExtraction(dictionaryId, userId, termCandidates) {
+    await db.transaction(async dbClient => {
+      // TODO Depending on performance, consider batching requests.
+      for (const {
+        kanonicnaoblika: term,
+        definicija: other
+      } of termCandidates) {
+        const values = [
+          dictionaryId,
+          false,
+          'suggestion',
+          term || null,
+          userId,
+          null,
+          null,
+          null,
+          null,
+          [],
+          null,
+          null,
+          null,
+          [],
+          other || null,
+          [],
+          null,
+          null,
+          null
+        ]
+
+        const text = `SELECT entry_new (${db.genParamStr(values)})`
+
+        await dbClient.query(text, values)
+      }
+
+      await this.updateMetadataAfterModifyingEntries(dictionaryId, dbClient)
+    })
+  }
+
   // Index entries of specific dictionary from DB into search engine.
   static async indexIntoSearchEngine(dictionaryId) {
-    // TODO Luka: I expect "source" needing a rework once linked portals and dictionaries start working.
-    const source = {
-      code: await getInstanceSetting('portal_code'),
-      name: await getInstanceSetting('portal_name')
-    }
     const dbClient = await db.getClient()
     try {
+      const {
+        rows: [{ code: linkedPortalCode, name: linkedPortalName } = {}]
+      } = await dbClient.query(
+        'SELECT p.code, p.name FROM linked_dictionary d LEFT JOIN linked_portal p ON p.id = d.linked_portal_id WHERE d.id = $1',
+        [dictionaryId]
+      )
+
+      // TODO i18n Luka: index portal name for both languages?
+      const source = {
+        code: linkedPortalCode ?? (await getInstanceSetting('portal_code')),
+        name: linkedPortalName ?? (await getInstanceSetting('portal_name_sl'))
+      }
+
       const queryValues = [dictionaryId]
 
       const dictionaryQueryText = `
@@ -1060,7 +1140,7 @@ class Dictionary {
               )
             )
             FROM entry_foreign ef
-            LEFT JOIN LANGUAGE l ON l.id = ef.language_id
+            LEFT JOIN language l ON l.id = ef.language_id
             WHERE entry_id = e.id
           ) foreign_entries
         FROM entry e
@@ -1154,6 +1234,41 @@ class Dictionary {
     return result
   }
 
+  static async fetchFilteredPaginationDomainLabels(
+    dictionaryId,
+    query,
+    resultsPerPage,
+    page
+  ) {
+    const {
+      rows: [{ result }]
+    } = await db.query(
+      `
+        SELECT jsonb_build_object(
+          'pages_total', (
+            SELECT CEIL(COUNT(*) / $3::float)
+            FROM domain_label
+            WHERE dictionary_id = $1 and name LIKE '%' || $2 || '%'
+          ),
+          'results', ARRAY(
+            SELECT jsonb_build_object(
+              'id', id,
+              'name', name,
+              'isVisible', is_visible
+            )
+            FROM domain_label
+            WHERE dictionary_id = $1 and LOWER(name) LIKE '%' || LOWER($2) || '%'
+            ORDER BY name
+            LIMIT $3
+            OFFSET $4
+          )
+        ) result`,
+      [dictionaryId, query, resultsPerPage, resultsPerPage * (page - 1)]
+    )
+
+    return result
+  }
+
   // Fetch all secondary domains from DB.
   static async fetchAllSecondaryDomains(resultsPerPage, page) {
     const {
@@ -1184,6 +1299,38 @@ class Dictionary {
     return result
   }
 
+  // Fetch filtered secondary domains from DB.
+  static async fetchFilteredSecondaryDomains(query, resultsPerPage, page) {
+    const {
+      rows: [{ result }]
+    } = await db.query(
+      `
+        SELECT jsonb_build_object(
+          'pages_total', (
+            SELECT CEIL(COUNT(*) / $2::float)
+            FROM domain_secondary
+            WHERE name_sl LIKE '%' || $1 || '%'
+          ),
+          'results', ARRAY(
+            SELECT jsonb_build_object(
+              'id', id,
+              'isApproved', approved,
+              'nameSl', name_sl,
+              'nameEn', name_en
+            )
+            FROM domain_secondary
+            WHERE LOWER(name_sl) LIKE '%' || LOWER($1) || '%'
+            ORDER BY name_sl
+            LIMIT $2
+            OFFSET $3
+          )
+        ) result`,
+      [query, resultsPerPage, resultsPerPage * (page - 1)]
+    )
+
+    return result
+  }
+
   static async updateDomainLabel(dictionaryId, data) {
     const dbClient = await db.getClient()
     try {
@@ -1343,24 +1490,54 @@ class Dictionary {
     }
   }
 
-  static async fetchAllImports(dictionaryId) {
-    const text = `
-      SELECT
-        time_started,
-        status,
-        delete_existing_entries,
-        file_format,
-        count_valid_entries
-      FROM import_file_job
-      WHERE dictionary_id = $1`
+  // static async fetchAllImports(dictionaryId) {
+  //   const text = `
+  //     SELECT
+  //       time_started,
+  //       status,
+  //       delete_existing_entries,
+  //       file_format,
+  //       count_valid_entries
+  //     FROM import_file_job
+  //     WHERE dictionary_id = $1`
 
-    const value = [dictionaryId]
-    const { rows: fetchedImports } = await db.query(text, value)
+  //   const value = [dictionaryId]
+  //   const { rows: fetchedImports } = await db.query(text, value)
 
-    const deserializedImports = fetchedImports.map(oneImport =>
-      deserialize.imports(oneImport)
+  //   const deserializedImports = fetchedImports.map(oneImport =>
+  //     deserialize.imports(oneImport)
+  //   )
+  //   return deserializedImports
+  // }
+
+  static async fetchAllImports(dictionaryId, resultsPerPage, page) {
+    const {
+      rows: [{ result }]
+    } = await db.query(
+      `
+      SELECT jsonb_build_object(
+        'pages_total', (
+          SELECT CEIL(COUNT(*) / $2::float)
+          FROM import_file_job
+          WHERE dictionary_id = $1
+        ),
+        'results', ARRAY(
+          SELECT jsonb_build_object(
+            'time_started', time_started,
+            'status', status,
+            'delete_existing_entries', delete_existing_entries,
+            'file_format', file_format,
+            'count_valid_entries', count_valid_entries
+          )
+          FROM import_file_job
+          WHERE dictionary_id = $1
+          LIMIT $2
+          OFFSET $3
+        )
+      ) result`,
+      [dictionaryId, resultsPerPage, resultsPerPage * (page - 1)]
     )
-    return deserializedImports
+    return result
   }
 
   static async delete(dictionaryId) {
@@ -1368,6 +1545,464 @@ class Dictionary {
     const value = [dictionaryId]
     await db.query(text, value)
   }
+
+  // static async fetchExports(dictionaryId) {
+  //   const text = `
+  //     SELECT
+  //       id,
+  //       status,
+  //       to_char(time_created, 'FMDD. FMMM. YYYY') date_created,
+  //       entry_count,
+  //       is_valid_filter,
+  //       is_published_filter,
+  //       is_terminology_reviewed_filter,
+  //       is_language_reviewed_filter,
+  //       status_filter,
+  //       export_file_format
+  //     FROM dictionary_export
+  //     WHERE dictionary_id = $1
+  //     ORDER BY id DESC`
+  //   const value = [dictionaryId]
+
+  //   const { rows: fetchedExports } = await db.query(text, value)
+
+  //   const deserializedExports = fetchedExports.map(eachExport =>
+  //     deserialize.exports(eachExport)
+  //   )
+  //   return deserializedExports
+  // }
+
+  static async fetchExports(dictionaryId, resultsPerPage, page) {
+    const {
+      rows: [{ result }]
+    } = await db.query(
+      `
+      SELECT jsonb_build_object(
+        'pages_total', (
+          SELECT CEIL(COUNT(*) / $2::float)
+          FROM dictionary_export
+          WHERE dictionary_id = $1
+        ),
+        'results', ARRAY(
+          SELECT jsonb_build_object(
+            'id', id,
+            'status', status,
+            'time_created', time_created,
+            'entry_count', entry_count,
+            'is_valid_filter', is_valid_filter,
+            'is_published_filter', is_published_filter,
+            'is_terminology_reviewed_filter', is_terminology_reviewed_filter,
+            'is_language_reviewed_filter', is_language_reviewed_filter,
+            'status_filter', status_filter,
+            'export_file_format', export_file_format
+          )
+          FROM dictionary_export
+          WHERE dictionary_id = $1
+          LIMIT $2
+          OFFSET $3
+        )
+      ) result`,
+      [dictionaryId, resultsPerPage, resultsPerPage * (page - 1)]
+    )
+    return result
+  }
+
+  static async beginExport(dictionaryId, exportParams) {
+    const text = `
+      INSERT INTO dictionary_export (
+        dictionary_id,
+        is_valid_filter,
+        is_published_filter,
+        is_terminology_reviewed_filter,
+        is_language_reviewed_filter,
+        status_filter,
+        export_file_format
+      )
+      VALUES ($1, $2, $3, $4, $5, $6, $7)
+      RETURNING id
+    `
+
+    const values = [
+      dictionaryId,
+      exportParams.isValidFilter,
+      exportParams.isPublishedFilter,
+      exportParams.isTerminologyReviewedFilter,
+      exportParams.isLanguageReviewedFilter,
+      exportParams.statusFilter,
+      exportParams.exportFileFormat
+    ]
+
+    const {
+      rows: [{ id }]
+    } = await db.query(text, values)
+
+    return id
+  }
+
+  static async processExport(exportId) {
+    const dbClient = await db.getClient()
+    let exportFilePath
+    let exportFile
+    let cursor
+
+    try {
+      const exportQueryText = `
+        UPDATE dictionary_export
+        SET status = 'in progress', time_started = NOW()
+        WHERE id = $1
+        RETURNING
+          dictionary_id,
+          is_valid_filter,
+          is_published_filter,
+          is_terminology_reviewed_filter,
+          is_language_reviewed_filter,
+          status_filter,
+          export_file_format
+      `
+
+      const exportQueryParams = [exportId]
+
+      const {
+        rows: [
+          {
+            dictionary_id: dictionaryId,
+            is_valid_filter: isValidFilter,
+            is_published_filter: isPublishedFilter,
+            is_terminology_reviewed_filter: isTerminologyReviewedFilter,
+            is_language_reviewed_filter: isLanguageReviewedFilter,
+            status_filter: statusFilter,
+            export_file_format: exportFileFormat
+          }
+        ]
+      } = await dbClient.query(exportQueryText, exportQueryParams)
+
+      const dictionaryQueryText = `
+        SELECT
+          entries_have_domain_labels,
+          entries_have_label,
+          entries_have_definition,
+          entries_have_synonyms,
+          entries_have_links,
+          entries_have_other,
+          entries_have_foreign_languages,
+          entries_have_foreign_definitions,
+          entries_have_foreign_synonyms,
+          entries_have_images,
+          entries_have_audio,
+          entries_have_videos
+        FROM dictionary
+        WHERE id = $1
+      `
+
+      const dictionaryQueryParams = [dictionaryId]
+
+      const {
+        rows: [dictionaryStructure]
+      } = await dbClient.query(dictionaryQueryText, dictionaryQueryParams)
+
+      const exportFields = {
+        domainLabels: dictionaryStructure.entries_have_domain_labels,
+        label: dictionaryStructure.entries_have_label,
+        definition: dictionaryStructure.entries_have_definition,
+        synonyms: dictionaryStructure.entries_have_synonyms,
+        links: dictionaryStructure.entries_have_links,
+        other: dictionaryStructure.entries_have_other,
+        foreignTerms: dictionaryStructure.entries_have_foreign_languages,
+        foreignDefinitions:
+          dictionaryStructure.entries_have_foreign_definitions,
+        foreignSynonyms: dictionaryStructure.entries_have_foreign_synonyms,
+        images: dictionaryStructure.entries_have_images,
+        audio: dictionaryStructure.entries_have_audio,
+        videos: dictionaryStructure.entries_have_videos
+      }
+
+      let entryQueryText = `
+        SELECT
+          term${
+            exportFields.label
+              ? `,
+                label`
+              : ''
+          }${
+        exportFields.definition
+          ? `,
+            definition`
+          : ''
+      }${
+        exportFields.synonyms
+          ? `,
+            synonym synonyms`
+          : ''
+      }${
+        exportFields.other
+          ? `,
+            other`
+          : ''
+      }${
+        exportFields.images
+          ? `,
+            image`
+          : ''
+      }${
+        exportFields.audio
+          ? `,
+            audio`
+          : ''
+      }${
+        exportFields.videos
+          ? `,
+            video`
+          : ''
+      }${
+        exportFields.domainLabels
+          ? `,
+            ARRAY(
+              SELECT name
+              FROM entry_domain_label edl
+              LEFT JOIN domain_label dl ON dl.id = edl.domain_label_id
+              WHERE entry_id = e.id
+            ) domain_labels`
+          : ''
+      }${
+        exportFields.links
+          ? `,
+            ARRAY(
+              SELECT jsonb_build_object(
+                'link', link,
+                'type', type)
+              FROM entry_link
+              WHERE entry_id = e.id
+            ) links`
+          : ''
+      }${
+        exportFields.foreignTerms
+          ? `,
+            ARRAY(
+              SELECT jsonb_build_object(
+                'lang_code', l.code,
+                'terms', ef.term${
+                  exportFields.foreignDefinitions
+                    ? `,
+                      'definition', ef.definition`
+                    : ''
+                }${
+              exportFields.foreignSynonyms
+                ? `,
+                    'synonyms', ef.synonym`
+                : ''
+            }
+              )
+            FROM dictionary_language dl
+            LEFT JOIN language l ON l.id = dl.language_id
+            LEFT JOIN entry_foreign ef ON ef.language_id = l.id
+            WHERE dl.dictionary_id = $1 AND ef.entry_id = e.id
+            ORDER BY dl.selection_order
+            ) foreign_entries`
+          : ''
+      }
+        FROM entry e
+        WHERE
+          e.dictionary_id = $1`
+
+      const entryQueryParams = [dictionaryId]
+
+      if (isValidFilter !== null) {
+        entryQueryParams.push(isValidFilter)
+        entryQueryText += ` AND is_valid = $${entryQueryParams.length}`
+      }
+      if (isPublishedFilter !== null) {
+        entryQueryParams.push(isPublishedFilter)
+        entryQueryText += ` AND is_published = $${entryQueryParams.length}`
+      }
+      if (isTerminologyReviewedFilter !== null) {
+        entryQueryParams.push(isTerminologyReviewedFilter)
+        entryQueryText += ` AND is_terminology_reviewed = $${entryQueryParams.length}`
+      }
+      if (isLanguageReviewedFilter !== null) {
+        entryQueryParams.push(isLanguageReviewedFilter)
+        entryQueryText += ` AND is_language_reviewed = $${entryQueryParams.length}`
+      }
+      if (statusFilter !== null) {
+        entryQueryParams.push(statusFilter)
+        entryQueryText += ` AND status = $${entryQueryParams.length}`
+      }
+      entryQueryText += '\nORDER BY term'
+
+      const exportFilesPath = getExportFilesPath(dictionaryId)
+      await mkdir(exportFilesPath, { recursive: true })
+      exportFilePath = `${exportFilesPath}/${exportId}`
+      exportFile = await open(exportFilePath, 'ax')
+
+      const isXmlFileFormat = exportFileFormat === 'xml'
+      const isDsvFileFormat = ['csv', 'tsv'].includes(exportFileFormat)
+      const isTbxFileFormat = exportFileFormat === 'tbx'
+      let dsvConfig
+      if (isXmlFileFormat) {
+        const openingMarkup =
+          '<?xml version="1.0" encoding="utf-8"?>\n<dictionary>\n'
+        await exportFile.write(openingMarkup)
+      } else if (isDsvFileFormat) {
+        if (exportFileFormat === 'csv') dsvConfig = { delimiter: ';' }
+        else if (exportFileFormat === 'tsv') dsvConfig = { delimiter: '\t' }
+
+        if (exportFields.foreignTerms) {
+          const { rows: languages } = await dbClient.query(
+            `
+              SELECT l.code
+              FROM dictionary d
+              LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
+              LEFT JOIN language l ON l.id = dl.language_id
+              WHERE d.id = $1
+              ORDER BY dl.selection_order
+            `,
+            [dictionaryId]
+          )
+
+          dsvConfig.languageCodes = languages.map(language => language.code)
+        }
+
+        const fieldNamesArr = ['term']
+        if (exportFields.domainLabels) fieldNamesArr.push('domainLabels')
+        if (exportFields.label) fieldNamesArr.push('label')
+        if (exportFields.definition) fieldNamesArr.push('def')
+        if (exportFields.synonyms) fieldNamesArr.push('syns')
+        if (exportFields.links) fieldNamesArr.push('links')
+        if (exportFields.other) fieldNamesArr.push('other')
+        dsvConfig.languageCodes?.forEach(languageCode => {
+          fieldNamesArr.push(`[${languageCode}]fTerms`)
+          if (exportFields.foreignDefinitions) {
+            fieldNamesArr.push(`[${languageCode}]fDef`)
+          }
+          if (exportFields.foreignSynonyms) {
+            fieldNamesArr.push(`[${languageCode}]fSyns`)
+          }
+        })
+        if (exportFields.images) fieldNamesArr.push('images')
+        if (exportFields.audio) fieldNamesArr.push('audios')
+        if (exportFields.videos) fieldNamesArr.push('videos')
+        const headerLine = `${fieldNamesArr.join(dsvConfig.delimiter)}\n`
+        await exportFile.write(headerLine)
+      } else if (isTbxFileFormat) {
+        const {
+          rows: [tbxMetadata]
+        } = await dbClient.query(
+          `
+            SELECT
+              name_sl,
+              name_en,
+              author,
+              to_char(time_modified, 'YYYY-MM-DD') modified_date_string,
+              status,
+              (
+                SELECT to_char(time_created, 'YYYY-MM-DD')
+                FROM dictionary_export de
+                WHERE de.id = $1
+              ) export_date_string
+            FROM dictionary
+            WHERE id = $2
+          `,
+          [exportId, dictionaryId]
+        )
+        const portalName = await getInstanceSetting('portal_name_sl')
+        const authorsString = tbxMetadata.author?.join(', ')
+        const urlPublished =
+          tbxMetadata.status === 'published'
+            ? new URL(`/slovarji/${dictionaryId}/o-slovarju`, origin).href
+            : null
+
+        let openingMarkup = '<?xml version="1.0" encoding="utf-8"?>\n'
+        openingMarkup += '<!DOCTYPE martif SYSTEM "TBXcoreStructV02.dtd">\n'
+        openingMarkup += '<martif type="TBX" xml:lang="sl">\n<martifHeader>\n'
+        openingMarkup += '<fileDesc>\n<titleStmt>\n'
+        openingMarkup += `<title>${tbxMetadata.name_sl}</title>\n`
+        openingMarkup += `<note xml:lang="en">${tbxMetadata.name_en}</note>\n`
+        openingMarkup += '</titleStmt>\n<publicationStmt>\n'
+        openingMarkup += `<p>Datum objave: ${tbxMetadata.export_date_string}</p>\n`
+        openingMarkup +=
+          '<p>Avtorske pravice: Delo je dostopno pod pogoji licence CC BY 4.0.</p>\n'
+        openingMarkup += '</publicationStmt>\n<sourceDesc>\n'
+        openingMarkup += `<p>Vir: ${portalName}</p>\n`
+        if (authorsString) openingMarkup += `<p>Avtorji: ${authorsString}</p>\n`
+        openingMarkup += `<p>Datum objave: ${tbxMetadata.modified_date_string}</p>\n`
+        if (urlPublished) {
+          openingMarkup += `<p>Mesto objave: ${urlPublished}</p>\n`
+        }
+        openingMarkup += '</sourceDesc>\n</fileDesc>\n'
+        openingMarkup += '</martifHeader>\n<text>\n<body>\n'
+
+        await exportFile.write(openingMarkup)
+      }
+
+      cursor = dbClient.query(new Cursor(entryQueryText, entryQueryParams))
+
+      let entries = []
+      let batchEntriesCount
+      let entriesWritten = 0
+      do {
+        // Keep getting and writing entries to file in batches of 100.
+        entries = await cursor.read(100)
+        batchEntriesCount = entries.length
+
+        if (batchEntriesCount) {
+          await transformAndAppend(
+            entries,
+            exportFields,
+            exportFile,
+            exportFileFormat,
+            dsvConfig
+          )
+          entriesWritten += batchEntriesCount
+        }
+      } while (batchEntriesCount === 100)
+
+      if (isXmlFileFormat) {
+        const closingMarkup = '</dictionary>\n'
+        await exportFile.write(closingMarkup)
+      } else if (isTbxFileFormat) {
+        const closingMarkup = '</body>\n</text>\n</martif>\n'
+        await exportFile.write(closingMarkup)
+      }
+
+      await dbClient.query(
+        "UPDATE dictionary_export SET status = 'finished', time_finished = NOW(), entry_count = $1 WHERE id = $2",
+        [entriesWritten, exportId]
+      )
+    } catch (error) {
+      await cursor?.close()
+      await dbClient.query(
+        "UPDATE dictionary_export SET status = 'failed', time_finished = NOW() WHERE id = $1",
+        [exportId]
+      )
+      await unlink(exportFilePath)
+      throw error
+    } finally {
+      dbClient.release()
+      await exportFile?.close()
+    }
+  }
+
+  static async fetchExportDownloadMetadata(exportId) {
+    const {
+      rows: [fetchedMetadata]
+    } = await db.query(
+      `
+      SELECT
+        e.status,
+        e.dictionary_id,
+        d.name_sl_short name_string,
+        to_char(e.time_created, 'YYYYMMDDHH24MISS') time_string,
+        e.export_file_format
+      FROM dictionary_export e
+      LEFT JOIN dictionary d ON d.id = e.dictionary_id
+      WHERE e.id = $1`,
+      [exportId]
+    )
+
+    const deserializedMetadata =
+      deserialize.exportDownloadMetadata(fetchedMetadata)
+
+    return deserializedMetadata
+  }
 }
 
 module.exports = Dictionary

express/models/email.js

@@ -3,15 +3,28 @@ const htmlToText = require('nodemailer-html-to-text').htmlToText()
 const {
   smtpHost,
   smtpPort,
-  smtpTlsRejectUnauthorized,
+  smtpUser,
+  smtpPassword,
+  smtpSecure,
+  smtpRequireTls,
+  smtpAllowInvalidCerts,
   smtpFrom
 } = require('../config/keys')
 
 const options = {
   host: smtpHost,
   port: smtpPort,
-  tls: { rejectUnauthorized: smtpTlsRejectUnauthorized }
+  secure: smtpSecure,
+  requireTLS: smtpRequireTls,
+  tls: { rejectUnauthorized: !smtpAllowInvalidCerts }
 }
+if (smtpUser || smtpPassword) {
+  options.auth = {
+    user: smtpUser,
+    pass: smtpPassword
+  }
+}
+
 const defaults = { from: smtpFrom }
 
 const transporter = nodemailer.createTransport(options, defaults)

express/models/entry.js

@@ -184,7 +184,12 @@ Entry.fetchFull = async entryId => {
             SELECT jsonb_strip_nulls(
              jsonb_build_object(
                 'version', version,
-                'version_time', version_time
+                'version_time', version_time,
+                'version_author', (
+                  SELECT username
+                  FROM "user"
+                  WHERE id = (version_snapshot['version_author'])::int
+                )
               )
             )
             FROM entry_version_history
@@ -232,6 +237,7 @@ Entry.fetchFullWithOrderedForeignLanguages = async entryId => {
           'audio', e.audio,
           'video', e.video,
           'time_modified', e.time_modified,
+          'external_url', e.external_url,
           'domain_labels', ARRAY(
             SELECT name
             FROM entry_domain_label edl
@@ -421,6 +427,8 @@ Entry.deleteAllLinks = async dictionaryId => {
 
 // (Re)index specific entry into entry search index.
 Entry.indexIntoSearchEngine = async (entryId, shouldWait) => {
+  // TODO If this method is ever used for linked portals/dictionaries,
+  // TODO rework the source object below (already done in Dictionary.indexIntoSearchEngine).
   const values = [entryId]
   const text = `
     SELECT
@@ -465,7 +473,7 @@ Entry.indexIntoSearchEngine = async (entryId, shouldWait) => {
               )
             )
             FROM entry_foreign ef
-            LEFT JOIN LANGUAGE l ON l.id = ef.language_id
+            LEFT JOIN language l ON l.id = ef.language_id
             WHERE entry_id = e.id
           )
         )
@@ -495,10 +503,11 @@ Entry.indexIntoSearchEngine = async (entryId, shouldWait) => {
 
   const { dictionary, primary_domain: primaryDomain } = dataToIndex
   let { entry } = dataToIndex
-  // TODO Luka: I expect "source" needing a rework once linked portals and dictionaries start working.
+
+  // TODO i18n Luka: index portal name for both languages?
   const source = {
     code: await getInstanceSetting('portal_code'),
-    name: await getInstanceSetting('portal_name')
+    name: await getInstanceSetting('portal_name_sl')
   }
 
   entry = prepareEntryForIndexing(entry)
@@ -618,13 +627,17 @@ Entry.update = async (userId, entry) => {
 // Fetch a single version snapshot of a single entry from DB.
 Entry.fetchVersionSnapshot = async (entryId, version) => {
   const {
-    rows: [{ version_snapshot: historySnapshot }]
+    rows: [{ version_snapshot: historySnapshot, author }]
   } = await db.query(
-    'SELECT version_snapshot FROM entry_version_history WHERE entry_id = $1 and version = $2',
+    `SELECT v.version_snapshot, (
+        SELECT u.username
+        FROM "user" u
+        WHERE u.id = (v.version_snapshot['version_author'])::int) as author
+    FROM entry_version_history v WHERE v.entry_id = $1 and v.version = $2`,
     [entryId, version]
   )
 
-  return historySnapshot
+  return { data: historySnapshot, author }
 }
 
 /* Fetch by language and entry Id. Note that this version includes the language name */

express/models/extraction.js

@@ -12,13 +12,14 @@ const {
   getFileNamesInFolder,
   getFileStatsInFolder
 } = require('./helpers/extraction')
+const { extractionApiOrigin } = require('../config/keys')
 
 const Extraction = {}
 
 // Fetch all extractions for a specific user.
 Extraction.fetchAllForUser = async userId => {
   const { rows: fetchedExtractions } = await db.query(
-    'SELECT id, name, status, corpus_id, oss_params, time_started, time_finished FROM extraction WHERE user_id = $1 ORDER BY id',
+    'SELECT id, name, status, corpus_id, oss_params, time_started, time_finished FROM extraction WHERE user_id = $1 ORDER BY status ASC, id DESC, time_finished DESC',
     [userId]
   )
 
@@ -74,19 +75,19 @@ Extraction.fetch = async id => {
   return deserialize.extraction(fetchedExtraction)
 }
 
-// Fetch author email of a specific extraction entry from DB.
-Extraction.fetchAuthorEmail = async id => {
+// Fetch data of the author of a specific extraction entry from DB.
+Extraction.fetchAuthorData = async id => {
   const {
-    rows: [{ email }]
+    rows: [authorData]
   } = await db.query(
-    `SELECT u.email
+    `SELECT u.email, u.language
     FROM extraction e
     LEFT JOIN "user" u ON u.id = e.user_id
     WHERE e.id = $1`,
     [id]
   )
 
-  return email
+  return authorData
 }
 
 // Update extraction entry in DB.
@@ -160,6 +161,18 @@ Extraction.fetchTermCandidatesCount = async function (extractionId) {
   return termCandidates.length
 }
 
+// Fetch term candidates slice for a specific extraction.
+Extraction.fetchTermCandidatesSlice = async function (
+  extractionId,
+  fromIndex,
+  toIndex
+) {
+  const termCandidatesJson = await this.fetchTermCandidatesJson(extractionId)
+  const termCandidates = JSON.parse(termCandidatesJson).terminoloski_kandidati
+  const termCandidatesSlice = termCandidates.slice(fromIndex, toIndex)
+  return termCandidatesSlice
+}
+
 // Mark extraction from own documents as began.
 Extraction.beginOwn = async (extractionId, documentsNames) => {
   let timeStarted
@@ -195,20 +208,20 @@ Extraction.beginOwn = async (extractionId, documentsNames) => {
 Extraction.beginOss = async extractionId => {
   let timeStarted
   await db.transaction(async dbClient => {
-    const insertExtractionJob = dbClient.query(
-      'INSERT INTO extraction_job (extraction_id, job_type, filename) VALUES ($1, $2, $3)',
-      [extractionId, 'oss term candidates', '']
-    )
-    const updateExtraction = dbClient.query(
-      "UPDATE extraction SET status = 'in progress', time_started = NOW() WHERE id = $1 RETURNING time_started",
-      [extractionId]
-    )
-
     ;[
       {
         rows: [{ time_started: timeStarted }]
       }
-    ] = await Promise.all([updateExtraction, insertExtractionJob])
+    ] = await Promise.all([
+      dbClient.query(
+        "UPDATE extraction SET status = 'in progress', time_started = NOW() WHERE id = $1 RETURNING time_started",
+        [extractionId]
+      ),
+      dbClient.query(
+        'INSERT INTO extraction_job (extraction_id, job_type, filename) VALUES ($1, $2, $3)',
+        [extractionId, 'oss term candidates', '']
+      )
+    ])
   })
 
   return timeStarted
@@ -235,6 +248,7 @@ Extraction.processOwn = async function (extractionId, extractionName) {
   const documentNames = await this.fetchAllDocumentsNames(extractionId)
   const conllusPath = getConllusPath(extractionId)
   const conllusPaths = []
+  const MAX_BODY_LENGTH = 10 ** 9 // 1 GB
   // Using remote API, transform each document into conllu format.
   for (const documentName of documentNames) {
     const filePath = `${documentsPath}/${documentName}`
@@ -242,17 +256,18 @@ Extraction.processOwn = async function (extractionId, extractionName) {
     form.append('file', createReadStream(filePath), documentName)
     try {
       const { data: data1 } = await axios.post(
-        'http://rsdo.lhrs.feri.um.si:8080/datotekaVConlluAsync',
+        `${extractionApiOrigin}/datotekaVConlluAsync`,
         form,
         {
           headers: {
             ...form.getHeaders()
-          }
+          },
+          maxBodyLength: MAX_BODY_LENGTH
         }
       )
       const remotejobId = +data1.check_job_url.split('/').at(-1)
       await db.query(
-        "UPDATE extraction_job SET status = 'in progress', remote_job_id = $1 WHERE extraction_id = $2 AND job_type = $3 AND filename = $4",
+        "UPDATE extraction_job SET status = 'in progress', remote_job_id = $1, time_started = NOW() WHERE extraction_id = $2 AND job_type = $3 AND filename = $4",
         [remotejobId, extractionId, 'doc to conllu', documentName]
       )
 
@@ -262,10 +277,15 @@ Extraction.processOwn = async function (extractionId, extractionName) {
       while (true) {
         await sleep(5)
         const { data: data2 } = await axios.get(
-          `http://rsdo.lhrs.feri.um.si:8080/job/${remotejobId}`
+          `${extractionApiOrigin}/job/${remotejobId}`
         )
         if (data2.finished_on) {
-          if (data2.job_status !== 'finished processing (OK)') throw Error()
+          if (data2.job_status !== 'finished processing (OK)') {
+            throw Error(
+              `Remote job with id ${remotejobId} failed with result:\n${data2.job_result}`
+            )
+          }
+
           // TODO Read the response as a stream and try to parse it's contents into a file (write stream)('stream-json' package?).
           const fileSavePath = `${conllusPath}/${documentName}.conllu`
           await writeFile(fileSavePath, data2.job_result)
@@ -277,7 +297,8 @@ Extraction.processOwn = async function (extractionId, extractionName) {
           break
         }
       }
-    } catch {
+    } catch (error) {
+      logExtractionError(error, extractionId, 'doc to conllu', documentName)
       await failTheJob(extractionId, 'doc to conllu', documentName)
     }
   }
@@ -309,15 +330,18 @@ Extraction.processOwn = async function (extractionId, extractionName) {
 
   try {
     const { data: data3 } = await axios.post(
-      'http://rsdo.lhrs.feri.um.si:8080/izlusciAsync',
+      `${extractionApiOrigin}/izlusciAsync`,
       {
         conllus: conllusArr,
-        prepovedaneBesede: Array.from(stopTermsSet)
-      }
+        prepovedaneBesede: Array.from(stopTermsSet),
+        // TODO Enabled for all cases. Add a switch for users later.
+        definicije: true
+      },
+      { maxBodyLength: MAX_BODY_LENGTH }
     )
     const remotejobId = +data3.check_job_url.split('/').at(-1)
     await db.query(
-      "UPDATE extraction_job SET status = 'in progress', remote_job_id = $1 WHERE extraction_id = $2 AND job_type = $3 AND filename = $4",
+      "UPDATE extraction_job SET status = 'in progress', remote_job_id = $1, time_started = NOW() WHERE extraction_id = $2 AND job_type = $3 AND filename = $4",
       [remotejobId, extractionId, 'conllus to term candidates', '']
     )
 
@@ -325,14 +349,21 @@ Extraction.processOwn = async function (extractionId, extractionName) {
     while (true) {
       await sleep(5)
       const { data: data4 } = await axios.get(
-        `http://rsdo.lhrs.feri.um.si:8080/job/${remotejobId}`
+        `${extractionApiOrigin}/job/${remotejobId}`
       )
       if (data4.finished_on) {
-        if (data4.job_status !== 'finished processing (OK)') throw Error()
+        const { job_result: jobResult } = data4
+        if (
+          data4.job_status !== 'finished processing (OK)' ||
+          !jobResult.terminoloski_kandidati
+        ) {
+          throw Error(
+            `Remote job with id ${remotejobId} failed with result:\n${jobResult}`
+          )
+        }
+
         // TODO Read the response as a stream and try to parse it's contents into a file (write stream)('stream-json' package?).
-        await writeFile(termCandidatesPath, JSON.stringify(data4.job_result))
-        // TODO Once returned JSON is properly formed, use the bottom line instead.
-        // await writeFile(termCandidatesPath, data4.job_result.terminoloski_kandidati)
+        await writeFile(termCandidatesPath, JSON.stringify(jobResult))
         await db.query(
           "UPDATE extraction_job SET status = 'finished', time_finished = NOW() WHERE extraction_id = $1 AND job_type = $2 AND filename = $3",
           [extractionId, 'conllus to term candidates', '']
@@ -340,8 +371,10 @@ Extraction.processOwn = async function (extractionId, extractionName) {
         break
       }
     }
-  } catch {
+  } catch (error) {
+    logExtractionError(error, extractionId, 'conllus to term candidates')
     await failTheJob(extractionId, 'conllus to term candidates', '')
+    await skipConcordancerJob(extractionId)
     await failExtraction(extractionId)
     return
   }
@@ -350,7 +383,7 @@ Extraction.processOwn = async function (extractionId, extractionName) {
   // Start concondancer corpus processing.
   try {
     await db.query(
-      "UPDATE extraction_job SET status = 'in progress' WHERE extraction_id = $1 AND job_type = $2 AND filename = $3",
+      "UPDATE extraction_job SET status = 'in progress', time_started = NOW() WHERE extraction_id = $1 AND job_type = $2 AND filename = $3",
       [extractionId, 'concordancer', '']
     )
     console.log('CREATING CORPUS')
@@ -361,32 +394,90 @@ Extraction.processOwn = async function (extractionId, extractionName) {
     } = await axios.post('http://concordancer:5000/dashboard/corpus', {
       title: extractionName
     })
+
+    // Wait for creation of corpus.
+    while (true) {
+      console.log('SLEEP FOR 5 SECS')
+      await sleep(5)
+      const {
+        data: { status }
+      } = await axios.get(
+        `http://concordancer:5000/dashboard/corpus/${corpusId}`
+      )
+
+      if (status === 'Creating') continue
+      if (status === 'Active') break
+      throw Error('Error creating concorcander corpus')
+    }
     console.log('CORPUS CREATED')
-    console.log('SLEEP FOR 10 SECS')
-    await sleep(10)
+
+    const inProgressStatusList = [
+      'Waiting',
+      'Importing',
+      'ImportingCompleted',
+      'Indexing',
+      'IndexingCompleted'
+    ]
     for (const conlluPath of conllusPaths) {
       const textPathParts = conlluPath.split('/')
       textPathParts[0] = '/data'
       const textPath = textPathParts.join('/')
       console.log('ADDING TEXT')
-      await axios.post(
+      const {
+        data: {
+          entityInfo: { id: textId }
+        }
+      } = await axios.post(
         `http://concordancer:5000/dashboard/corpus/${corpusId}/text`,
         { sourceFile: textPath }
       )
+
+      // Wait for text ingestion.
+      while (true) {
+        console.log('SLEEP FOR 5 SECS')
+        await sleep(5)
+        const {
+          data: { status }
+        } = await axios.get(
+          `http://concordancer:5000/dashboard/corpus/${corpusId}/text/${textId}`
+        )
+
+        if (inProgressStatusList.includes(status)) continue
+        if (status === 'Active') break
+        throw Error('Error importing concorcander text')
+      }
       console.log('TEXT ADDED')
-      console.log('SLEEP FOR 10 SECS')
-      await sleep(10)
     }
 
     const termListPathParts = termCandidatesPath.split('/')
     termListPathParts[0] = '/data'
     const termListPath = termListPathParts.join('/')
     console.log('ADDING TERMS')
-    await axios.post(
+    const {
+      data: {
+        entityInfo: { id: termListId }
+      }
+    } = await axios.post(
       `http://concordancer:5000/dashboard/corpus/${corpusId}/termList`,
       { sourceFile: termListPath }
     )
+
+    // Wait for term list ingestion.
+    while (true) {
+      console.log('SLEEP FOR 5 SECS')
+      await sleep(5)
+      const {
+        data: { status }
+      } = await axios.get(
+        `http://concordancer:5000/dashboard/corpus/${corpusId}/termList/${termListId}`
+      )
+
+      if (inProgressStatusList.includes(status)) continue
+      if (status === 'Active') break
+      throw Error('Error importing concorcander text')
+    }
     console.log('TERMS ADDED')
+
     await db.query(
       "UPDATE extraction_job SET status = 'finished', time_finished = NOW() WHERE extraction_id = $1 AND job_type = $2 AND filename = $3",
       [extractionId, 'concordancer', '']
@@ -397,9 +488,8 @@ Extraction.processOwn = async function (extractionId, extractionName) {
       [corpusId, extractionId]
     )
     console.log('EXTRACTION SUCCESSFUL')
-  } catch (e) {
-    console.log('EXTRACTION ERROR')
-    console.log(e)
+  } catch (error) {
+    logExtractionError(error, extractionId, 'concordancer')
     await failTheJob(extractionId, 'concordancer', '')
     await failExtraction(extractionId)
   }
@@ -435,15 +525,17 @@ Extraction.processOss = async function (extractionId, ossParams) {
     ...(ossParams.documentType && { vrste: ossParams.documentType }),
     ...(ossParams.keywords && { kljucneBesede: ossParams.keywords }),
     ...(ossParams.domainUdk && { udk: ossParams.domainUdk }),
-    ...(stopTerms.length && { prepovedaneBesede: stopTerms })
+    ...(stopTerms.length && { prepovedaneBesede: stopTerms }),
+    // TODO Enabled for all cases. Add a switch for users later.
+    definicije: true
   })
 
-  const extractApiUrl = `http://rsdo.lhrs.feri.um.si:8080/oss/izlusciPoIskanjuAsync?${searchParams}`
+  const extractApiUrl = `${extractionApiOrigin}/oss/izlusciPoIskanjuAsync?${searchParams}`
   try {
     const { data: data1 } = await axios.get(extractApiUrl)
     const remotejobId = +data1.check_job_url.split('/').at(-1)
     await db.query(
-      "UPDATE extraction_job SET status = 'in progress', remote_job_id = $1 WHERE extraction_id = $2 AND job_type = $3 AND filename = $4",
+      "UPDATE extraction_job SET status = 'in progress', remote_job_id = $1, time_started = NOW() WHERE extraction_id = $2 AND job_type = $3 AND filename = $4",
       [remotejobId, extractionId, 'oss term candidates', '']
     )
 
@@ -451,15 +543,21 @@ Extraction.processOss = async function (extractionId, ossParams) {
     while (true) {
       await sleep(5)
       const { data: data2 } = await axios.get(
-        `http://rsdo.lhrs.feri.um.si:8080/job/${remotejobId}`
+        `${extractionApiOrigin}/job/${remotejobId}`
       )
       if (data2.finished_on) {
-        if (data2.job_status !== 'finished processing (OK)') throw Error()
+        if (
+          data2.job_status !== 'finished processing (OK)' ||
+          !Array.isArray(data2.job_result?.terminoloski_kandidati)
+        ) {
+          throw Error(
+            `Remote job with id ${remotejobId} failed with result:\n${data2.job_result}`
+          )
+        }
+
         // TODO Read the response as a stream and try to parse it's contents into a file (write stream)('stream-json' package?).
         const termCandidatesPath = getTermCandidatesPath(extractionId)
         await writeFile(termCandidatesPath, JSON.stringify(data2.job_result))
-        // TODO Once returned JSON is properly formed, use the bottom line instead.
-        // await writeFile(termCandidatesPath, data4.job_result.terminoloski_kandidati)
         await db.query(
           "UPDATE extraction_job SET status = 'finished', time_finished = NOW() WHERE extraction_id = $1 AND job_type = $2 AND filename = $3",
           [extractionId, 'oss term candidates', '']
@@ -474,7 +572,8 @@ Extraction.processOss = async function (extractionId, ossParams) {
       "UPDATE extraction SET status = 'finished', time_finished = NOW() WHERE id = $1",
       [extractionId]
     )
-  } catch {
+  } catch (error) {
+    logExtractionError(error, extractionId, 'oss term candidates')
     await failTheJob(extractionId, 'oss term candidates', '')
     await failExtraction(extractionId)
   }
@@ -487,6 +586,13 @@ async function failTheJob(extractionId, jobType, documentName) {
   )
 }
 
+async function skipConcordancerJob(extractionId) {
+  await db.query(
+    "UPDATE extraction_job SET status = 'skipped' WHERE extraction_id = $1 AND job_type = $2",
+    [extractionId, 'concordancer']
+  )
+}
+
 async function failExtraction(extractionId) {
   await db.query(
     "UPDATE extraction SET status = 'failed', time_finished = NOW() WHERE id = $1",
@@ -498,4 +604,18 @@ function sleep(seconds) {
   return new Promise(resolve => setTimeout(resolve, seconds * 1000))
 }
 
+function logExtractionError(error, extractionId, jobType, filename) {
+  // eslint-disable-next-line no-console
+  console.error(
+    Error(`Failed extraction job:
+      extractionId: ${extractionId},
+      jobType: ${jobType},
+      filename: ${filename}`)
+  )
+
+  if (error.isAxiosError) error = Error(`Axios message: ${error.message}`)
+  // eslint-disable-next-line no-console
+  console.error(error)
+}
+
 module.exports = Extraction

express/models/helpers/dictionary/export-file.js

@@ -0,0 +1,486 @@
+const xmlFlow = require('xml-flow')
+const xss = require('xss')
+
+exports.transformAndAppend = async (
+  entries,
+  exportFields,
+  exportFile,
+  exportFileFormat,
+  dsvConfig
+) => {
+  let transformEntry
+  if (exportFileFormat === 'xml') transformEntry = intoXml
+  else if (dsvConfig) transformEntry = intoDsv(dsvConfig)
+  else if (exportFileFormat === 'tbx') transformEntry = intoTbx
+  else throw Error('Specified export format not supported yet')
+
+  for (const entry of entries) {
+    const transformedEntry = transformEntry(entry, exportFields)
+    await exportFile.write(`${transformedEntry}\n`)
+  }
+}
+
+function intoXml(entry, exportFields) {
+  const entryObj = { $name: 'entry', $markup: [] }
+
+  if (entry.term) {
+    entryObj.$markup.push({ term: entry.term })
+  }
+
+  if (exportFields.domainLabels && entry.domain_labels.length) {
+    const domainLabelsObj = {
+      $name: 'domainLabels',
+      $markup: entry.domain_labels.map(domainLabel => {
+        return { domainLabel }
+      })
+    }
+    entryObj.$markup.push(domainLabelsObj)
+  }
+
+  if (exportFields.label && entry.label) {
+    entryObj.$markup.push({ label: entry.label })
+  }
+
+  if (exportFields.definition && entry.definition) {
+    entryObj.$markup.push({ def: entry.definition })
+  }
+
+  if (exportFields.synonyms && entry.synonyms?.length) {
+    const SynonymsObj = {
+      $name: 'syns',
+      $markup: entry.synonyms.map(synonym => {
+        return { syn: synonym }
+      })
+    }
+    entryObj.$markup.push(SynonymsObj)
+  }
+
+  if (exportFields.links && entry.links.length) {
+    const LinksObj = {
+      $name: 'links',
+      $markup: entry.links.map(linkObj => {
+        return {
+          $name: 'link',
+          $attrs: { type: linkObj.type },
+          $text: linkObj.link
+        }
+      })
+    }
+    entryObj.$markup.push(LinksObj)
+  }
+
+  if (exportFields.other && entry.other) {
+    entryObj.$markup.push({ other: entry.other })
+  }
+
+  if (exportFields.foreignTerms && entry.foreign_entries.length) {
+    const fLangsObj = {
+      $name: 'fLangs',
+      $markup: entry.foreign_entries.map(fEntryObj => {
+        const fLangObj = {
+          $name: 'fLang',
+          $attrs: { lang: fEntryObj.lang_code },
+          $markup: []
+        }
+
+        if (fEntryObj.terms) {
+          const fTermsObj = {
+            $name: 'fTerms',
+            $markup: fEntryObj.terms.map(term => {
+              return { fTerm: term }
+            })
+          }
+          fLangObj.$markup.push(fTermsObj)
+        }
+
+        if (exportFields.foreignDefinitions && fEntryObj.definition) {
+          fLangObj.$markup.push({ fDef: fEntryObj.definition })
+        }
+
+        if (exportFields.foreignSynonyms && fEntryObj.synonyms?.length) {
+          const fSynsObj = {
+            $name: 'fSyns',
+            $markup: fEntryObj.synonyms.map(synonym => {
+              return { fSyn: synonym }
+            })
+          }
+          fLangObj.$markup.push(fSynsObj)
+        }
+
+        return fLangObj
+      })
+    }
+    entryObj.$markup.push(fLangsObj)
+  }
+
+  const shouldExportImages = exportFields.images && entry.image?.length
+  const shouldExportAudio = exportFields.audio && entry.audio?.length
+  const shouldExportVideos = exportFields.videos && entry.video?.length
+  const shouldCreateMm =
+    shouldExportImages || shouldExportAudio || shouldExportVideos
+
+  if (shouldCreateMm) {
+    const MmObj = {
+      $name: 'mm',
+      $markup: []
+    }
+
+    if (shouldExportImages) {
+      entry.image.forEach(el => {
+        MmObj.$markup.push({ image: el })
+      })
+    }
+
+    if (shouldExportAudio) {
+      entry.audio.forEach(el => {
+        MmObj.$markup.push({ audio: el })
+      })
+    }
+
+    if (shouldExportVideos) {
+      entry.video.forEach(el => {
+        MmObj.$markup.push({ video: el })
+      })
+    }
+
+    entryObj.$markup.push(MmObj)
+  }
+
+  return xmlFlow.toXml(entryObj, { escape: str => str })
+}
+
+function intoDsv({ delimiter, languageCodes }) {
+  return function (entry, exportFields) {
+    const fieldsArr = [intoDsvField(entry.term)]
+
+    if (exportFields.domainLabels) {
+      fieldsArr.push(
+        intoDsvField(
+          entry.domain_labels.reduce((agg, domainLabel, index) => {
+            agg += `${index ? '\n' : ''}${domainLabel}`
+            return agg
+          }, '')
+        )
+      )
+    }
+
+    if (exportFields.label) fieldsArr.push(intoDsvField(entry.label))
+
+    if (exportFields.definition) fieldsArr.push(intoDsvField(entry.definition))
+
+    if (exportFields.synonyms) {
+      fieldsArr.push(
+        intoDsvField(
+          entry.synonyms?.reduce((agg, synonym, index) => {
+            agg += `${index ? '\n' : ''}${synonym}`
+            return agg
+          }, '')
+        )
+      )
+    }
+
+    if (exportFields.links) {
+      fieldsArr.push(
+        intoDsvField(
+          entry.links.reduce((agg, linkObj, index) => {
+            agg += `${index ? '\n' : ''}[${linkObj.type[0]}t]${linkObj.link}`
+            return agg
+          }, '')
+        )
+      )
+    }
+
+    if (exportFields.other) fieldsArr.push(intoDsvField(entry.other))
+
+    languageCodes?.forEach(languageCode => {
+      const fEntryObj = entry.foreign_entries.find(
+        entryObj => entryObj.lang_code === languageCode
+      )
+
+      fieldsArr.push(
+        intoDsvField(
+          fEntryObj?.terms?.reduce((agg, term, index) => {
+            agg += `${index ? '\n' : ''}${term}`
+            return agg
+          }, '')
+        )
+      )
+      if (exportFields.foreignDefinitions) {
+        fieldsArr.push(intoDsvField(fEntryObj?.definition))
+      }
+      if (exportFields.foreignSynonyms) {
+        fieldsArr.push(
+          intoDsvField(
+            fEntryObj?.synonyms?.reduce((agg, synonym, index) => {
+              agg += `${index ? '\n' : ''}${synonym}`
+              return agg
+            }, '')
+          )
+        )
+      }
+    })
+
+    if (exportFields.images) {
+      fieldsArr.push(
+        intoDsvField(
+          entry.image?.reduce((agg, el, index) => {
+            agg += `${index ? '\n' : ''}${el}`
+            return agg
+          }, '')
+        )
+      )
+    }
+
+    if (exportFields.audio) {
+      fieldsArr.push(
+        intoDsvField(
+          entry.audio?.reduce((agg, el, index) => {
+            agg += `${index ? '\n' : ''}${el}`
+            return agg
+          }, '')
+        )
+      )
+    }
+
+    if (exportFields.videos) {
+      fieldsArr.push(
+        intoDsvField(
+          entry.video?.reduce((agg, el, index) => {
+            agg += `${index ? '\n' : ''}${el}`
+            return agg
+          }, '')
+        )
+      )
+    }
+
+    return fieldsArr.join(delimiter)
+  }
+}
+
+function intoDsvField(fieldString) {
+  if (!fieldString) return ''
+  return `"${fieldString.replaceAll('"', '""')}"`
+}
+
+const tbxLinkTypeMap = {
+  related: 'relatedConcept',
+  broader: 'relatedConceptBroader',
+  narrow: 'relatedConceptNarrower'
+}
+const brTagPattern = /<br[^>]*>/
+function intoTbx(entry, exportFields) {
+  const entryObj = { $name: 'termEntry', $markup: [] }
+
+  if (exportFields.images) {
+    entry.image?.forEach(el => {
+      entryObj.$markup.push({
+        $name: 'xref',
+        $attrs: { type: 'xGraphic' },
+        $text: el
+      })
+    })
+  }
+
+  if (exportFields.audio) {
+    entry.audio?.forEach(el => {
+      entryObj.$markup.push({
+        $name: 'xref',
+        $attrs: { type: 'xAudio' },
+        $text: el
+      })
+    })
+  }
+
+  if (exportFields.videos) {
+    entry.video?.forEach(el => {
+      entryObj.$markup.push({
+        $name: 'xref',
+        $attrs: { type: 'xVideo' },
+        $text: el
+      })
+    })
+  }
+
+  const slLangObj = {
+    $name: 'langSet',
+    $attrs: { 'xml:lang': 'sl' },
+    $markup: []
+  }
+
+  if (exportFields.label && entry.label) {
+    slLangObj.$markup.push({
+      $name: 'descrip',
+      $attrs: { type: 'explanation' },
+      $text: intoTbxMixed(entry.label)
+    })
+  }
+
+  if (exportFields.definition && entry.definition) {
+    slLangObj.$markup.push({
+      $name: 'descrip',
+      $attrs: { type: 'definition' },
+      $text: intoTbxMixed(entry.definition)
+    })
+  }
+
+  if (exportFields.links) {
+    entry.links.forEach(linkObj => {
+      slLangObj.$markup.push({
+        $name: 'descrip',
+        $attrs: { type: tbxLinkTypeMap[linkObj.type] },
+        $text: intoTbxMixed(linkObj.link)
+      })
+    })
+  }
+
+  if (exportFields.other && entry.other) {
+    const otherLines = entry.other.split(brTagPattern)
+    otherLines.forEach(line => {
+      slLangObj.$markup.push({
+        $name: 'descrip',
+        $attrs: { type: 'other' },
+        $text: intoTbxMixed(line)
+      })
+    })
+  }
+
+  const shouldExportTerm = entry.term
+  const shouldExportDomainLabels =
+    exportFields.domainLabels && entry.domain_labels.length
+  const shouldCreateTermNtig = shouldExportTerm || shouldExportDomainLabels
+
+  if (shouldCreateTermNtig) {
+    const termNtigObj = {
+      $name: 'ntig',
+      $markup: [{ $name: 'termGrp', $markup: [] }]
+    }
+    const termGrpMarkup = termNtigObj.$markup[0].$markup
+
+    if (shouldExportTerm) {
+      termGrpMarkup.push({ term: intoTbxMixed(entry.term) })
+      termGrpMarkup.push({
+        $name: 'termNote',
+        $attrs: { type: 'termType' },
+        $text: 'entryTerm'
+      })
+    }
+
+    if (shouldExportDomainLabels) {
+      entry.domain_labels.forEach(domainLabel => {
+        termGrpMarkup.push({
+          $name: 'termNote',
+          $attrs: { type: 'domain' },
+          $text: domainLabel
+        })
+      })
+    }
+
+    slLangObj.$markup.push(termNtigObj)
+  }
+
+  if (exportFields.synonyms) {
+    entry.synonyms?.forEach(synonym => {
+      slLangObj.$markup.push({
+        $name: 'ntig',
+        $markup: [
+          {
+            $name: 'termGrp',
+            $markup: [
+              { term: intoTbxMixed(synonym) },
+              {
+                $name: 'termNote',
+                $attrs: { type: 'termType' },
+                $text: 'synonym'
+              }
+            ]
+          }
+        ]
+      })
+    })
+  }
+
+  entryObj.$markup.push(slLangObj)
+
+  if (exportFields.foreignTerms) {
+    entry.foreign_entries.forEach(fEntryObj => {
+      const langSetObj = {
+        $name: 'langSet',
+        $attrs: { 'xml:lang': fEntryObj.lang_code },
+        $markup: []
+      }
+
+      if (exportFields.foreignDefinitions && fEntryObj.definition) {
+        langSetObj.$markup.push({
+          $name: 'descrip',
+          $attrs: { type: 'definition' },
+          $text: intoTbxMixed(fEntryObj.definition)
+        })
+      }
+
+      fEntryObj.terms?.forEach(term => {
+        langSetObj.$markup.push({
+          $name: 'ntig',
+          $markup: [
+            {
+              $name: 'termGrp',
+              $markup: [{ term: intoTbxMixed(term) }]
+            }
+          ]
+        })
+      })
+
+      if (exportFields.foreignSynonyms) {
+        fEntryObj.synonyms?.forEach(synonym => {
+          langSetObj.$markup.push({
+            $name: 'ntig',
+            $markup: [
+              {
+                $name: 'termGrp',
+                $markup: [
+                  { term: intoTbxMixed(synonym) },
+                  {
+                    $name: 'termNote',
+                    $attrs: { type: 'termType' },
+                    $text: 'synonym'
+                  }
+                ]
+              }
+            ]
+          })
+        })
+      }
+
+      entryObj.$markup.push(langSetObj)
+    })
+  }
+
+  return xmlFlow.toXml(entryObj, { escape: str => str })
+}
+
+const tbxRichFilter = new xss.FilterXSS({
+  whiteList: {
+    sup: [],
+    sub: [],
+    b: [],
+    i: []
+  },
+  stripIgnoreTag: true,
+  stripIgnoreTagBody: ['script', 'style']
+})
+const tagPattern = /<\s*(\/)?\s*([^\s/>]+)\s*>/g
+function intoTbxMixed(mixedContentStr) {
+  return tbxRichFilter
+    .process(mixedContentStr)
+    .replace(tagPattern, tbxMixedReplacer)
+}
+
+const tbxMixedTypeMap = {
+  sup: 'superscript',
+  sub: 'subscript',
+  b: 'bold',
+  i: 'italics'
+}
+function tbxMixedReplacer(match, closingSlash, tagName) {
+  if (closingSlash) return '</hi>'
+  return `<hi type="${tbxMixedTypeMap[tagName]}">`
+}

express/models/helpers/dictionary/import-file.js

@@ -250,7 +250,7 @@ function customTagHandler(tag, html, { isWhite, isClosing }) {
   const matchUrl = html.match(/href="?(?<url>https?:\/\/.*?)"?[\s>]/)
   const url = matchUrl ? xss.escapeAttrValue(matchUrl.groups.url) : undefined
 
-  return `<a href${url ? `="${url}" target="_blank"` : ''}>`
+  return `<a href="${url || ''}" target="_blank">`
 }
 
 function toText(markupObj) {

express/models/helpers/dictionary/index.js

@@ -1,7 +1,22 @@
 const { removeHtmlTags } = require('../../helpers')
 const { searchEngineClient, ENTRY_INDEX } = require('../../search-engine')
+const { DATA_FILES_PATH } = require('../../../config/settings')
 
 exports.deserialize = {
+  dictionary(dictionary) {
+    const deserializedDictionary = {
+      id: dictionary.id,
+      nameSl: dictionary.name_sl,
+      timeModified: dictionary.time_modified,
+      status: dictionary.status,
+      countEntries: dictionary.count_entries,
+      countComments: dictionary.count_comments,
+      isAdmin: dictionary.is_admin
+    }
+
+    return deserializedDictionary
+  },
+
   primaryDomain(domain) {
     const deserializedDomain = {
       id: domain.id,
@@ -102,16 +117,78 @@ exports.deserialize = {
     return deserializedDomainLabel
   },
 
-  imports(oneImport) {
-    const deserializedImports = {
-      timeStarted: oneImport.time_started,
-      status: oneImport.status,
-      deleteExisting: oneImport.delete_existing_entries,
-      fileFormat: oneImport.file_format,
-      countValidEntries: oneImport.count_valid_entries
+  // imports(oneImport) {
+  //   const deserializedImports = {
+  //     timeStarted: oneImport.time_started,
+  //     status: oneImport.status,
+  //     deleteExisting: oneImport.delete_existing_entries,
+  //     fileFormat: oneImport.file_format,
+  //     countValidEntries: oneImport.count_valid_entries
+  //   }
+
+  //   return deserializedImports
+  // },
+
+  exports(oneExport) {
+    const deserializedExports = {
+      id: oneExport.id,
+      status: oneExport.status,
+      dateCreated: oneExport.date_created,
+      entryCount: oneExport.entry_count ?? '',
+      typeString: `${
+        oneExport.is_valid_filter === true
+          ? 2
+          : oneExport.is_valid_filter === false
+          ? 3
+          : 1
+      }${
+        oneExport.is_published_filter === true
+          ? 2
+          : oneExport.is_published_filter === false
+          ? 3
+          : 1
+      }${
+        oneExport.status_filter === 'complete'
+          ? 2
+          : oneExport.status_filter === 'in_edit'
+          ? 3
+          : 1
+      }${
+        oneExport.is_terminology_reviewed_filter === true
+          ? 2
+          : oneExport.is_terminology_reviewed_filter === false
+          ? 3
+          : 1
+      }${
+        oneExport.is_language_reviewed_filter === true
+          ? 2
+          : oneExport.is_language_reviewed_filter === false
+          ? 3
+          : 1
+      }-${
+        oneExport.export_file_format === 'xml'
+          ? 1
+          : oneExport.export_file_format === 'csv'
+          ? 2
+          : oneExport.export_file_format === 'tsv'
+          ? 3
+          : 0
+      }`
     }
 
-    return deserializedImports
+    return deserializedExports
+  },
+
+  exportDownloadMetadata(metadata) {
+    const deserializedMetadata = {
+      exportStatus: metadata.status,
+      dictionaryId: metadata.dictionary_id,
+      nameString: metadata.name_string,
+      timeString: metadata.time_string,
+      fileFormat: metadata.export_file_format
+    }
+
+    return deserializedMetadata
   }
 }
 
@@ -194,3 +271,7 @@ function prepareEntryForIndexing(entry) {
 }
 
 exports.prepareEntryForIndexing = prepareEntryForIndexing
+
+exports.getExportFilesPath = dictId => {
+  return `${DATA_FILES_PATH}/dict_export/${dictId}`
+}

express/models/helpers/extraction.js

@@ -1,4 +1,5 @@
 const { readdir, stat } = require('fs/promises')
+const path = require('path')
 const { partial } = require('filesize')
 const { DATA_FILES_PATH } = require('../../config/settings')
 
@@ -43,17 +44,21 @@ exports.getFileNamesInFolder = async folderPath => {
   return filenames
 }
 
+exports.getFileStats = getFileStats
+
 exports.getFileStatsInFolder = async folderPath => {
   const filenames = await readdir(folderPath)
-  const fileStats = await Promise.all(
-    filenames.map(async filename => {
-      const filePath = `${folderPath}/${filename}`
-      const { mtimeMs: timeModified, size } = await stat(filePath)
-      const sizeHumanReadable = formatFileSize(size)
-      const fileStats = { filename, size: sizeHumanReadable, timeModified }
-      return fileStats
-    })
-  )
+  const filePaths = filenames.map(filename => `${folderPath}/${filename}`)
+  const fileStats = await Promise.all(filePaths.map(getFileStats))
+  fileStats.sort((a, b) => a.timeModified - b.timeModified)
+  return fileStats
+}
+
+async function getFileStats(filePath) {
+  const filename = path.basename(filePath)
+  const { mtimeMs: timeModified, size } = await stat(filePath)
+  const sizeHumanReadable = formatFileSize(size)
+  const fileStats = { filename, size: sizeHumanReadable, timeModified }
   return fileStats
 }
 

express/models/helpers/portal/index.js

@@ -9,8 +9,10 @@ exports.aggregateSettings = settings => {
 exports.deserialize = {
   settings(settings) {
     const deserializedSettings = {
-      name: settings.portal_name,
-      description: settings.portal_description,
+      nameSl: settings.portal_name_sl,
+      nameEn: settings.portal_name_en,
+      descriptionSl: settings.portal_description_sl,
+      descriptionEn: settings.portal_description_en,
       code: settings.portal_code,
       isExtractionEnabled: settings.is_extraction_enabled,
       isDictionariesEnabled: settings.is_dictionaries_enabled,

express/models/helpers/search/generate-query/consultancy/all.js

@@ -9,7 +9,7 @@ module.exports = function (filters, hitsPerPage, page) {
         filter: []
       }
     },
-    sort: ['_score', 'timeCreated']
+    sort: ['_score', { timeCreated: 'desc' }]
   }
 
   if (filters.status) {

express/models/helpers/search/generate-query/consultancy/phrase.js

@@ -45,7 +45,7 @@ module.exports = function (searchString, filters, hitsPerPage, page) {
         filter: []
       }
     },
-    sort: ['_score', 'timeCreated']
+    sort: ['_score', { timeCreated: 'desc' }]
   }
 
   if (filters.status) {

express/models/helpers/search/generate-query/consultancy/wildcard.js

@@ -39,7 +39,7 @@ module.exports = function (searchString, filters, hitsPerPage, page) {
         filter: []
       }
     },
-    sort: ['_score', 'timeCreated']
+    sort: ['_score', { timeCreated: 'desc' }]
   }
 
   if (filters.status) {

express/models/helpers/search/generate-query/consultancy/words.js

@@ -54,7 +54,7 @@ module.exports = function (searchString, filters, hitsPerPage, page) {
         filter: []
       }
     },
-    sort: ['_score', 'timeCreated']
+    sort: ['_score', { timeCreated: 'desc' }]
   }
 
   if (filters.status) {

express/models/helpers/search/generate-query/editor/all.js

@@ -2,7 +2,7 @@ const { EDITOR_MAX_HITS } = require('../../../../../config/settings')
 
 module.exports = function (dictionaryId, filters, searchFieldFilters) {
   const queryDsl = {
-    _source: ['id', 'isValid', 'isPublished', 'term'],
+    _source: ['id', 'isValid', 'isPublished', 'term', 'homonymSort'],
     fields: ['foreignEntries.terms'],
     script_fields: {
       commentActivityIndicator: {

express/models/helpers/user/index.js

@@ -7,6 +7,7 @@ exports.deserialize = {
       lastName: user.last_name,
       email: user.email,
       hitsPerPage: user.hits_per_page,
+      language: user.language,
       userRoles: user.user_roles,
       assignedConsultancyEntries: user.assigned_consultancy_entries
     }
@@ -21,7 +22,8 @@ exports.deserialize = {
       firstName: userData.first_name,
       lastName: userData.last_name,
       email: userData.email,
-      password: userData.password
+      status: userData.status,
+      language: userData.language
     }
 
     return deserializedData

express/models/inter_instance_sync.js

@@ -73,6 +73,9 @@ class InterInstanceSync {
       'SELECT l.code AS language_code, ef.entry_id, ef.term, ef.definition, ef.synonym FROM entry_foreign AS ef' +
       ' INNER JOIN language AS l ON l.id = ef.language_id' +
       ' WHERE entry_id = ANY($1::int[]) ORDER BY ef.entry_id, l.code'
+    const sqlLinks =
+      'SELECT entry_id, link, type FROM entry_link WHERE entry_id = ANY($1::int[]) ORDER BY entry_id, type, link'
+
     const { rows: entryList } = await db.query(sqlEntries, [
       dictionaryId,
       since
@@ -83,6 +86,7 @@ class InterInstanceSync {
     const { rows: translationList } = await db.query(sqlTranslations, [
       entryIds
     ])
+    const { rows: linkList } = await db.query(sqlLinks, [entryIds])
     let currentEntryId = 0
     let lastEntryId = 0
     translationList.forEach(t => {
@@ -102,6 +106,23 @@ class InterInstanceSync {
       }
       entry.translations.push(t)
     })
+    linkList.forEach(l => {
+      currentEntryId = l.entry_id
+      const entry = entryList.find(e => {
+        return e.id === currentEntryId
+      })
+      if (!entry) return
+      if (lastEntryId === 0) {
+        entry.links = []
+        lastEntryId = l.entry_id
+      } else if (currentEntryId !== lastEntryId) {
+        // new entry translations : save previous
+        entry.links = []
+        lastEntryId = currentEntryId
+        currentEntryId = l.entry_id
+      }
+      entry.links.push({ type: l.type, link: l.link })
+    })
     return entryList
   }
 }

express/models/portal.js

@@ -12,9 +12,11 @@ Portal.fetchInstanceSettings = async () => {
     WHERE
       name
     IN (
-      'portal_name',
+      'portal_name_sl',
+      'portal_name_en',
       'portal_code',
-      'portal_description',
+      'portal_description_sl',
+      'portal_description_en',
       'is_consultancy_enabled',
       'is_dictionaries_enabled',
       'is_extraction_enabled')`
@@ -32,9 +34,11 @@ Portal.updateInstaceSettings = async payload => {
   const isConsultancyEnabled = payload.isConsultancyEnabled ? 'T' : 'F'
 
   const values = [
-    payload.portalName,
+    payload.portalNameSl,
+    payload.portalNameEn,
     payload.portalCode,
-    payload.portalDescription,
+    payload.portalDescriptionSl,
+    payload.portalDescriptionEn,
     isExtractionEnabled,
     isDictionariesEnabled,
     isConsultancyEnabled
@@ -47,17 +51,21 @@ Portal.updateInstaceSettings = async payload => {
       value
     = CASE name
     WHEN
-      'portal_name' THEN $1
+      'portal_name_sl' THEN $1
     WHEN
-      'portal_code' THEN $2
+      'portal_name_en' THEN $2
     WHEN
-      'portal_description' THEN $3
+      'portal_code' THEN $3
     WHEN
-      'is_extraction_enabled' THEN $4
+      'portal_description_sl' THEN $4
     WHEN
-      'is_dictionaries_enabled' THEN $5
+      'portal_description_en' THEN $5
     WHEN
-      'is_consultancy_enabled' THEN $6
+      'is_extraction_enabled' THEN $6
+    WHEN
+      'is_dictionaries_enabled' THEN $7
+    WHEN
+      'is_consultancy_enabled' THEN $8
     ELSE value
     END`
 

express/models/system/eurotermbank.js

@@ -47,7 +47,7 @@ Eurotermbank.push = async () => {
               'definition', ef.definition,
               'synonyms', ef.synonym)
           FROM entry_foreign ef
-          LEFT JOIN LANGUAGE l ON l.id = ef.language_id
+          LEFT JOIN language l ON l.id = ef.language_id
           WHERE entry_id = e.id
           ) foreign_entries
         FROM entry e

express/models/user.js

@@ -17,6 +17,7 @@ User.create = async user => {
     user.email || null,
     bcryptHash || null
   ]
+  if (user.language) values.push(user.language)
 
   const text = `INSERT INTO "user" (
       username,
@@ -24,6 +25,7 @@ User.create = async user => {
       last_name,
       email,
       bcrypt_hash
+      ${user.language ? ', language' : ''}
     )
     VALUES (${db.genParamStr(values)})
     RETURNING id`
@@ -67,7 +69,10 @@ User.fetchByActivationToken = async activationToken => {
 
 // Activate user account.
 User.activateAccount = async user => {
-  await db.query(`UPDATE "user" SET status = 'active' WHERE id = $1`, [user.id])
+  await db.query(
+    `UPDATE "user" SET status = 'active', time_activated = NOW() WHERE id = $1`,
+    [user.id]
+  )
 }
 
 // Generate a user remember me token.
@@ -101,6 +106,7 @@ User.fetchDeserializedDataById = async userId => {
       u.last_name,
       u.email,
       u.hits_per_page,
+      u.language,
       ARRAY(
         SELECT jsonb_build_object(
           'roleName', r.role_name,
@@ -258,9 +264,9 @@ User.updatePortalRoles = async rolesPerUser => {
 
 User.fetchUser = async userId => {
   const text = `
-    SELECT id, username, first_name, last_name, email
+    SELECT id, username, first_name, last_name, email, status, language
     FROM "user"
-    WHERE id=$1`
+    WHERE id = $1`
   const value = [userId]
 
   const { rows } = await db.query(text, value)
@@ -271,17 +277,33 @@ User.fetchUser = async userId => {
 }
 
 User.updateUser = async (userId, payload) => {
-  const text = `
+  const previousStatusText = 'SELECT status FROM "user" WHERE id = $1'
+  const { rows } = await db.query(previousStatusText, [userId])
+  const previousStatus = rows[0].status
+
+  let statusValue
+  if (previousStatus === 'registered') {
+    statusValue = !payload.status ? 'registered' : 'active'
+  } else statusValue = !payload.status ? 'inactive' : 'active'
+
+  const updateText = `
     UPDATE "user"
     SET
       username = $2,
       first_name = $3,
-      last_name = $4
+      last_name = $4,
+      status = $5
     WHERE id = $1`
 
-  const values = [userId, payload.username, payload.firstName, payload.lastName]
+  const values = [
+    userId,
+    payload.username,
+    payload.firstName,
+    payload.lastName,
+    statusValue
+  ]
 
-  await db.query(text, values)
+  await db.query(updateText, values)
 }
 
 User.fetchUserRoles = async userId => {
@@ -467,4 +489,12 @@ User.updateHitsPerPage = async (username, hitsPerPageAmount) => {
   )
 }
 
+// Update user's language.
+User.updateLanguage = async (userId, languageCode) => {
+  await db.query('UPDATE "user" SET language = $1 WHERE id = $2', [
+    languageCode,
+    userId
+  ])
+}
+
 module.exports = User