First full release
This commit is contained in:
Luka Romih
+675
-40
@@ -1,8 +1,15 @@
|
||||
const { mkdir, open, unlink } = require('fs/promises')
|
||||
const Cursor = require('pg-cursor')
|
||||
const db = require('./db')
|
||||
const { intoDbArray, getInstanceSetting } = require('./helpers')
|
||||
const { deserialize, bulkIndex } = require('./helpers/dictionary')
|
||||
const {
|
||||
deserialize,
|
||||
bulkIndex,
|
||||
getExportFilesPath
|
||||
} = require('./helpers/dictionary')
|
||||
const { readFileIntoDb } = require('./helpers/dictionary/import-file')
|
||||
const { transformAndAppend } = require('./helpers/dictionary/export-file')
|
||||
const { origin } = require('../config/keys')
|
||||
// const debug = require('debug')('termPortal:models/dictionary')
|
||||
|
||||
class Dictionary {
|
||||
@@ -25,6 +32,7 @@ class Dictionary {
|
||||
}
|
||||
|
||||
// Fetch all dictionaries from DB.
|
||||
// TODO i18n name_sl
|
||||
static async fetchAll() {
|
||||
// TODO Implement SQL stored procedures or functions.
|
||||
const { rows: fetchedDictionaries } = await db.query(`
|
||||
@@ -65,6 +73,7 @@ class Dictionary {
|
||||
}
|
||||
*/
|
||||
|
||||
// TODO i18n name_sl
|
||||
// Fetch all dictionaries from DB for which the user has at least one dictionary role.
|
||||
static async fetchAllByUser(userId) {
|
||||
// TODO Implement SQL stored procedures or functions.
|
||||
@@ -75,7 +84,12 @@ class Dictionary {
|
||||
time_modified,
|
||||
status,
|
||||
count_entries,
|
||||
count_comments
|
||||
count_comments,
|
||||
(
|
||||
SELECT administration
|
||||
FROM user_role
|
||||
WHERE user_id = $1 and dictionary_id = id
|
||||
) as is_admin
|
||||
FROM dictionary
|
||||
WHERE id IN (
|
||||
SELECT dictionary_id
|
||||
@@ -86,8 +100,8 @@ class Dictionary {
|
||||
`
|
||||
const values = [userId]
|
||||
const { rows: fetchedDictionaries } = await db.query(text, values)
|
||||
const deserializedDictionaries = fetchedDictionaries.map(
|
||||
dictionary => new this(dictionary)
|
||||
const deserializedDictionaries = fetchedDictionaries.map(dictionary =>
|
||||
deserialize.dictionary(dictionary)
|
||||
)
|
||||
return deserializedDictionaries
|
||||
}
|
||||
@@ -138,13 +152,13 @@ class Dictionary {
|
||||
}
|
||||
|
||||
// Fetch all languages from DB.
|
||||
static async fetchAllLanguages(lang) {
|
||||
static async fetchAllLanguages(lang, excludeSlovene) {
|
||||
// TODO Implement SQL stored procedures or functions.
|
||||
const { rows: fetchedLanguages } = await db.query(`
|
||||
SELECT
|
||||
id,
|
||||
${lang}
|
||||
FROM language
|
||||
FROM language${excludeSlovene ? "\nWHERE code <> 'sl'" : ''}
|
||||
ORDER BY ${lang}`)
|
||||
const deserializedLanguages = fetchedLanguages.map(language =>
|
||||
deserialize.language(language)
|
||||
@@ -413,15 +427,15 @@ class Dictionary {
|
||||
description,
|
||||
l.name_sl languageSl,
|
||||
ds.name_sl domainSecondarySl,
|
||||
lp.name portalname,
|
||||
lp.name portalnamesl,
|
||||
lp.code portalcode
|
||||
FROM
|
||||
dictionary d
|
||||
INNER JOIN dictionary_language dl ON dl.dictionary_id = d.id
|
||||
INNER JOIN language l ON dl.language_id = l.id
|
||||
LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
|
||||
LEFT JOIN language l ON dl.language_id = l.id
|
||||
LEFT JOIN dictionary_domain_secondary dds on dds.dictionary_id = d.id
|
||||
LEFT JOIN domain_secondary ds ON dds.domain_secondary_id = ds.id
|
||||
INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
LEFT JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
LEFT JOIN linked_dictionary ld ON ld.target_dictionary_id = d.id
|
||||
LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
|
||||
WHERE d.id = $1`
|
||||
@@ -447,7 +461,7 @@ class Dictionary {
|
||||
d.author,
|
||||
dp.name_sl domain_primary,
|
||||
description,
|
||||
lp.name portalname,
|
||||
lp.name portalnamesl,
|
||||
lp.code portalcode
|
||||
FROM
|
||||
dictionary d
|
||||
@@ -509,7 +523,7 @@ class Dictionary {
|
||||
d.author,
|
||||
dp.name_sl domain_primary,
|
||||
description,
|
||||
lp.name portalname,
|
||||
lp.name portalnamesl,
|
||||
lp.code portalcode
|
||||
FROM
|
||||
dictionary d
|
||||
@@ -577,12 +591,13 @@ class Dictionary {
|
||||
distinct (d.id),
|
||||
d.${lang} dictionarysl,
|
||||
d.count_entries,
|
||||
d.count_comments,
|
||||
to_char(d.time_modified,'YYYY-MM-DD') time_modified,
|
||||
d.issn,
|
||||
d.author,
|
||||
dp.name_sl domain_primary,
|
||||
description,
|
||||
lp.name portalname,
|
||||
lp.name portalnamesl,
|
||||
lp.code portalcode
|
||||
FROM
|
||||
dictionary d
|
||||
@@ -590,7 +605,7 @@ class Dictionary {
|
||||
INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
LEFT JOIN linked_dictionary ld ON ld.target_dictionary_id = d.id
|
||||
LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
|
||||
WHERE d.name_sl LIKE '%' || $1 || '%' ${queryAppend}
|
||||
WHERE d.status = 'published' AND LOWER(d.name_sl) LIKE '%' || LOWER($1) || '%' ${queryAppend}
|
||||
ORDER BY ${orderBy}
|
||||
LIMIT $2
|
||||
OFFSET $3`
|
||||
@@ -643,7 +658,7 @@ class Dictionary {
|
||||
INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
LEFT JOIN linked_dictionary ld ON ld.target_dictionary_id = d.id
|
||||
LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
|
||||
WHERE d.name_sl LIKE '%' || $1 || '%' ${queryAppend}`
|
||||
WHERE d.status = 'published' AND d.name_sl LIKE '%' || $1 || '%' ${queryAppend}`
|
||||
|
||||
const { rows } = await db.query(text, [searchQuery])
|
||||
|
||||
@@ -656,8 +671,8 @@ class Dictionary {
|
||||
count(d.id)
|
||||
FROM
|
||||
dictionary d
|
||||
INNER JOIN dictionary_language dl ON dl.dictionary_id = d.id
|
||||
INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
|
||||
LEFT JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
LEFT JOIN linked_dictionary ld ON ld.target_dictionary_id = d.id
|
||||
LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id`
|
||||
|
||||
@@ -666,6 +681,23 @@ class Dictionary {
|
||||
return rows[0]
|
||||
}
|
||||
|
||||
static async fetchAllDictionariesPublishedCount() {
|
||||
const text = `
|
||||
SELECT
|
||||
count(d.id)
|
||||
FROM
|
||||
dictionary d
|
||||
LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
|
||||
LEFT JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
LEFT JOIN linked_dictionary ld ON ld.target_dictionary_id = d.id
|
||||
LEFT JOIN linked_portal lp ON ld.linked_portal_id = lp.id
|
||||
WHERE d.status = 'published'`
|
||||
|
||||
const { rows } = await db.query(text)
|
||||
|
||||
return rows[0]
|
||||
}
|
||||
|
||||
// Fetch single dictionary data for editing structure from DB.
|
||||
static async fetchDictionaryWithEditStructure(dictionaryId) {
|
||||
const text = `
|
||||
@@ -734,10 +766,11 @@ class Dictionary {
|
||||
SELECT
|
||||
d.id,
|
||||
${lang} dictionarysl,
|
||||
dp.name_sl domain_primary
|
||||
dp.name_sl domain_primary,
|
||||
d.count_comments
|
||||
FROM dictionary d
|
||||
INNER JOIN domain_primary dp ON d.domain_primary_id = dp.id
|
||||
where d.time_published is not NULL
|
||||
where d.time_published is not NULL and d.status = 'published'
|
||||
ORDER BY d.time_published desc
|
||||
limit 3`
|
||||
|
||||
@@ -984,15 +1017,62 @@ class Dictionary {
|
||||
}
|
||||
}
|
||||
|
||||
// Import entries from extraction into DB.
|
||||
static async importFromExtraction(dictionaryId, userId, termCandidates) {
|
||||
await db.transaction(async dbClient => {
|
||||
// TODO Depending on performance, consider batching requests.
|
||||
for (const {
|
||||
kanonicnaoblika: term,
|
||||
definicija: other
|
||||
} of termCandidates) {
|
||||
const values = [
|
||||
dictionaryId,
|
||||
false,
|
||||
'suggestion',
|
||||
term || null,
|
||||
userId,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
[],
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
[],
|
||||
other || null,
|
||||
[],
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
|
||||
const text = `SELECT entry_new (${db.genParamStr(values)})`
|
||||
|
||||
await dbClient.query(text, values)
|
||||
}
|
||||
|
||||
await this.updateMetadataAfterModifyingEntries(dictionaryId, dbClient)
|
||||
})
|
||||
}
|
||||
|
||||
// Index entries of specific dictionary from DB into search engine.
|
||||
static async indexIntoSearchEngine(dictionaryId) {
|
||||
// TODO Luka: I expect "source" needing a rework once linked portals and dictionaries start working.
|
||||
const source = {
|
||||
code: await getInstanceSetting('portal_code'),
|
||||
name: await getInstanceSetting('portal_name')
|
||||
}
|
||||
const dbClient = await db.getClient()
|
||||
try {
|
||||
const {
|
||||
rows: [{ code: linkedPortalCode, name: linkedPortalName } = {}]
|
||||
} = await dbClient.query(
|
||||
'SELECT p.code, p.name FROM linked_dictionary d LEFT JOIN linked_portal p ON p.id = d.linked_portal_id WHERE d.id = $1',
|
||||
[dictionaryId]
|
||||
)
|
||||
|
||||
// TODO i18n Luka: index portal name for both languages?
|
||||
const source = {
|
||||
code: linkedPortalCode ?? (await getInstanceSetting('portal_code')),
|
||||
name: linkedPortalName ?? (await getInstanceSetting('portal_name_sl'))
|
||||
}
|
||||
|
||||
const queryValues = [dictionaryId]
|
||||
|
||||
const dictionaryQueryText = `
|
||||
@@ -1060,7 +1140,7 @@ class Dictionary {
|
||||
)
|
||||
)
|
||||
FROM entry_foreign ef
|
||||
LEFT JOIN LANGUAGE l ON l.id = ef.language_id
|
||||
LEFT JOIN language l ON l.id = ef.language_id
|
||||
WHERE entry_id = e.id
|
||||
) foreign_entries
|
||||
FROM entry e
|
||||
@@ -1154,6 +1234,41 @@ class Dictionary {
|
||||
return result
|
||||
}
|
||||
|
||||
static async fetchFilteredPaginationDomainLabels(
|
||||
dictionaryId,
|
||||
query,
|
||||
resultsPerPage,
|
||||
page
|
||||
) {
|
||||
const {
|
||||
rows: [{ result }]
|
||||
} = await db.query(
|
||||
`
|
||||
SELECT jsonb_build_object(
|
||||
'pages_total', (
|
||||
SELECT CEIL(COUNT(*) / $3::float)
|
||||
FROM domain_label
|
||||
WHERE dictionary_id = $1 and name LIKE '%' || $2 || '%'
|
||||
),
|
||||
'results', ARRAY(
|
||||
SELECT jsonb_build_object(
|
||||
'id', id,
|
||||
'name', name,
|
||||
'isVisible', is_visible
|
||||
)
|
||||
FROM domain_label
|
||||
WHERE dictionary_id = $1 and LOWER(name) LIKE '%' || LOWER($2) || '%'
|
||||
ORDER BY name
|
||||
LIMIT $3
|
||||
OFFSET $4
|
||||
)
|
||||
) result`,
|
||||
[dictionaryId, query, resultsPerPage, resultsPerPage * (page - 1)]
|
||||
)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Fetch all secondary domains from DB.
|
||||
static async fetchAllSecondaryDomains(resultsPerPage, page) {
|
||||
const {
|
||||
@@ -1184,6 +1299,38 @@ class Dictionary {
|
||||
return result
|
||||
}
|
||||
|
||||
// Fetch filtered secondary domains from DB.
|
||||
static async fetchFilteredSecondaryDomains(query, resultsPerPage, page) {
|
||||
const {
|
||||
rows: [{ result }]
|
||||
} = await db.query(
|
||||
`
|
||||
SELECT jsonb_build_object(
|
||||
'pages_total', (
|
||||
SELECT CEIL(COUNT(*) / $2::float)
|
||||
FROM domain_secondary
|
||||
WHERE name_sl LIKE '%' || $1 || '%'
|
||||
),
|
||||
'results', ARRAY(
|
||||
SELECT jsonb_build_object(
|
||||
'id', id,
|
||||
'isApproved', approved,
|
||||
'nameSl', name_sl,
|
||||
'nameEn', name_en
|
||||
)
|
||||
FROM domain_secondary
|
||||
WHERE LOWER(name_sl) LIKE '%' || LOWER($1) || '%'
|
||||
ORDER BY name_sl
|
||||
LIMIT $2
|
||||
OFFSET $3
|
||||
)
|
||||
) result`,
|
||||
[query, resultsPerPage, resultsPerPage * (page - 1)]
|
||||
)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static async updateDomainLabel(dictionaryId, data) {
|
||||
const dbClient = await db.getClient()
|
||||
try {
|
||||
@@ -1343,24 +1490,54 @@ class Dictionary {
|
||||
}
|
||||
}
|
||||
|
||||
static async fetchAllImports(dictionaryId) {
|
||||
const text = `
|
||||
SELECT
|
||||
time_started,
|
||||
status,
|
||||
delete_existing_entries,
|
||||
file_format,
|
||||
count_valid_entries
|
||||
FROM import_file_job
|
||||
WHERE dictionary_id = $1`
|
||||
// static async fetchAllImports(dictionaryId) {
|
||||
// const text = `
|
||||
// SELECT
|
||||
// time_started,
|
||||
// status,
|
||||
// delete_existing_entries,
|
||||
// file_format,
|
||||
// count_valid_entries
|
||||
// FROM import_file_job
|
||||
// WHERE dictionary_id = $1`
|
||||
|
||||
const value = [dictionaryId]
|
||||
const { rows: fetchedImports } = await db.query(text, value)
|
||||
// const value = [dictionaryId]
|
||||
// const { rows: fetchedImports } = await db.query(text, value)
|
||||
|
||||
const deserializedImports = fetchedImports.map(oneImport =>
|
||||
deserialize.imports(oneImport)
|
||||
// const deserializedImports = fetchedImports.map(oneImport =>
|
||||
// deserialize.imports(oneImport)
|
||||
// )
|
||||
// return deserializedImports
|
||||
// }
|
||||
|
||||
static async fetchAllImports(dictionaryId, resultsPerPage, page) {
|
||||
const {
|
||||
rows: [{ result }]
|
||||
} = await db.query(
|
||||
`
|
||||
SELECT jsonb_build_object(
|
||||
'pages_total', (
|
||||
SELECT CEIL(COUNT(*) / $2::float)
|
||||
FROM import_file_job
|
||||
WHERE dictionary_id = $1
|
||||
),
|
||||
'results', ARRAY(
|
||||
SELECT jsonb_build_object(
|
||||
'time_started', time_started,
|
||||
'status', status,
|
||||
'delete_existing_entries', delete_existing_entries,
|
||||
'file_format', file_format,
|
||||
'count_valid_entries', count_valid_entries
|
||||
)
|
||||
FROM import_file_job
|
||||
WHERE dictionary_id = $1
|
||||
LIMIT $2
|
||||
OFFSET $3
|
||||
)
|
||||
) result`,
|
||||
[dictionaryId, resultsPerPage, resultsPerPage * (page - 1)]
|
||||
)
|
||||
return deserializedImports
|
||||
return result
|
||||
}
|
||||
|
||||
static async delete(dictionaryId) {
|
||||
@@ -1368,6 +1545,464 @@ class Dictionary {
|
||||
const value = [dictionaryId]
|
||||
await db.query(text, value)
|
||||
}
|
||||
|
||||
// static async fetchExports(dictionaryId) {
|
||||
// const text = `
|
||||
// SELECT
|
||||
// id,
|
||||
// status,
|
||||
// to_char(time_created, 'FMDD. FMMM. YYYY') date_created,
|
||||
// entry_count,
|
||||
// is_valid_filter,
|
||||
// is_published_filter,
|
||||
// is_terminology_reviewed_filter,
|
||||
// is_language_reviewed_filter,
|
||||
// status_filter,
|
||||
// export_file_format
|
||||
// FROM dictionary_export
|
||||
// WHERE dictionary_id = $1
|
||||
// ORDER BY id DESC`
|
||||
// const value = [dictionaryId]
|
||||
|
||||
// const { rows: fetchedExports } = await db.query(text, value)
|
||||
|
||||
// const deserializedExports = fetchedExports.map(eachExport =>
|
||||
// deserialize.exports(eachExport)
|
||||
// )
|
||||
// return deserializedExports
|
||||
// }
|
||||
|
||||
static async fetchExports(dictionaryId, resultsPerPage, page) {
|
||||
const {
|
||||
rows: [{ result }]
|
||||
} = await db.query(
|
||||
`
|
||||
SELECT jsonb_build_object(
|
||||
'pages_total', (
|
||||
SELECT CEIL(COUNT(*) / $2::float)
|
||||
FROM dictionary_export
|
||||
WHERE dictionary_id = $1
|
||||
),
|
||||
'results', ARRAY(
|
||||
SELECT jsonb_build_object(
|
||||
'id', id,
|
||||
'status', status,
|
||||
'time_created', time_created,
|
||||
'entry_count', entry_count,
|
||||
'is_valid_filter', is_valid_filter,
|
||||
'is_published_filter', is_published_filter,
|
||||
'is_terminology_reviewed_filter', is_terminology_reviewed_filter,
|
||||
'is_language_reviewed_filter', is_language_reviewed_filter,
|
||||
'status_filter', status_filter,
|
||||
'export_file_format', export_file_format
|
||||
)
|
||||
FROM dictionary_export
|
||||
WHERE dictionary_id = $1
|
||||
LIMIT $2
|
||||
OFFSET $3
|
||||
)
|
||||
) result`,
|
||||
[dictionaryId, resultsPerPage, resultsPerPage * (page - 1)]
|
||||
)
|
||||
return result
|
||||
}
|
||||
|
||||
static async beginExport(dictionaryId, exportParams) {
|
||||
const text = `
|
||||
INSERT INTO dictionary_export (
|
||||
dictionary_id,
|
||||
is_valid_filter,
|
||||
is_published_filter,
|
||||
is_terminology_reviewed_filter,
|
||||
is_language_reviewed_filter,
|
||||
status_filter,
|
||||
export_file_format
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id
|
||||
`
|
||||
|
||||
const values = [
|
||||
dictionaryId,
|
||||
exportParams.isValidFilter,
|
||||
exportParams.isPublishedFilter,
|
||||
exportParams.isTerminologyReviewedFilter,
|
||||
exportParams.isLanguageReviewedFilter,
|
||||
exportParams.statusFilter,
|
||||
exportParams.exportFileFormat
|
||||
]
|
||||
|
||||
const {
|
||||
rows: [{ id }]
|
||||
} = await db.query(text, values)
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
static async processExport(exportId) {
|
||||
const dbClient = await db.getClient()
|
||||
let exportFilePath
|
||||
let exportFile
|
||||
let cursor
|
||||
|
||||
try {
|
||||
const exportQueryText = `
|
||||
UPDATE dictionary_export
|
||||
SET status = 'in progress', time_started = NOW()
|
||||
WHERE id = $1
|
||||
RETURNING
|
||||
dictionary_id,
|
||||
is_valid_filter,
|
||||
is_published_filter,
|
||||
is_terminology_reviewed_filter,
|
||||
is_language_reviewed_filter,
|
||||
status_filter,
|
||||
export_file_format
|
||||
`
|
||||
|
||||
const exportQueryParams = [exportId]
|
||||
|
||||
const {
|
||||
rows: [
|
||||
{
|
||||
dictionary_id: dictionaryId,
|
||||
is_valid_filter: isValidFilter,
|
||||
is_published_filter: isPublishedFilter,
|
||||
is_terminology_reviewed_filter: isTerminologyReviewedFilter,
|
||||
is_language_reviewed_filter: isLanguageReviewedFilter,
|
||||
status_filter: statusFilter,
|
||||
export_file_format: exportFileFormat
|
||||
}
|
||||
]
|
||||
} = await dbClient.query(exportQueryText, exportQueryParams)
|
||||
|
||||
const dictionaryQueryText = `
|
||||
SELECT
|
||||
entries_have_domain_labels,
|
||||
entries_have_label,
|
||||
entries_have_definition,
|
||||
entries_have_synonyms,
|
||||
entries_have_links,
|
||||
entries_have_other,
|
||||
entries_have_foreign_languages,
|
||||
entries_have_foreign_definitions,
|
||||
entries_have_foreign_synonyms,
|
||||
entries_have_images,
|
||||
entries_have_audio,
|
||||
entries_have_videos
|
||||
FROM dictionary
|
||||
WHERE id = $1
|
||||
`
|
||||
|
||||
const dictionaryQueryParams = [dictionaryId]
|
||||
|
||||
const {
|
||||
rows: [dictionaryStructure]
|
||||
} = await dbClient.query(dictionaryQueryText, dictionaryQueryParams)
|
||||
|
||||
const exportFields = {
|
||||
domainLabels: dictionaryStructure.entries_have_domain_labels,
|
||||
label: dictionaryStructure.entries_have_label,
|
||||
definition: dictionaryStructure.entries_have_definition,
|
||||
synonyms: dictionaryStructure.entries_have_synonyms,
|
||||
links: dictionaryStructure.entries_have_links,
|
||||
other: dictionaryStructure.entries_have_other,
|
||||
foreignTerms: dictionaryStructure.entries_have_foreign_languages,
|
||||
foreignDefinitions:
|
||||
dictionaryStructure.entries_have_foreign_definitions,
|
||||
foreignSynonyms: dictionaryStructure.entries_have_foreign_synonyms,
|
||||
images: dictionaryStructure.entries_have_images,
|
||||
audio: dictionaryStructure.entries_have_audio,
|
||||
videos: dictionaryStructure.entries_have_videos
|
||||
}
|
||||
|
||||
let entryQueryText = `
|
||||
SELECT
|
||||
term${
|
||||
exportFields.label
|
||||
? `,
|
||||
label`
|
||||
: ''
|
||||
}${
|
||||
exportFields.definition
|
||||
? `,
|
||||
definition`
|
||||
: ''
|
||||
}${
|
||||
exportFields.synonyms
|
||||
? `,
|
||||
synonym synonyms`
|
||||
: ''
|
||||
}${
|
||||
exportFields.other
|
||||
? `,
|
||||
other`
|
||||
: ''
|
||||
}${
|
||||
exportFields.images
|
||||
? `,
|
||||
image`
|
||||
: ''
|
||||
}${
|
||||
exportFields.audio
|
||||
? `,
|
||||
audio`
|
||||
: ''
|
||||
}${
|
||||
exportFields.videos
|
||||
? `,
|
||||
video`
|
||||
: ''
|
||||
}${
|
||||
exportFields.domainLabels
|
||||
? `,
|
||||
ARRAY(
|
||||
SELECT name
|
||||
FROM entry_domain_label edl
|
||||
LEFT JOIN domain_label dl ON dl.id = edl.domain_label_id
|
||||
WHERE entry_id = e.id
|
||||
) domain_labels`
|
||||
: ''
|
||||
}${
|
||||
exportFields.links
|
||||
? `,
|
||||
ARRAY(
|
||||
SELECT jsonb_build_object(
|
||||
'link', link,
|
||||
'type', type)
|
||||
FROM entry_link
|
||||
WHERE entry_id = e.id
|
||||
) links`
|
||||
: ''
|
||||
}${
|
||||
exportFields.foreignTerms
|
||||
? `,
|
||||
ARRAY(
|
||||
SELECT jsonb_build_object(
|
||||
'lang_code', l.code,
|
||||
'terms', ef.term${
|
||||
exportFields.foreignDefinitions
|
||||
? `,
|
||||
'definition', ef.definition`
|
||||
: ''
|
||||
}${
|
||||
exportFields.foreignSynonyms
|
||||
? `,
|
||||
'synonyms', ef.synonym`
|
||||
: ''
|
||||
}
|
||||
)
|
||||
FROM dictionary_language dl
|
||||
LEFT JOIN language l ON l.id = dl.language_id
|
||||
LEFT JOIN entry_foreign ef ON ef.language_id = l.id
|
||||
WHERE dl.dictionary_id = $1 AND ef.entry_id = e.id
|
||||
ORDER BY dl.selection_order
|
||||
) foreign_entries`
|
||||
: ''
|
||||
}
|
||||
FROM entry e
|
||||
WHERE
|
||||
e.dictionary_id = $1`
|
||||
|
||||
const entryQueryParams = [dictionaryId]
|
||||
|
||||
if (isValidFilter !== null) {
|
||||
entryQueryParams.push(isValidFilter)
|
||||
entryQueryText += ` AND is_valid = $${entryQueryParams.length}`
|
||||
}
|
||||
if (isPublishedFilter !== null) {
|
||||
entryQueryParams.push(isPublishedFilter)
|
||||
entryQueryText += ` AND is_published = $${entryQueryParams.length}`
|
||||
}
|
||||
if (isTerminologyReviewedFilter !== null) {
|
||||
entryQueryParams.push(isTerminologyReviewedFilter)
|
||||
entryQueryText += ` AND is_terminology_reviewed = $${entryQueryParams.length}`
|
||||
}
|
||||
if (isLanguageReviewedFilter !== null) {
|
||||
entryQueryParams.push(isLanguageReviewedFilter)
|
||||
entryQueryText += ` AND is_language_reviewed = $${entryQueryParams.length}`
|
||||
}
|
||||
if (statusFilter !== null) {
|
||||
entryQueryParams.push(statusFilter)
|
||||
entryQueryText += ` AND status = $${entryQueryParams.length}`
|
||||
}
|
||||
entryQueryText += '\nORDER BY term'
|
||||
|
||||
const exportFilesPath = getExportFilesPath(dictionaryId)
|
||||
await mkdir(exportFilesPath, { recursive: true })
|
||||
exportFilePath = `${exportFilesPath}/${exportId}`
|
||||
exportFile = await open(exportFilePath, 'ax')
|
||||
|
||||
const isXmlFileFormat = exportFileFormat === 'xml'
|
||||
const isDsvFileFormat = ['csv', 'tsv'].includes(exportFileFormat)
|
||||
const isTbxFileFormat = exportFileFormat === 'tbx'
|
||||
let dsvConfig
|
||||
if (isXmlFileFormat) {
|
||||
const openingMarkup =
|
||||
'<?xml version="1.0" encoding="utf-8"?>\n<dictionary>\n'
|
||||
await exportFile.write(openingMarkup)
|
||||
} else if (isDsvFileFormat) {
|
||||
if (exportFileFormat === 'csv') dsvConfig = { delimiter: ';' }
|
||||
else if (exportFileFormat === 'tsv') dsvConfig = { delimiter: '\t' }
|
||||
|
||||
if (exportFields.foreignTerms) {
|
||||
const { rows: languages } = await dbClient.query(
|
||||
`
|
||||
SELECT l.code
|
||||
FROM dictionary d
|
||||
LEFT JOIN dictionary_language dl ON dl.dictionary_id = d.id
|
||||
LEFT JOIN language l ON l.id = dl.language_id
|
||||
WHERE d.id = $1
|
||||
ORDER BY dl.selection_order
|
||||
`,
|
||||
[dictionaryId]
|
||||
)
|
||||
|
||||
dsvConfig.languageCodes = languages.map(language => language.code)
|
||||
}
|
||||
|
||||
const fieldNamesArr = ['term']
|
||||
if (exportFields.domainLabels) fieldNamesArr.push('domainLabels')
|
||||
if (exportFields.label) fieldNamesArr.push('label')
|
||||
if (exportFields.definition) fieldNamesArr.push('def')
|
||||
if (exportFields.synonyms) fieldNamesArr.push('syns')
|
||||
if (exportFields.links) fieldNamesArr.push('links')
|
||||
if (exportFields.other) fieldNamesArr.push('other')
|
||||
dsvConfig.languageCodes?.forEach(languageCode => {
|
||||
fieldNamesArr.push(`[${languageCode}]fTerms`)
|
||||
if (exportFields.foreignDefinitions) {
|
||||
fieldNamesArr.push(`[${languageCode}]fDef`)
|
||||
}
|
||||
if (exportFields.foreignSynonyms) {
|
||||
fieldNamesArr.push(`[${languageCode}]fSyns`)
|
||||
}
|
||||
})
|
||||
if (exportFields.images) fieldNamesArr.push('images')
|
||||
if (exportFields.audio) fieldNamesArr.push('audios')
|
||||
if (exportFields.videos) fieldNamesArr.push('videos')
|
||||
const headerLine = `${fieldNamesArr.join(dsvConfig.delimiter)}\n`
|
||||
await exportFile.write(headerLine)
|
||||
} else if (isTbxFileFormat) {
|
||||
const {
|
||||
rows: [tbxMetadata]
|
||||
} = await dbClient.query(
|
||||
`
|
||||
SELECT
|
||||
name_sl,
|
||||
name_en,
|
||||
author,
|
||||
to_char(time_modified, 'YYYY-MM-DD') modified_date_string,
|
||||
status,
|
||||
(
|
||||
SELECT to_char(time_created, 'YYYY-MM-DD')
|
||||
FROM dictionary_export de
|
||||
WHERE de.id = $1
|
||||
) export_date_string
|
||||
FROM dictionary
|
||||
WHERE id = $2
|
||||
`,
|
||||
[exportId, dictionaryId]
|
||||
)
|
||||
const portalName = await getInstanceSetting('portal_name_sl')
|
||||
const authorsString = tbxMetadata.author?.join(', ')
|
||||
const urlPublished =
|
||||
tbxMetadata.status === 'published'
|
||||
? new URL(`/slovarji/${dictionaryId}/o-slovarju`, origin).href
|
||||
: null
|
||||
|
||||
let openingMarkup = '<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
openingMarkup += '<!DOCTYPE martif SYSTEM "TBXcoreStructV02.dtd">\n'
|
||||
openingMarkup += '<martif type="TBX" xml:lang="sl">\n<martifHeader>\n'
|
||||
openingMarkup += '<fileDesc>\n<titleStmt>\n'
|
||||
openingMarkup += `<title>${tbxMetadata.name_sl}</title>\n`
|
||||
openingMarkup += `<note xml:lang="en">${tbxMetadata.name_en}</note>\n`
|
||||
openingMarkup += '</titleStmt>\n<publicationStmt>\n'
|
||||
openingMarkup += `<p>Datum objave: ${tbxMetadata.export_date_string}</p>\n`
|
||||
openingMarkup +=
|
||||
'<p>Avtorske pravice: Delo je dostopno pod pogoji licence CC BY 4.0.</p>\n'
|
||||
openingMarkup += '</publicationStmt>\n<sourceDesc>\n'
|
||||
openingMarkup += `<p>Vir: ${portalName}</p>\n`
|
||||
if (authorsString) openingMarkup += `<p>Avtorji: ${authorsString}</p>\n`
|
||||
openingMarkup += `<p>Datum objave: ${tbxMetadata.modified_date_string}</p>\n`
|
||||
if (urlPublished) {
|
||||
openingMarkup += `<p>Mesto objave: ${urlPublished}</p>\n`
|
||||
}
|
||||
openingMarkup += '</sourceDesc>\n</fileDesc>\n'
|
||||
openingMarkup += '</martifHeader>\n<text>\n<body>\n'
|
||||
|
||||
await exportFile.write(openingMarkup)
|
||||
}
|
||||
|
||||
cursor = dbClient.query(new Cursor(entryQueryText, entryQueryParams))
|
||||
|
||||
let entries = []
|
||||
let batchEntriesCount
|
||||
let entriesWritten = 0
|
||||
do {
|
||||
// Keep getting and writing entries to file in batches of 100.
|
||||
entries = await cursor.read(100)
|
||||
batchEntriesCount = entries.length
|
||||
|
||||
if (batchEntriesCount) {
|
||||
await transformAndAppend(
|
||||
entries,
|
||||
exportFields,
|
||||
exportFile,
|
||||
exportFileFormat,
|
||||
dsvConfig
|
||||
)
|
||||
entriesWritten += batchEntriesCount
|
||||
}
|
||||
} while (batchEntriesCount === 100)
|
||||
|
||||
if (isXmlFileFormat) {
|
||||
const closingMarkup = '</dictionary>\n'
|
||||
await exportFile.write(closingMarkup)
|
||||
} else if (isTbxFileFormat) {
|
||||
const closingMarkup = '</body>\n</text>\n</martif>\n'
|
||||
await exportFile.write(closingMarkup)
|
||||
}
|
||||
|
||||
await dbClient.query(
|
||||
"UPDATE dictionary_export SET status = 'finished', time_finished = NOW(), entry_count = $1 WHERE id = $2",
|
||||
[entriesWritten, exportId]
|
||||
)
|
||||
} catch (error) {
|
||||
await cursor?.close()
|
||||
await dbClient.query(
|
||||
"UPDATE dictionary_export SET status = 'failed', time_finished = NOW() WHERE id = $1",
|
||||
[exportId]
|
||||
)
|
||||
await unlink(exportFilePath)
|
||||
throw error
|
||||
} finally {
|
||||
dbClient.release()
|
||||
await exportFile?.close()
|
||||
}
|
||||
}
|
||||
|
||||
static async fetchExportDownloadMetadata(exportId) {
|
||||
const {
|
||||
rows: [fetchedMetadata]
|
||||
} = await db.query(
|
||||
`
|
||||
SELECT
|
||||
e.status,
|
||||
e.dictionary_id,
|
||||
d.name_sl_short name_string,
|
||||
to_char(e.time_created, 'YYYYMMDDHH24MISS') time_string,
|
||||
e.export_file_format
|
||||
FROM dictionary_export e
|
||||
LEFT JOIN dictionary d ON d.id = e.dictionary_id
|
||||
WHERE e.id = $1`,
|
||||
[exportId]
|
||||
)
|
||||
|
||||
const deserializedMetadata =
|
||||
deserialize.exportDownloadMetadata(fetchedMetadata)
|
||||
|
||||
return deserializedMetadata
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Dictionary
|
||||
|
||||
Reference in New Issue
Block a user