Initial commit

This commit is contained in:
Luka Romih
2022-12-07 04:43:36 +01:00
commit 257f3c354f
496 changed files with 55373 additions and 0 deletions
+57
View File
@@ -0,0 +1,57 @@
const Comment = require('../../../models/comment')
const Entry = require('../../../models/entry')
const { DEFAULT_HITS_PER_PAGE } = require('../../../config/settings')
exports.listComments = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const filters = { ctxType: req.query.ctx_type, ctxId: req.query.ctx_id }
if (filters.ctxId === 'null') {
filters.ctxId = null
}
const {
pages_total: numberOfAllPages,
comments,
comment_count: commentCount
} = await Comment.list(filters, req.user, resultsPerPage, page)
res.send({ page, numberOfAllPages, comments, commentCount })
}
exports.createComment = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const receivedComment = req.body
const { ctxType, ctxId } = receivedComment
await Comment.create(receivedComment, req.user?.id)
const filters = { ctxType, ctxId }
const { pages_total: pagesTotal, comments } = await Comment.list(
filters,
req.user,
resultsPerPage,
'last'
)
if (ctxType === 'entry_dict_int' || ctxType === 'entry_dict_ext') {
await Entry.indexIntoSearchEngine(ctxId)
}
res.send({ comments, pagesTotal })
}
exports.seedComments = async (req, res) => {
const { commentCount } = req.params
await Comment.seed(commentCount)
res.send(`${commentCount} new comments generated`)
}
exports.clearComments = async (req, res) => {
await Comment.clear()
res.send('All comments cleared')
}
exports.updateStatus = async (req, res) => {
const commentId = req.body.params.id
const commentStatus = req.body.params.status
await Comment.updateStatus(commentId, commentStatus)
res.send('Visibility changed')
}
+321
View File
@@ -0,0 +1,321 @@
const ConsultancyEntry = require('../../../models/consultancy-entry')
const User = require('../../../models/user')
const { promisify } = require('util')
const {
deleteConsultancyEntryFromIndex
} = require('../../../models/search-engine')
const email = require('../../../models/email')
const helper = require('../../../models/helpers')
const consultancy = {}
consultancy.listEntries = async (req, res) => {
const consultancyEntryList = await ConsultancyEntry.fetchAll()
const data = {}
data.consEntryList = consultancyEntryList
res.send(data)
}
consultancy.listNewEntries = async (req, res) => {
const consultancyNewEntryList = await ConsultancyEntry.fetchAllNew()
const data = {}
data.consultancyNewEntryList = consultancyNewEntryList
res.send(data)
}
consultancy.createQuestion = async (req, res) => {
const q = req.body
const consultancyEntry = {}
consultancyEntry.status = 'new'
consultancyEntry.authorId = req.user.id
consultancyEntry.institution = q.institution
const { description } = q
if (!description) {
return res.status(400).send('description is a required parameter!')
}
consultancyEntry.description = description
consultancyEntry.domainPrimaryIdInitial = q.domainPrimary
if (!consultancyEntry.domainPrimaryIdInitial || q.domainPrimary === '-1') {
consultancyEntry.domainPrimaryIdInitial = null // undefined
}
consultancyEntry.existingSolutions = q.existing_solutions
consultancyEntry.examplesOfUse = q.examples_of_use
Object.keys(consultancyEntry).forEach(key => {
consultancyEntry[key] = helper.removeHtmlTags(consultancyEntry[key]).trim()
})
const questionId = await ConsultancyEntry.createQuestion(consultancyEntry)
await ConsultancyEntry.indexIntoSearchEngine(questionId, true)
// TODO SEND EMAIL
// TODOOOOOOOOO
const emails = await ConsultancyEntry.fetchConsultancyAdminEmails()
const renderAsync = promisify(req.app.render.bind(req.app))
const emailHtml = await renderAsync('email/consultancy-creation-notify', {
propertyToPassGoesHere: 'test1234'
})
await email.send({
to: emails,
subject: 'Ustvarjeno novo vprašanje v svetovalnici',
html: emailHtml
})
/// /////////////////
res.status(201).send()
}
consultancy.updateDomain = async (req, res) => {
const { id, value } = req.body
if (!id) return res.status(400).send()
await User.updateConsultancyDomains(id, value)
res.status(204).send()
}
consultancy.insertConsultantAndHisDomainsByUsername = async (req, res) => {
const { username, domains } = req.body
if (!username) {
return res.status(400).send()
}
await User.insertNewConsultantWithDomainByUsername(username, domains)
res.status(204).send()
}
consultancy.removeConsultant = async (req, res) => {
const { id } = req.body
if (!id) return res.status(400).send()
await User.removeConsultant(id)
res.send()
}
consultancy.getSharedAuthors = async (req, res) => {
const { id } = req.query
if (!id) return res.status(400).send()
const authors = await ConsultancyEntry.getSharedAuthorsArray(id)
res.send(authors)
}
consultancy.insertNonModerator = async (req, res) => {
const userId = req.body.user_id
const entryId = req.body.entry_id
if (!userId || !entryId) return res.status(400).send()
await ConsultancyEntry.insertNonModerator(entryId, userId)
await ConsultancyEntry.indexIntoSearchEngine(entryId, true)
res.status(201).send()
}
consultancy.getSharedAuthorsBeforePublish = async (req, res) => {
const { id } = req.query
if (!id) return res.status(400).send()
const authors = await ConsultancyEntry.getSharedAuthorsArrayBeforePublish(id)
res.send(authors)
}
consultancy.updateSharedAuthors = async (req, res) => {
const { id, authors } = req.body
if (!id || !authors.length) return res.status(400).send()
await ConsultancyEntry.updateSharedAuthorsArray(id, authors)
await ConsultancyEntry.indexIntoSearchEngine(id, true)
res.send({})
}
/*
function assign - assigns from any state of the consultancy
question to work in-progress
inputs:
id -> id of the question
*/
consultancy.assign = async (req, res) => {
const questionId = req.body.question_id
let userId = req.body.user_id
if (!questionId) return res.status(400).send()
if (!userId) {
const user = await ConsultancyEntry.getModerator(questionId)
userId = user.id
if (!userId) return res.status(400).send({})
}
await ConsultancyEntry.assignWorkInProgress(questionId, userId)
await ConsultancyEntry.indexIntoSearchEngine(questionId, true)
// TODO SEND MAIL TO MODERATOR
/*
Skrbnik svetovalnice vam je v urejanje dodelil novo terminološko vprašanje.
*/
const emails = await ConsultancyEntry.fetchModeratorEmail(questionId)
const renderAsync = promisify(req.app.render.bind(req.app))
const emailHtml = await renderAsync('email/consultancy-assigned')
await email.send({
to: emails,
subject: 'Novo terminološko vprašanje',
html: emailHtml
})
res.send()
}
consultancy.reject = async (req, res) => {
const questionId = req.body.question_id
if (!questionId) return res.status(400).send()
await ConsultancyEntry.rejectEntry(questionId)
await ConsultancyEntry.indexIntoSearchEngine(questionId, true)
res.send()
}
consultancy.sendToReview = async (req, res) => {
const questionId = req.body.question_id
if (!questionId) return res.status(400).send()
await ConsultancyEntry.sendToReview(questionId)
await ConsultancyEntry.indexIntoSearchEngine(questionId, true)
/* SEND MAIL TO ALL CONSULTANCY ADMINS
Za potrditev objave ste prejeli novo terminološko vprašanje.
const allEmailsSet = new Set([...adminEmails, ...dictionariesAdminEmails])
const allEmails = []
for (const email of allEmailsSet) allEmails.push(email)
const type = 'delete'
const renderAsync = promisify(appRef.render.bind(appRef))
const emailHtml = await renderAsync('email/dictionary-status-change', {
type,
nameSl
})
await email.send({
to: allEmails,
subject: 'Obvestilo o številu gesel',
html: emailHtml
})
*/
const emails = await ConsultancyEntry.fetchConsultancyAdminEmails()
const renderAsync = promisify(req.app.render.bind(req.app))
const emailHtml = await renderAsync('email/consultancy-item-review')
await email.send({
to: emails,
subject: 'Potrditev objave',
html: emailHtml
})
res.send()
}
consultancy.publish = async (req, res) => {
const questionId = req.body.question_id
const answerAuthors = req.body.answer_authors
if (!questionId) return res.status(400).send()
const entry = await ConsultancyEntry.fetchById(questionId)
if (!entry.title) {
return res.status(400).send('Answer not completed')
}
await ConsultancyEntry.publish(questionId, answerAuthors)
await ConsultancyEntry.indexIntoSearchEngine(questionId, true)
res.send()
}
consultancy.updateQuestion = async (req, res) => {
const { id, questionTitle, domain: domainId, question, answer } = req.body
if (!id) return res.status(400).send({})
if (questionTitle === '' || question === '' || answer === '') {
return res.status(422).send('Polja naslov, vprašanje in mnenje so obvezna!')
}
const entry = await ConsultancyEntry.fetchById(id)
entry.domainPrimaryId = domainId > 0 ? domainId : null
entry.question = question
entry.answer = answer // helper.removeHtmlTags(answer).trim()
entry.title = questionTitle
// TODO Luka: Miha, update only fields that were updated.
await ConsultancyEntry.updateQuestion(entry)
await ConsultancyEntry.indexIntoSearchEngine(id, true)
res.send({})
}
consultancy.insertNonModerator = async (req, res) => {
const questionId = req.body.question_id
const userId = req.body.user_id
if (!questionId || !userId) return res.status(400).send()
await ConsultancyEntry.insertNonModerator(questionId, userId)
await ConsultancyEntry.indexIntoSearchEngine(questionId, true)
res.send({})
}
consultancy.deleteConsultantForEntry = async (req, res) => {
const questionId = req.body.question_id
const userId = req.body.user_id
if (!questionId || !userId) return res.status(400).send()
await ConsultancyEntry.removeConsultantForEntry(questionId, userId)
await ConsultancyEntry.indexIntoSearchEngine(questionId, true)
res.send({})
}
consultancy.deleteQuestion = async (req, res) => {
const { id } = req.body // question/entry id
if (!id) return res.status(400).send()
await ConsultancyEntry.removeEntry(id)
await deleteConsultancyEntryFromIndex(id, true)
res.send()
}
module.exports = consultancy
@@ -0,0 +1,15 @@
const DemoPaginacija = require('../../../models/demo-paginacija')
const { DEFAULT_HITS_PER_PAGE } = require('../../../config/settings')
exports.list = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const { pages_total: numberOfAllPages, results } = await DemoPaginacija.fetch(
resultsPerPage,
page
)
res.send({ page, numberOfAllPages, results })
}
+189
View File
@@ -0,0 +1,189 @@
const Dictionary = require('../../../models/dictionary')
const Entry = require('../../../models/entry')
const {
searchEntryIndex,
deleteEntryFromIndex,
deleteDictionaryEntriesFromIndex
} = require('../../../models/search-engine')
const genEditorAllQuery = require('../../../models/helpers/search/generate-query/editor/all')
const { prepareEditorEntries } = require('../../../models/helpers/search')
const { DEFAULT_HITS_PER_PAGE } = require('../../../config/settings')
const { minEntriesRequirementCheckAndAct } = require('../../helpers/dictionary')
const dictionary = {}
dictionary.getEntry = async (req, res) => {
const entryId = req.query.id
const [entry, domainLabels] = await Promise.all([
Entry.fetchFull(entryId),
Dictionary.fetchDomainLabelsFromEntryId(entryId)
])
const data = {
entry,
allDomainLabels: domainLabels
}
res.send(data)
}
dictionary.createEntry = async (req, res) => {
const { body } = req
const { dictionaryId } = body
const entryId = await Entry.create(req.user.id, dictionaryId, body)
await Promise.all([
Dictionary.updateMetadataAfterModifyingEntries(dictionaryId),
Entry.indexIntoSearchEngine(entryId, true)
])
res.send({ entryId })
}
dictionary.updateEntry = async (req, res) => {
const { entryId } = req.body
const dictionaryId = await Entry.update(req.user.id, req.body)
await Promise.all([
Dictionary.updateMetadataAfterModifyingEntry(entryId),
Entry.indexIntoSearchEngine(entryId, true),
minEntriesRequirementCheckAndAct.onUpdate(dictionaryId)
])
res.end()
}
dictionary.fetchEntries = async (req, res) => {
const dictionaryId = req.query.id
const hitsQuery = genEditorAllQuery(dictionaryId)
const hits = await searchEntryIndex(hitsQuery)
const terms = prepareEditorEntries(hits)
res.send(terms)
}
// This function deletes selected entry.
dictionary.deleteEntry = async (req, res) => {
const { entryId } = req.query
const dictionaryId = await Entry.delete(entryId)
await Promise.all([
Dictionary.updateMetadataAfterModifyingEntries(dictionaryId),
deleteEntryFromIndex(entryId, true),
minEntriesRequirementCheckAndAct.onDelete(dictionaryId, req.app)
])
res.end()
}
// This function deletes all entries in seleceted dictionary.
dictionary.deleteAllEntries = async (req, res) => {
// TODO This method executes all delete operations before sending the response,
// TODO after which the client tells the user it might take a few minutes.
const dictionaryId = +req.params.dictionaryId
await Entry.deleteAll(dictionaryId)
await Promise.all([
Dictionary.updateMetadataAfterModifyingEntries(dictionaryId),
deleteDictionaryEntriesFromIndex(dictionaryId),
minEntriesRequirementCheckAndAct.onDelete(dictionaryId, req.app)
])
res.end()
}
dictionary.publishAllEntries = async (req, res) => {
// // TODO This method executes all operations before sending the response,
// // TODO after which the client tells the user it might take a few minutes.
// // TODO Also, the message is generic and the same as with deleting a dictionary or all of its entries.
const dictionaryId = +req.params.dictionaryId
await Entry.publishAllQualified(dictionaryId)
// TODO Consider reducing the following two index operations into a single one.
await Promise.all([
Dictionary.updateMetadataAfterModifyingEntries(dictionaryId),
deleteDictionaryEntriesFromIndex(dictionaryId)
])
await Dictionary.indexIntoSearchEngine(dictionaryId)
res.end()
}
dictionary.delete = async (req, res) => {
// TODO This method executes all operations before sending the response,
// TODO after which the client tells the user it might take a few minutes.
// TODO It also keeps the user on a page which shouldn't exist anymore.
// TODO In fact, any value is apparently valid as dictionaryId URL parameter (no validation yet).
const dictionaryId = +req.params.dictionaryId
await Dictionary.delete(dictionaryId)
await deleteDictionaryEntriesFromIndex(dictionaryId)
res.end()
}
dictionary.updateDomainLabels = async (req, res) => {
const { dictionaryId, payload } = req.body.params
await Dictionary.updateDomainLabel(dictionaryId, payload)
res.end()
}
dictionary.renovateSecondaryDomains = async (req, res) => {
const data = req.body.params.payload
await Dictionary.renovateSecondaryDomains(data)
res.end()
}
dictionary.getEntryVersionSnapshot = async (req, res) => {
const { entryId, version } = req.params
const historySnapshot = await Entry.fetchVersionSnapshot(entryId, version)
res.send(historySnapshot)
}
dictionary.listDictionaries = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const { pages_total: numberOfAllPages, results } =
await Dictionary.fetchAllAdminDictionaries(resultsPerPage, page)
res.send({ page, numberOfAllPages, results })
}
dictionary.listDomainLabels = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const { dictionaryId } = req.params
const page = +req.query.p > 0 ? +req.query.p : 1
const { pages_total: numberOfAllPages, results } =
await Dictionary.fetchPaginationDomainLabels(
dictionaryId,
resultsPerPage,
page
)
res.send({ page, numberOfAllPages, results })
}
dictionary.listSecondaryDomains = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const { pages_total: numberOfAllPages, results } =
await Dictionary.fetchAllSecondaryDomains(resultsPerPage, page)
res.send({ page, numberOfAllPages, results })
}
dictionary.extractionImport = async (req, res) => {
// TODO Import logic (Luka's task)
// const { id: dictionaryId, extractionId } = req.params
// const { from, to } = req.query
// const fromIndex = +from > 1 ? Math.floor(from) - 1 : 0
// const toIndex = Number.isInteger(+to) ? Math.abs(to) : undefined
// console.log({ dictionaryId, extractionId, fromIndex, toIndex })
res.send('IMPORTING')
}
module.exports = dictionary
+328
View File
@@ -0,0 +1,328 @@
const { unlink, rm, mkdir } = require('fs/promises')
const { promisify } = require('util')
const { URLSearchParams } = require('url')
const multer = require('multer')
const validator = require('validator')
const axios = require('axios')
const {
getExtractionFilesPath,
getDocumentsPath,
getStopTermsPath,
getConllusPath
} = require('../../../models/helpers/extraction')
const { checkIfcanBegin } = require('../../helpers/extraction')
const Extraction = require('../../../models/extraction')
const Domain = require('../../../models/domain')
const email = require('../../../models/email')
const { intoDbArray } = require('../../../models/helpers')
const { origin } = require('../../../config/keys')
const { DEFAULT_HITS_PER_PAGE } = require('../../../config/settings')
const MAX_FILE_NAME_LENGTH = 100
const MAX_FILE_SIZE = 10 ** 9 // 1 GB
const VALID_DOCUMENT_EXTENSIONS = ['txt', 'doc', 'docx', 'pdf']
const VALID_STOP_TERMS_FILE_EXTENSION = 'txt'
const MAX_OSS_DOCUMENT_COUNT = 500
const extractionFileStorage = multer.diskStorage({
destination: (req, file, cb) => {
const {
params: { id: extractionId },
fileType
} = req
const destinationPath =
fileType === 'stopTerms'
? getStopTermsPath(extractionId)
: getDocumentsPath(extractionId)
cb(null, destinationPath)
},
filename: (req, file, cb) => cb(null, file.originalname)
})
const extractionFileBodyParser = multer({
storage: extractionFileStorage,
limits: {
fieldNameSize: 15,
fieldSize: 10,
fields: 1,
fileSize: MAX_FILE_SIZE,
headerPairs: 500
},
fileFilter: extractionFileFilter
}).single('extractionFile')
const parseExtractionFileBody = promisify(extractionFileBodyParser)
const extraction = {}
extraction.delete = async (req, res) => {
const extractionId = req.params.id
const corpusId = await Extraction.delete(extractionId)
if (corpusId) {
await axios.delete(`http://concordancer:5000/dashboard/corpus/${corpusId}`)
}
const extractionFilesPath = getExtractionFilesPath(extractionId)
await rm(extractionFilesPath, { recursive: true })
res.end()
}
extraction.docsList = async (req, res) => {
const extractionId = req.params.id
const documentsStats = await Extraction.fetchAllDocumentsStats(extractionId)
res.send(documentsStats)
}
extraction.docsUpdate = async (req, res) => {
try {
await parseExtractionFileBody(req, res)
} catch (error) {
if (
error instanceof multer.MulterError &&
error.code === 'LIMIT_FILE_SIZE'
) {
throw Error('File too large. Must not be over 1 GB.')
}
throw error
}
res.end()
}
extraction.docDelete = async (req, res) => {
const { id: extractionId, filename } = req.params
const documentsPath = getDocumentsPath(extractionId)
const filePath = `${documentsPath}/${filename}`
await unlink(filePath)
res.end()
}
extraction.stopTermsList = async (req, res) => {
const extractionId = req.params.id
const stopTermsFilesStats = await Extraction.fetchAllStopTermsFilesStats(
extractionId
)
res.send(stopTermsFilesStats)
}
extraction.stopTermsUpdate = async (req, res) => {
try {
await parseExtractionFileBody(req, res)
} catch (error) {
if (
error instanceof multer.MulterError &&
error.code === 'LIMIT_FILE_SIZE'
) {
throw Error('File too large. Must not be over 1 GB.')
}
throw error
}
res.end()
}
extraction.stopTermDelete = async (req, res) => {
const { id: extractionId, filename } = req.params
const stopTermsPath = getStopTermsPath(extractionId)
const filePath = `${stopTermsPath}/${filename}`
await unlink(filePath)
res.end()
}
extraction.ossSaveParams = [saveOssParams, (req, res) => res.end()]
extraction.ossSearch = [
saveOssParams,
async (req, res) => {
const { id: extractionId } = req.params
const { ossParams } = req
const searchParams = new URLSearchParams({
...(ossParams.year && { leta: ossParams.year }),
...(ossParams.documentType && { vrste: ossParams.documentType }),
...(ossParams.keywords && { kljucneBesede: ossParams.keywords }),
...(ossParams.domainUdk && { udk: ossParams.domainUdk })
})
const searchApiUrl = `http://rsdo.lhrs.feri.um.si:8080/oss/steviloBesedilPoIskanju?${searchParams}`
const { data: documentCount } = await axios.get(searchApiUrl)
const canSave = documentCount && documentCount <= MAX_OSS_DOCUMENT_COUNT
await Extraction.updateOssParams(extractionId, {
params: ossParams,
status: canSave ? 'valid' : 'invalid'
})
res.send({ documentCount, canSave })
}
]
extraction.ossConfirmParams = async (req, res) => {
const { id: extractionId } = req.params
const { ossParams } = await Extraction.fetch(extractionId)
if (ossParams.status !== 'valid') throw Error('OSS params not valid')
await Extraction.updateOssParams(extractionId, {
params: ossParams.params,
status: 'confirmed'
})
res.end()
}
extraction.begin = async (req, res) => {
const extractionId = req.params.id
const extraction = await Extraction.fetch(extractionId)
const canBegin = await checkIfcanBegin(extraction)
if (!canBegin) throw Error('Extraction does not qualify to be ran')
let timeStarted
const { ossParams, name: extractionName } = extraction
if (ossParams) {
timeStarted = await Extraction.beginOss(extractionId)
} else {
const conllusPath = getConllusPath(extractionId)
await mkdir(conllusPath, { recursive: true })
const documentsNames = await Extraction.fetchAllDocumentsNames(extractionId)
timeStarted = await Extraction.beginOwn(extractionId, documentsNames)
}
res.send(timeStarted)
if (ossParams) {
// TODO This next method is only a temporary solution.
// TODO It should be called before response and its execution delegated to a seperate process or at least a seperate thread.
await Extraction.processOss(extractionId, ossParams.params)
} else {
// TODO This next method is only a temporary solution.
// TODO It should be called before response and its execution delegated to a seperate process or at least a seperate thread.
await Extraction.processOwn(extractionId, extractionName)
}
const extractionLink = new URL('/luscenje', origin)
const renderAsync = promisify(req.app.render.bind(req.app))
const authorEmail = await Extraction.fetchAuthorEmail(extractionId)
const emailHtml = await renderAsync('email/extraction-done', {
extractionName,
extractionLink
})
await email.send({
to: authorEmail,
subject: 'Luščenje končano',
html: emailHtml
})
}
extraction.duplicate = async (req, res) => {
// TODO Validate, if can be duplicated: status = finished or failed.
// TODO Execute duplication.
res.send('DUPLICATING!')
}
extraction.termCandidatesExport = async (req, res) => {
// TODO CSV logic (Luka's task)
// const extractionId = req.params.id
// const { from, to } = req.query
// const fromIndex = +from > 1 ? Math.floor(from) - 1 : 0
// const toIndex = Number.isInteger(+to) ? Math.abs(to) : undefined
// console.log({ extractionId, fromIndex, toIndex })
res.download('public/images/help-amebis-logo-pug-demo.png')
}
extraction.listFinishedForUser = async (req, res) => {
const extractions = await Extraction.fetchFinishedForUser(req.user.id)
res.send(extractions)
}
extraction.listTermCandidates = async (req, res) => {
const extractionId = req.params.id
const termCandidatesJson = await Extraction.fetchTermCandidatesJson(
extractionId
)
const termCandidates = JSON.parse(termCandidatesJson).terminoloski_kandidati
const hitsPerPage = +req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const numberOfAllPages = Math.ceil(termCandidates.length / hitsPerPage)
res.send({ hitsPerPage, numberOfAllPages, termCandidates })
}
function extractionFileFilter(req, file, cb) {
let fileType
switch (req.path.split('/').at(-1)) {
case 'documents':
fileType = 'document'
break
case 'stop-terms':
fileType = 'stopTerms'
break
default:
return cb(Error('Invalid API endpoint'))
}
const filenamePartsArray = file.originalname.split('.')
const fileExtension = filenamePartsArray.pop()
if (
(fileType === 'document' &&
!VALID_DOCUMENT_EXTENSIONS.includes(fileExtension)) ||
(fileType === 'stopTerms' &&
fileExtension !== VALID_STOP_TERMS_FILE_EXTENSION)
) {
return cb(Error('Invalid file type'))
}
const fileName = filenamePartsArray.join('.')
if (!fileName || fileName.length > MAX_FILE_NAME_LENGTH) {
return cb(
Error(
`Filename must be between 1 and ${MAX_FILE_NAME_LENGTH} characters long.`
)
)
}
if (!validator.isAlphanumeric(fileName[0], 'sl-SI', { ignore: '_' })) {
return cb(
Error(
'Filename must begin with an alphanumeric character or an underscore.'
)
)
}
if (!validator.isAlphanumeric(fileName, 'sl-SI', { ignore: ' _-.' })) {
return cb(
Error(
'Filename can only contain alphanumeric characters, spaces, underscores, minuses and periods.'
)
)
}
req.fileType = fileType
cb(null, true)
}
async function saveOssParams(req, res, next) {
const { id: extractionId } = req.params
const { body } = req
const ossParams = {
...(body.domain && {
domainUdk: intoDbArray(
(await Domain.fetchById(body.domain)).udkCode,
'always'
)
}),
...(body.documentType && {
documentType: intoDbArray(body.documentType, 'always').map(type => +type)
}),
...(body.year && {
year: intoDbArray(body.year, 'always').map(year => +year)
}),
...(body.keywords && {
keywords: intoDbArray(body.keywords, 'always')
})
}
await Extraction.update(extractionId, body.name)
await Extraction.updateOssParams(extractionId, {
params: ossParams,
status: 'unvalidated'
})
req.ossParams = ossParams
next()
}
module.exports = extraction
@@ -0,0 +1,18 @@
const InterInstanceSync = require('../../../models/inter_instance_sync')
exports.listDictionaries = async (req, res) => {
const dictionaries = await InterInstanceSync.listDictionaries()
res.send(dictionaries)
}
exports.syncDictionary = async (req, res) => {
const did = req.params.dictionaryId
const since = req.query.lastSynced
? req.query.lastSynced
: '2000-01-01 00:00:00'
const entriesToSync = await InterInstanceSync.getUpdatedEntriesSince(
did,
since
)
res.send(entriesToSync)
}
+63
View File
@@ -0,0 +1,63 @@
const Portal = require('../../../models/portal')
const axios = require('axios')
const { DEFAULT_HITS_PER_PAGE } = require('../../../config/settings')
const portal = {}
portal.create = async (req, res) => {
const portal = await Portal.create(req.body)
res.send(portal)
}
portal.syncRemoteDictionaries = async (req, res) => {
const { linkedPortalId } = req.params
const { indexURL } = await Portal.fetchPortal(linkedPortalId)
const { data: dictionaries } = await axios.get(indexURL)
await Portal.syncRemoteDictionaries(linkedPortalId, dictionaries)
res.end()
}
portal.deleteLinkedDictionary = async (req, res) => {
const { linkedPortalId } = req.params
await Portal.deleteLinkedDictionary(linkedPortalId)
res.end()
}
portal.fetchDictionary = async (req, res) => {
const portalId = req.query.id
const dictionaries = await Portal.fetchDictionaries(portalId)
res.send(dictionaries)
}
portal.updatePortalStatus = async (req, res) => {
const portalId = req.body.params.id
const isEnabled = req.body.params.isEnabled
await Portal.updatePortalStatus(portalId, isEnabled)
res.end()
}
portal.listSelectedLinkedDicts = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const { portalId } = req.params
const page = +req.query.p > 0 ? +req.query.p : 1
const { pages_total: numberOfAllPages, results } =
await Portal.fetchSelectedLinkedDictionaries(portalId, resultsPerPage, page)
res.send({ page, numberOfAllPages, results })
}
portal.listAllLinkedDicts = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const { pages_total: numberOfAllPages, results } =
await Portal.fetchAllLinkedDictionaries(resultsPerPage, page)
res.send({ page, numberOfAllPages, results })
}
module.exports = portal
+230
View File
@@ -0,0 +1,230 @@
const Dictionary = require('../../../models/dictionary')
const { searchEntryIndex } = require('../../../models/search-engine')
const { intoDbArray } = require('../../../models/helpers')
const {
prepareEntries,
prepareEditorEntries,
prepareAggregation,
prepareSeachFilterData
} = require('../../../models/helpers/search')
const generateQuery = require('../../../models/helpers/search/generate-query')
const { getInstanceSetting } = require('../../../models/helpers')
const { DEFAULT_HITS_PER_PAGE } = require('../../../config/settings')
exports.listMainEntries = async (req, res) => {
const searchString = req.query.q?.trim()
if (!searchString) return res.status(400).end()
const filters = {
sourceLanguages: intoDbArray(req.query.sl, 'always'),
targetLanguages: intoDbArray(req.query.tl, 'always'),
primaryDomains: intoDbArray(req.query.pd, 'always'),
dictionaries: intoDbArray(req.query.d, 'always'),
sources: intoDbArray(req.query.s, 'always')
}
const hitsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const hitsQuery = await generateQuery.main(
searchString,
filters,
hitsPerPage,
page
)
const hits = await searchEntryIndex(hitsQuery)
const numberOfAllHits = hits.body.hits.total.value
const numberOfAllPages = Math.ceil(numberOfAllHits / hitsPerPage)
let entriesByCategory = prepareEntries(hits)
// TODO Luka (note to self): Consider reworking Miha's logic below. It's hacky and possibly error prone.
// TODO copied code below, maybe refactor to a helper function?
// check express/controllers/index.js search
entriesByCategory = Object.entries(entriesByCategory).reduce(
(acc, category, index) => {
acc[category[0]] = category[1].map(entry => {
if (entry.foreignEntries) {
entry.foreignEntries = entry.foreignEntries.map(fe => {
fe.terms = fe.terms ? fe.terms.filter(term => !!term) : []
fe.synonyms = fe.synonyms
? fe.synonyms.filter(synonym => !!synonym)
: []
fe.nbspCount = fe.terms.length + fe.synonyms.length - 1
return fe
})
}
return entry
})
return acc
},
{}
)
// res.send({ page, numberOfAllPages, entries })
res.append('page', page)
res.append('number-of-all-pages', numberOfAllPages)
res.render('utilities/response-pug-wrapper/entryLister', {
entriesByCategory
})
}
exports.listEditorEntries = async (req, res) => {
const { dictionaryId } = req.params
if (!dictionaryId) return res.status(400).end()
const qs = req.query
const searchField = qs.field
const searchString = qs.q?.trim() ?? ''
const filters = {
isValid: qs.isValid === undefined ? undefined : qs.isValid !== 'false',
isPublished:
qs.isPublished === undefined ? undefined : qs.isPublished !== 'false',
hasComments:
qs.hasComments === undefined ? undefined : qs.hasComments !== 'false',
isComplete:
qs.isComplete === undefined ? undefined : qs.isComplete !== 'false',
isTerminologyReviewed:
qs.isTerminologyReviewed === undefined
? undefined
: qs.isTerminologyReviewed !== 'false',
isLanguageReviewed:
qs.isLanguageReviewed === undefined
? undefined
: qs.isLanguageReviewed !== 'false'
}
const hitsQuery = generateQuery.editor(
dictionaryId,
searchField,
searchString,
filters
)
const hits = await searchEntryIndex(hitsQuery)
const entries = prepareEditorEntries(hits)
res.send(entries)
}
exports.listFilteredDictionaries = async (req, res) => {
let searchString = req.query.q?.trim()
if (!searchString) {
searchString = ''
}
const filters = {
// sourceLanguages: intoDbArray(req.query.sl, 'always'),
// targetLanguages: intoDbArray(req.query.tl, 'always'),
primaryDomains: intoDbArray(req.query.pd, 'always')
// sources: intoDbArray(req.query.s, 'always')
}
const orderType = req.query.orderType
const orderIndex = req.query.orderIndex === 'true'
// TODO: User another constant since this one is also user in the search results
const hitsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
// const defaultportalname = await getInstanceSetting('portal_name')
const defaultportalcode = await getInstanceSetting('portal_code')
let dictionaries
// mapping done here to not expose database info to public
let orderAttribute
if (orderType === 'domainName') {
orderAttribute = 'dp'
} else {
orderAttribute = 'd'
}
dictionaries = await Dictionary.fetchBasicInfoPerPageFilteredWithOrdering(
searchString,
filters.primaryDomains,
hitsPerPage,
page,
orderAttribute,
orderIndex
)
dictionaries = dictionaries.map(e => {
if (!e.portalcode) {
e.portalcode = defaultportalcode
// e.portalname = defaultportalname
}
return e
})
const numberOfAllHits = parseInt(
(await Dictionary.fetchFilteredCount(searchString, filters.primaryDomains))
.count
)
const numberOfAllPages = Math.ceil(numberOfAllHits / hitsPerPage)
// res.send({ page, numberOfAllPages, entries })
res.append('page', page)
res.append('number-of-all-pages', numberOfAllPages)
res.render('utilities/response-pug-wrapper/dictionaryLister', {
dictionaries
})
}
exports.showModalFilterResults = async (req, res) => {
const searchString = req.query.q?.trim()
if (!searchString) return res.status(400).end()
const selectedFilter = req.query.selectedFilter
// The selected
const filters = {
sourceLanguages:
selectedFilter === 'sourceLanguages'
? []
: intoDbArray(req.query.sl, 'always'),
targetLanguages:
selectedFilter === 'targetLanguages'
? []
: intoDbArray(req.query.tl, 'always'),
primaryDomains:
selectedFilter === 'primaryDomains'
? []
: intoDbArray(req.query.pd, 'always'),
dictionaries:
selectedFilter === 'dictionaries'
? []
: intoDbArray(req.query.d, 'always'),
sources:
selectedFilter === 'sources' ? [] : intoDbArray(req.query.s, 'always')
}
const hitsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const [, aggregateQuery] = await generateQuery.main(
searchString,
filters,
hitsPerPage,
page,
true
)
const aggregationRaw = await searchEntryIndex(aggregateQuery)
const aggregation = await prepareAggregation(aggregationRaw)
res.send(prepareSeachFilterData(aggregation, filters))
}
+16
View File
@@ -0,0 +1,16 @@
const debug = require('debug')('termPortal:controllers/api/v1/system')
const Eurotermbank = require('../../../models/system/eurotermbank')
exports.handleCspReports = (req, res) => {
debug(req.body)
res.sendStatus(200)
}
exports.syncWithEurotermbank = async (req, res) => {
try {
await Eurotermbank.push()
res.send('Sync successful')
} catch (error) {
res.send('Sync failed')
}
}
+35
View File
@@ -0,0 +1,35 @@
const User = require('../../../models/user')
const { DEFAULT_HITS_PER_PAGE } = require('../../../config/settings')
const users = {}
users.listUsers = async (req, res) => {
const resultsPerPage = req.user?.hitsPerPage || DEFAULT_HITS_PER_PAGE
const page = +req.query.p > 0 ? +req.query.p : 1
const { pages_total: numberOfAllPages, results } = await User.fetchAll(
resultsPerPage,
page
)
res.send({ page, numberOfAllPages, results })
}
users.updateHitsPerPage = async (req, res) => {
const hitAmount = req.body.hitAmount
await User.updateHitsPerPage(req.user.userName, hitAmount)
res.status(200).send()
}
users.updateFristNameAndSurname = async (req, res) => {
const firstname = req.body.name
const surname = req.body.surname
await User.updateFirstNameAndLastName(req.user.userName, firstname, surname)
res.status(200).send()
}
module.exports = users