const fs = require('fs') const xmlFlow = require('xml-flow') const xss = require('xss') const debug = require('debug')('termPortal:models/helpers/dictionary') const db = require('../../db') const { intoDbArray } = require('..') const JOBS_MAX = 50 const JOBS_MIN = 15 // Import given file into given dictionary. exports.readFileIntoDb = ( userId, dictionaryId, importFilePath, entryStatus, dbClient ) => { return new Promise((resolve, reject) => { const progress = { jobCount: 0, totalCount: 0, validCount: 0, isWholeFileRead: false, importError: null } entryStatus = entryStatus === 'complete' ? 'complete' : 'in_edit' const importFileReader = fs.createReadStream(importFilePath) const xmlStream = xmlFlow(importFileReader, { strict: true, trim: false, preserveMarkup: xmlFlow.ALWAYS, simplifyNodes: false, useArrays: xmlFlow.ALWAYS }) const handleError = handleImportError(xmlStream, progress, reject) const handleEntry = handleEntryXml( userId, dictionaryId, entryStatus, progress, dbClient, resolve, handleError ) xmlStream .on('error', () => {}) .once('error', handleError) .on('end', () => (progress.isWholeFileRead = true)) .on('tag:entry', handleEntry) }) } function handleImportError(xmlStream, progress, reject) { return function handleError(error) { if (progress.importError) return progress.importError = error xmlStream.removeAllListeners('tag:entry') reject(error) } } function handleEntryXml( userId, dictionaryId, entryStatus, progress, dbClient, resolve, handleError ) { return async function handleEntry(entry) { if (progress.importError) return try { if (++progress.jobCount > JOBS_MAX) this.pause() const term = toMixedBasic( entry.$markup.find(el => el.$name === 'term')?.$markup ) const hwGrp = entry.$markup.find(el => el.$name === 'hwGrp') const wordforms = hwGrp?.$attrs.wfs const accent = hwGrp?.$attrs.acc const pronunciation = hwGrp?.$attrs.pron const domainLabels = entry.$markup .find(el => el.$name === 'domainLabels') ?.$markup.filter(childEl => childEl.$name === 'domainLabel') .map(domainLabel => toText(domainLabel.$markup)) const label = toMixedExtended( entry.$markup.find(el => el.$name === 'label')?.$markup ) const definition = toMixedExtended( entry.$markup.find(el => el.$name === 'def')?.$markup ) const synonyms = entry.$markup .find(el => el.$name === 'syns') ?.$markup.filter(childEl => childEl.$name === 'syn') .map(synonym => toMixedBasic(synonym.$markup)) const links = entry.$markup .find(el => el.$name === 'links') ?.$markup.filter(childEl => childEl.$name === 'link') .map(link => ({ link: toMixedBasic(link.$markup), type: link.$attrs.type })) const other = toMixedOther( entry.$markup.find(el => el.$name === 'other')?.$markup ) let hasForeignTerms = false const foreignLanguageContent = entry.$markup .find(el => el.$name === 'fLangs') ?.$markup.filter(childEl => childEl.$name === 'fLang') .map(fLang => ({ language: fLang.$attrs.lang, terms: fLang.$markup .find(el => el.$name === 'fTerms') ?.$markup.filter(childEl => childEl.$name === 'fTerm') .map(term => { const content = toMixedBasic(term.$markup) if (content.length) hasForeignTerms = true return content }), definition: toMixedExtended( fLang.$markup.find(el => el.$name === 'fDef')?.$markup ) || null, synonyms: fLang.$markup .find(el => el.$name === 'fSyns') ?.$markup.filter(childEl => childEl.$name === 'fSyn') .map(synonym => toMixedBasic(synonym.$markup)) })) const multimedia = entry.$markup.find(el => el.$name === 'mm')?.$markup const images = [] const audio = [] const videos = [] multimedia?.forEach(mm => { switch (mm.$name) { case 'image': images.push(toText(mm.$markup)) break case 'audio': audio.push(toText(mm.$markup)) break case 'video': videos.push(toText(mm.$markup)) } }) const isValid = !!term && (!!definition || hasForeignTerms) if (!entry.$markup.length) return const values = [ dictionaryId, isValid, entryStatus, term || null, userId, null, wordforms, accent, pronunciation, intoDbArray(domainLabels, 'always'), label || null, definition || null, intoDbArray(synonyms), intoDbArray(links, 'always'), other || null, intoDbArray(foreignLanguageContent, 'always'), images.length ? images : null, audio.length ? audio : null, videos.length ? videos : null ] const text = `SELECT entry_new (${db.genParamStr(values)})` await dbClient.query(text, values) progress.totalCount++ if (isValid) progress.validCount++ } catch (error) { handleError(error) } finally { progress.jobCount-- if (progress.jobCount < JOBS_MIN) this.resume() if ( !progress.jobCount && progress.isWholeFileRead && !progress.importError ) { debug(`TOTAL ENTRIES: ${progress.totalCount}`) debug(`VALID ENTRIES: ${progress.validCount}`) resolve() } } } } const markupFilter = { noMixed: new xss.FilterXSS({ whiteList: {}, stripIgnoreTag: true, stripIgnoreTagBody: ['script', 'style'] }), mixedBasic: new xss.FilterXSS({ whiteList: { sup: [], sub: [] }, stripIgnoreTag: true, stripIgnoreTagBody: ['script', 'style'] }), mixedExtended: new xss.FilterXSS({ whiteList: { sup: [], sub: [], b: [], i: [], a: ['href'] }, stripIgnoreTag: true, stripIgnoreTagBody: ['script', 'style'], onTag: customTagHandler }), mixedOther: new xss.FilterXSS({ whiteList: { sup: [], sub: [], b: [], i: [], a: ['href'], br: [] }, stripIgnoreTag: true, stripIgnoreTagBody: ['script', 'style'], onTag: customTagHandler }) } function customTagHandler(tag, html, { isWhite, isClosing }) { // Special treatment only for whitelisted opening anchor tags. if (tag !== 'a' || !isWhite || isClosing) return const matchUrl = html.match(/href="?(?https?:\/\/.*?)"?[\s>]/) const url = matchUrl ? xss.escapeAttrValue(matchUrl.groups.url) : undefined return `` } function toText(markupObj) { return markupFilter.noMixed .process(xmlFlow.toXml(markupObj)) .replace(/\s+/g, ' ') .trim() } function toMixedBasic(markupObj) { return markupFilter.mixedBasic .process(xmlFlow.toXml(markupObj)) .replace(/\s+/g, ' ') .trim() } function toMixedExtended(markupObj) { return markupFilter.mixedExtended .process(xmlFlow.toXml(markupObj)) .replace(/\s+/g, ' ') .trim() } function toMixedOther(markupObj) { return markupFilter.mixedOther .process(xmlFlow.toXml(markupObj)) .replace(/\s+/g, ' ') .trim() }