/* eslint-disable no-console */
/**
 * Linked dictionary updater. It is meant to run periodically (cron).
 * Linked server shall implement an api that that shall generally exhibit 2 endpoints:
 * - /APIPATH/dictionaries : shall return list of linked dictionaries, refer to API spec for detail
 * - /APIPATH/dictionary/$SOURCE_ID/entries?lastSynced=$SINCE
 * There may be some variations to the api since tha prota provides setting to linked server and dictionary URL.
 * This process will take given urls to obtain data by replacing relevant parameters:
 * - linked_portal.url is used to obtain updatable entries for each dictionary. Url field normally contains placeholders for dictionaryId an lastSyncTime
 * - linked dictionaries' table is processed one by one and updatable entries are requested for each linked dictionary.
 *
 * Updates:
 * nov 2022: added parameters to allow single node/dictionary sync on request
 */

const { exec } = require('child_process')
const fs = require('fs')
const async = require('async')
const needle = require('needle')
const xmlFlow = require('xml-flow')
const debug = require('debug')('termPortal:batch_update_linked')
const dbc = require('../models/db').getExtraClient()
const ConsultancyEntry = require('../models/consultancy-entry')
const batch = require('../models/batch')
const path = require('path')
const stream = require('stream')
const jobName = 'update_linked_dictionaries'
const threadId = '' + process.pid
const args = process.argv.slice(2)
process.title = jobName
const report = new batch.Report(jobName, threadId)
let linkedPortals = []
let linkedDictionaries = []
let linkedDictionaryEntriesToUpdate = []
let languageList = []
let limitType = ''
let limitCode = ''

let dictionarySyncStartTimestamp = null // marks begining of data pump, UTC, SQL timestamp form 'YYYY-MM-DD hh:mm:ss'

function printSynopsisAndExit() {
  console.error('node batch_update_linked [node <nodeId>|dict <dictId>|zrc]')
  process.exit(1)
}

if (args.length > 0) {
  switch (args[0]) {
    case 'node':
      if (args.length !== 2) printSynopsisAndExit()
      limitType = args[0]
      limitCode = args[1]
      updateDictionaries()
      break
    case 'dict':
      if (args.length !== 2) printSynopsisAndExit()
      limitType = args[0]
      limitCode = args[1]
      updateDictionaries()
      break
    case 'zrc':
      updateZrcSvetovalnica(err => {
        if (err) console.error(err)
        process.exit()
      })
      break
    default:
      printSynopsisAndExit()
  }
} else {
  updateDictionaries()
}

function updateDictionaries() {
  async.waterfall(
    [
      function (cbw) {
        debug('init')
        batch.init(dbc, report, cbw)
      },
      function (cbw) {
        debug('phase')
        batch.reportPhase(dbc, report, 'preparing', 0, cbw)
      },
      function (cbw) {
        debug('item')
        batch.reportItem(dbc, report, 'languages', 50, cbw)
      },
      function (cbw) {
        debug('fetching languages')
        dbc.query(
          'SELECT code, id FROM language ORDER BY code',
          (err, result) => {
            if (err) return cbw(err)
            languageList = result.rows
            debug(languageList)
            cbw()
          }
        )
      },
      function (cbw) {
        debug('item')
        batch.reportItem(dbc, report, 'servers', 100, err => {
          cbw(err)
        })
      },
      function (cbw) {
        debug('fetching servers')
        let sql =
          'SELECT * FROM linked_portal WHERE is_enabled = true ORDER BY id'
        if (limitType === 'node')
          sql =
            'SELECT * FROM linked_portal WHERE code=' +
            limitCode +
            ' AND is_enabled = true'
        if (limitType === 'dict')
          sql =
            "SELECT * FROM linked_portal WHERE id IN (SELECT linked_portal_id FROM linked_dictionary WHERE source_dictionary_id ='" +
            limitCode +
            "' AND is_enabled = true)"
        dbc.query(sql, (err, result) => {
          if (err) return cbw(err)
          linkedPortals = result.rows
          debug(linkedPortals)
          cbw()
        })
      },
      function (cbw) {
        debug('item')
        batch.reportItem(dbc, report, 'dictionaries', 60, err => {
          cbw(err)
        })
      },
      function (cbw) {
        debug('fetching dictionaries')
        let sql =
          'SELECT ld.*, lp.url_update, lp.url_index FROM linked_dictionary AS ld' +
          ' INNER JOIN linked_portal AS lp ON lp.id = ld.linked_portal_id' +
          ' WHERE ld.is_enabled = true AND lp.is_enabled = true' +
          ' ORDER BY lp.id, ld.id'
        if (limitType === 'node')
          sql =
            'SELECT ld.*, lp.url_update, lp.url_index FROM linked_dictionary AS ld' +
            ' INNER JOIN linked_portal AS lp ON lp.id = ld.linked_portal_id' +
            " WHERE ld.is_enabled = true AND lp.code = '" +
            limitCode +
            "'"
        else if (limitType === 'dict')
          sql =
            'SELECT ld.*, lp.url_update, lp.url_index FROM linked_dictionary AS ld' +
            ' INNER JOIN linked_portal AS lp ON lp.id = ld.linked_portal_id' +
            " WHERE ld.is_enabled = true AND ld.source_dictionary_id = '" +
            limitCode +
            "'"
        dbc.query(sql, (err, result) => {
          if (err) return cbw(err)
          linkedDictionaries = result.rows
          debug(linkedDictionaries)
          cbw()
        })
      },
      function (cbw) {
        if (!linkedDictionaries || linkedDictionaries.length === 0) {
          debug('finalize')
          return batch.finalize(dbc, report, 'no linked dictionaries', cbw)
        }
        const stepPercent = 100 / linkedDictionaries.length
        async.eachOfSeries(
          linkedDictionaries,
          (ldict, key, cbe) => {
            debug('processing dictionary', ldict.name)
            async.waterfall(
              [
                function (cbwd) {
                  batch.reportPhase(
                    dbc,
                    report,
                    'fetching ' + ldict.name,
                    stepPercent * (key + 0.2),
                    errd => {
                      cbwd(errd)
                    }
                  )
                },
                function (cbwd) {
                  fetchUpdates(ldict, errd => {
                    cbwd(errd)
                  })
                },
                function (cbwd) {
                  batch.reportPhase(
                    dbc,
                    report,
                    'updating ' + ldict.name,
                    stepPercent * (key + 0.5),
                    errd => {
                      cbwd(errd)
                    }
                  )
                },
                function (cbwd) {
                  applyUpdates(dbc, ldict, errd => {
                    cbwd(errd)
                  })
                },
                function (cbwd) {
                  dbc.query(
                    'UPDATE linked_dictionary SET time_last_synced=$1 WHERE id=$2',
                    [dictionarySyncStartTimestamp, ldict.id],
                    errd => {
                      cbwd(errd)
                    }
                  )
                }
              ],
              errw2 => {
                cbe(errw2)
              }
            )
          },
          erre => {
            cbw(erre)
          }
        )
      }
    ],
    err => {
      if (err) {
        console.error(err)
        batch.fail(dbc, report, err, errf => {
          if (errf) console.error(errf)
          dbc.end()
        })
      } else {
        batch.finalize(dbc, report, 'completed', errf => {
          if (errf) console.error(errf)
          dbc.end()
        })
      }
    }
  )
}

/**
 * fetch upates for given linked dictionary.
 * Updates are stored in global linkedDictionaryEntriesToUpdate
 * @param ldict the linked dictionary record
 * @param done
 */
function fetchUpdates(ldict, done) {
  const lastSync = ldict.synced
    ? ldict.time_last_synced.toISOString().substring(0, 19).replace('T', ' ')
    : '2000-01-01 00:00:00'
  dictionarySyncStartTimestamp = new Date()
    .toISOString()
    .substring(0, 19)
    .replace('T', ' ')
  const url = ldict.url_update
    .replace('$SOURCE_ID', encodeURIComponent(ldict.source_dictionary_id))
    .replace('$SINCE', encodeURIComponent(lastSync))
  debug('retrieving from ', url)
  const options = {
    compressed: true,
    // follow     : 10,
    accept: 'application/json'
  }
  needle.get(url, options, function (error, response, body) {
    if (error) return done(error)
    if (body.error) {
      // server side error message
      return done(body)
    }
    // the body will contain decoded JSON array of dictionary items to update
    if (Array.isArray(body)) {
      debug('fetched:', body.length)
      linkedDictionaryEntriesToUpdate = body
    } else {
      debug('fetched:', body.entries.length)
      linkedDictionaryEntriesToUpdate = body.entries
    }
    done()
  })
}

const sqlUpsertEntry =
  'INSERT INTO entry ' +
  ' (dictionary_id, is_valid, is_published, term, label, definition, synonym, external_id, external_url)' +
  ' VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)' +
  ' ON CONFLICT (dictionary_id,external_id)' +
  ' DO UPDATE SET time_modified = now(),' +
  ' term = $4, label = $5, definition = $6, synonym = $7, external_url = $9'

const sqlUpsertEntryTranslation =
  'INSERT INTO entry_foreign ' +
  ' (entry_id, language_id, term, definition, synonym)' +
  ' VALUES ($1,$2,$3,$4,$5)' +
  ' ON CONFLICT (entry_id, language_id)' +
  ' DO UPDATE SET term = $3, definition = $4, synonym = $5'

/**
 * Apply updates from global linkedDictionaryEntriesToUpdate to given dictionary.
 * @param dbc database connection
 * @param ldict the linked dictionary record
 * @param done
 */
function applyUpdates(dbc, ldict, done) {
  debug('applying updates')
  const targetDictionaryId = ldict.target_dictionary_id
  let currentEntry = null
  let termsSqlArray = ''
  let synonymsSqlArray = null
  let processed = 0
  let failed = 0
  async.eachOfSeries(
    linkedDictionaryEntriesToUpdate,
    (entry, key, cbe) => {
      let entryId = 0
      currentEntry = entry
      async.waterfall(
        [
          function (cbw) {
            debug('upsert entry', entry)
            dbc.query(
              sqlUpsertEntry,
              [
                targetDictionaryId,
                true,
                true,
                entry.term,
                entry.label,
                entry.definition,
                entry.synonym,
                '' + entry.id,
                entry.url
              ],
              errq => {
                cbw(errq)
              }
            )
          },
          function (cbw) {
            debug('fetch entry id')
            dbc.query(
              'SELECT id FROM entry WHERE dictionary_id=$1 AND external_id=$2',
              [targetDictionaryId, '' + entry.id],
              (errq, result) => {
                if (errq) return cbw(errq)
                entryId = result.rows[0].id
                cbw()
              }
            )
          },
          function (cbw) {
            debug('deleting removed entry translations')
            let includedLangIds = [0]
            entry.translations.forEach(et => {
              const lng = languageList.find(l => {
                return l.code === et.language_code
              })
              if (lng) includedLangIds.push(lng.id)
            })
            // keep unique ids
            includedLangIds = Array.from(new Set(includedLangIds))
            debug('except for languages', includedLangIds)
            dbc.query(
              'DELETE FROM entry_foreign WHERE entry_id=$1 AND language_id NOT IN (' +
                includedLangIds.join(',') +
                ')',
              [entryId],
              (errq, result) => {
                if (!errq && result.rowCount)
                  debug('deleted translations: ', result.rowCount)
                cbw(errq)
              }
            )
          },
          function (cbw) {
            debug('upserting entry translations')
            async.eachOfSeries(
              entry.translations,
              (trans, key, cbst) => {
                const transLang = languageList.find(l => {
                  return l.code === trans.language_code
                })
                if (!transLang) {
                  debug(
                    'WARN: language not mapped, skipping translation upsert: ',
                    trans
                  )
                  report.trace.push(
                    'WARN: language not mapped, skipping translation upsert: ' +
                      trans.language_code
                  )
                  report.warnings++
                  return cbst()
                }
                debug(
                  'updaing translation for entry',
                  entryId,
                  trans.lang,
                  trans.terms
                )
                termsSqlArray = '{"' + trans.term.join('","') + '"}'
                if (trans.synonym && trans.synonym.length) {
                  synonymsSqlArray = '{"' + trans.synonym.join('","') + '"}'
                }
                dbc.query(
                  sqlUpsertEntryTranslation,
                  [
                    entryId,
                    transLang.id,
                    termsSqlArray,
                    trans.definition,
                    synonymsSqlArray
                  ],
                  errq => {
                    if (errq) {
                      debug('WARN: error upserting translation: ', trans, errq)
                      report.trace.push(
                        'WARN: error upserting translation: ' + errq
                      )
                      report.warnings++
                    }
                    cbst()
                  }
                )
              },
              errt => {
                // translations upsert callback
                cbw(errt)
              }
            )
          }
        ],
        errw => {
          // entry callback
          if (errw) {
            console.error('updating entry failed', currentEntry, errw)
            failed++
          } else processed++
          cbe()
        }
      )
    },
    erre => {
      // entries callback
      if (failed > 0)
        console.error('Done with some entries failed:', failed, '/0', processed)
      done(erre)
    }
  )
}

const sqlUpsertZRCConsultancy =
  'INSERT INTO consultancy_entry ' +
  ' (id_external, status, time_published, institution, title, question, answer, answer_authors, domain_primary_id)' +
  ' VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)' +
  ' ON CONFLICT (id_external)' +
  ' DO UPDATE SET status = $2, time_published = $3, title = $5, question = $6, answer = $7, answer_authors = $8, domain_primary_id = $9'

/**
 * Update consultancy entries/index for sites
 * that will have their consultancy linked to ZRC/Terminološka svetovalnica.
 * The data is available as full export on https://www.zrc-sazu.si/sl/sites/default/files/xml/term-exp.xml
 * Requires CULR installed on a machine.
 */
function updateZrcSvetovalnica(cbf) {
  const file = path.join(__dirname, './term-exp.xml')
  let domainPrimary = null
  let consultancyQueue = []
  async.waterfall(
    [
      function (cbw) {
        debug('check consultancy type')
        dbc.query(
          "SELECT value FROM instance_settings WHERE name = 'consultancy_type'",
          [],
          (errq, result) => {
            debug('fetched consultancy_type', errq, result)
            if (errq) return cbw(errq)
            if (result.rows[0].value !== 'ZRC') {
              return cbw('ZRC consultancy not enabled')
            }
            cbw()
          }
        )
      },
      function (cbw) {
        debug('load domain primary')
        dbc.query(
          'SELECT id, name_sl, udk_code FROM domain_primary',
          [],
          (errq, result) => {
            debug('fetched domain_primary', errq, result)
            if (errq) return cbw(errq)
            domainPrimary = result.rows
            cbw()
          }
        )
      },
      function (cbw) {
        debug('check last update')
        cbw()
      },
      function (cbw) {
        debug('download source file')
        const cmd =
          'curl https://www.zrc-sazu.si/sl/sites/default/files/xml/term-exp.xml > "' +
          file +
          '"'
        exec(cmd, function (err, stdout, stderr) {
          if (err) {
            console.error(stderr)
            cbw(err)
          } else {
            console.debug('done')
            cbw()
          }
        })
      },
      function (cbw) {
        debug('process source file')
        let targetRow
        const inFile = fs.createReadStream(file)
        const xmlStream = xmlFlow(inFile, {
          strict: true,
          trim: false,
          preserveMarkup: xmlFlow.ALWAYS,
          simplifyNodes: false,
          useArrays: xmlFlow.ALWAYS
        })
        xmlStream
          .on('tag:row', function (row) {
            // debug(row);
            targetRow = {
              status: 'published',
              institution: 'ZRC-SAZU',
              description: 'Terminološka svetovalnica',
              authors: [],
              domain_primary_id: null
            }
            row.$markup.forEach(field => {
              let domain = null
              // debug('processing field',field)
              switch (field.$attrs.name) {
                case 'nid':
                  targetRow.id_external = field.$markup.length
                    ? field.$markup[0]
                    : null
                  break
                case 'title':
                  targetRow.title = field.$markup.length
                    ? field.$markup.join(' ')
                    : ''
                  break
                case 'field_vprasanje_value':
                  targetRow.question = field.$markup.length
                    ? field.$markup.join(' ')
                    : ''
                  break
                case 'body_value':
                  targetRow.answer = field.$markup.length
                    ? field.$markup.join(' ')
                    : ''
                  break
                case 'changed':
                  // debug('processing time_published',field)
                  targetRow.time_published = field.$markup.length
                    ? new Date(parseInt(field.$markup[0]))
                    : null
                  break
                case 'term_field_UDK':
                  domain = domainPrimary.find(d => {
                    return d.udk_code === field.$markup[0]
                  })
                  if (domain) targetRow.domain_primary_id = domain.id
                  break
                case 'term_field':
                  domain = domainPrimary.find(d => {
                    return d.name_sl === field.$markup[0]
                  })
                  if (domain && !targetRow.domain_primary_id)
                    targetRow.domain_primary_id = domain.id
                  break
                case 'authors':
                  field.$markup.forEach(author => {
                    targetRow.authors.push(author.$markup[0])
                  })
                  break
                default:
                  break // ignore unmapped
              }
            })
            consultancyQueue.push(targetRow)
            debug('row from xml', targetRow)
            if (consultancyQueue.length > 10 && !xmlStream.isPaused) {
              // above will accumulate the queue while here we're draining it
              let lastX = -1
              xmlStream.pause()
              const tempQueue = []
              while (consultancyQueue.length > 0)
                tempQueue.push(consultancyQueue.shift())
              async.eachOfSeries(
                tempQueue,
                (qrow, x, cbrow) => {
                  //  ' (id_external, consultancy_entry_status, time_published, institution, title, question, answer, answer_authors, domain_primary_id)' +
                  if (qrow === null) return cbrow()
                  lastX = x
                  dbc.query(
                    sqlUpsertZRCConsultancy,
                    [
                      qrow.id_external,
                      qrow.status,
                      qrow.time_published,
                      qrow.institution,
                      qrow.title,
                      qrow.question,
                      qrow.answer,
                      qrow.authors,
                      qrow.domain_primary_id
                    ],
                    errdb => {
                      if (errdb) {
                        debug(
                          'WARN: error upserting ZRC consultancy: ',
                          targetRow,
                          errdb
                        )
                        report.trace.push(
                          'WARN: error upserting translation: ' + errdb
                        )
                        report.warnings++
                      }
                      cbrow()
                    }
                  )
                },
                qerr => {
                  consultancyQueue = consultancyQueue.splice(0, lastX + 1)
                  xmlStream.resume()
                }
              )
            }
          })
          .on('error', xerr => {
            console.error(xerr)
          })
          .on('end', () => {
            cbw()
          })
      },
      function (cbw) {
        ConsultancyEntry.reindexAll()
          .catch(e => console.error('consultancy reindex failed:', e))
          .finally(() => cbw())
      }
    ],
    err => {
      cbf(err)
    }
  )
}