package gui; import alg.XML_processing; import data.*; import javafx.application.HostServices; import javafx.beans.InvalidationListener; import javafx.beans.Observable; import javafx.beans.binding.StringBinding; import javafx.beans.property.ReadOnlyDoubleWrapper; import javafx.beans.value.ChangeListener; import javafx.beans.value.ObservableValue; import javafx.collections.FXCollections; import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; import javafx.concurrent.Task; import javafx.fxml.FXML; import javafx.scene.control.*; import javafx.scene.image.ImageView; import javafx.scene.layout.AnchorPane; import javafx.scene.layout.Pane; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.controlsfx.control.CheckComboBox; import java.io.File; import java.io.UnsupportedEncodingException; import java.util.*; import java.util.regex.Pattern; import static gui.GUIController.showAlert; @SuppressWarnings("Duplicates") public class CharacterAnalysisTab { public final static Logger logger = LogManager.getLogger(CharacterAnalysisTab.class); @FXML public AnchorPane characterAnalysisTab; @FXML public TextArea selectedFiltersTextArea; @FXML public Label stringLengthL; @FXML public Label calculateForL; @FXML public Label displayTaxonomyL; @FXML public Label dataLimitL; @FXML public Label msdL; @FXML public Label taxonomyL; @FXML public Label minimalOccurrencesL; @FXML public Label minimalTaxonomyL; @FXML public Label taxonomySetOperationL; @FXML public ImageView stringLengthI; @FXML public ImageView calculateForI; @FXML public ImageView displayTaxonomyI; @FXML public ImageView msdI; @FXML public ImageView taxonomyI; @FXML public ImageView minimalOccurrencesI; @FXML public ImageView minimalTaxonomyI; @FXML public ImageView taxonomySetOperationI; @FXML public Label solarFilters; @FXML private TextField msdTF; private ArrayList msd; private ArrayList msdStrings; @FXML private CheckComboBox taxonomyCCB; private ArrayList taxonomy; @FXML private CheckBox displayTaxonomyChB; private boolean displayTaxonomy; @FXML private CheckBox calculatecvvCB; private boolean calculateCvv; @FXML private TextField stringLengthTF; private Integer stringLength; @FXML private TextField minimalOccurrencesTF; private Integer minimalOccurrences; @FXML private TextField minimalTaxonomyTF; private Integer minimalTaxonomy; @FXML private ComboBox taxonomySetOperationCB; private String taxonomySetOperation; @FXML private ComboBox calculateForCB; private CalculateFor calculateFor; @FXML private Pane paneLetters; @FXML private Button computeNgramsB; @FXML private Button cancel; @FXML private Button changeLanguageB; @FXML public ProgressBar ngramProgressBar; @FXML public Label progressLabel; // @FXML // private Hyperlink helpH; private enum MODE { LETTER } private MODE currentMode; private Corpus corpus; private HashMap> solarFiltersMap; private HostServices hostService; private ListChangeListener taxonomyListener; private ChangeListener calculateForListener; private ChangeListener msdListener; private ChangeListener minimalOccurrencesListener; private ChangeListener minimalTaxonomyListener; private boolean useDb; private static final String [] N_GRAM_COMPUTE_FOR_LETTERS_ARRAY = {"calculateFor.WORD", "calculateFor.LOWERCASE_WORD", "calculateFor.LEMMA"}; private static final ArrayList N_GRAM_COMPUTE_FOR_LETTERS = new ArrayList<>(Arrays.asList(N_GRAM_COMPUTE_FOR_LETTERS_ARRAY)); private static final String [] N_GRAM_COMPUTE_FOR_WORDS_GOS_ARRAY = {"calculateFor.WORD", "calculateFor.LOWERCASE_WORD", "calculateFor.LEMMA", "calculateFor.NORMALIZED_WORD"}; private static final ArrayList N_GRAM_COMPUTE_FOR_WORDS_GOS = new ArrayList<>(Arrays.asList(N_GRAM_COMPUTE_FOR_WORDS_GOS_ARRAY)); private static final String [] TAXONOMY_SET_OPERATION_ARRAY = {"taxonomySetOperation.UNION", "taxonomySetOperation.INTERSECTION"}; private static final ArrayList TAXONOMY_SET_OPERATION = new ArrayList<>(Arrays.asList(TAXONOMY_SET_OPERATION_ARRAY)); // TODO: pass observables for taxonomy based on header scan // after header scan public void init() { characterAnalysisTab.getStylesheets().add("style.css"); characterAnalysisTab.getStyleClass().add("root"); manageTranslations(); currentMode = MODE.LETTER; toggleMode(currentMode); if (calculateForListener != null){ calculateForCB.valueProperty().removeListener(calculateForListener); } // calculateForCB calculateForListener = new ChangeListener() { boolean ignoreCode = false; @Override public void changed(ObservableValue observable, String oldValue, String newValue) { if (ignoreCode) { return; } boolean languageChanged = newValue == null; if (languageChanged) { ignoreCode = true; if (corpus.getCorpusType() == CorpusType.GOS) { newValue = I18N.getTranslatedValue(oldValue, N_GRAM_COMPUTE_FOR_WORDS_GOS); calculateForCB.getSelectionModel().select(newValue); } else { newValue = I18N.getTranslatedValue(oldValue, N_GRAM_COMPUTE_FOR_LETTERS); calculateForCB.getSelectionModel().select(newValue); } ignoreCode = false; } calculateFor = CalculateFor.factory(newValue); logger.info("calculateForCB:", calculateFor.toString()); } }; calculateForCB.valueProperty().addListener(calculateForListener); // calculateForCB.valueProperty().addListener((observable, oldValue, newValue) -> { // if(newValue == null){ // newValue = I18N.getTranslatedValue(oldValue, N_GRAM_COMPUTE_FOR_LETTERS); // calculateForCB.getSelectionModel().select(newValue); // } // calculateFor = CalculateFor.factory(newValue); // logger.info("calculateForCB:", calculateFor.toString()); // }); calculateForCB.getSelectionModel().select(0); taxonomySetOperationCB.setDisable(true); taxonomySetOperationCB.valueProperty().addListener((observable, oldValue, newValue) -> { if(newValue == null){ newValue = I18N.getTranslatedValue(oldValue, TAXONOMY_SET_OPERATION); taxonomySetOperationCB.getSelectionModel().select(newValue); } taxonomySetOperation = newValue; logger.info("Taxonomy set operation:", taxonomySetOperation); }); taxonomySetOperationCB.getSelectionModel().select(0); if (msdListener != null){ msdTF.focusedProperty().removeListener(msdListener); } // msd msdListener = new ChangeListener() { @Override public void changed(ObservableValue observable, Boolean oldValue, Boolean newValue) { if (!newValue) { // focus lost String value = msdTF.getText(); logger.info("msdTf: ", value); if (!ValidationUtil.isEmpty(value)) { ArrayList msdTmp = new ArrayList<>(Arrays.asList(value.split(" "))); int nOfRequiredMsdTokens = 1; if (msdTmp.size() != nOfRequiredMsdTokens) { String msg = String.format(I18N.get("message.WARNING_MISMATCHED_NGRAM_AND_TOKENS_VALUES"), nOfRequiredMsdTokens, msdTmp.size()); logAlert(msg); showAlert(Alert.AlertType.ERROR, msg); } msd = new ArrayList<>(); msdStrings = new ArrayList<>(); for (String msdToken : msdTmp) { msd.add(Pattern.compile(msdToken)); msdStrings.add(msdToken); } logger.info(String.format("msd accepted (%d)", msd.size())); } else if (!ValidationUtil.isEmpty(newValue)) { msd = new ArrayList<>(); msdStrings = new ArrayList<>(); } } } }; msdTF.focusedProperty().addListener(msdListener); msdTF.setText(""); msd = new ArrayList<>(); // taxonomy if (Tax.getCorpusTypesWithTaxonomy().contains(corpus.getCorpusType()) && corpus.getObservableListTaxonomy().size() > 0) { taxonomyCCB.setDisable(false); } else { taxonomyCCB.setDisable(true); } if (taxonomyListener != null){ taxonomyCCB.getCheckModel().getCheckedItems().removeListener(taxonomyListener); } taxonomyListener = new ListChangeListener() { boolean changing = true; @Override public void onChanged(Change c){ if(changing) { ObservableList checkedItems = taxonomyCCB.getCheckModel().getCheckedItems(); ArrayList checkedItemsTaxonomy = Taxonomy.modifyingTaxonomy(taxonomy, checkedItems, corpus); taxonomy = new ArrayList<>(); taxonomy.addAll(checkedItemsTaxonomy); taxonomyCCB.getItems().removeAll(); taxonomyCCB.getItems().setAll(corpus.getObservableListTaxonomy()); changing = false; taxonomyCCB.getCheckModel().clearChecks(); for (Taxonomy t : checkedItemsTaxonomy) { taxonomyCCB.getCheckModel().check(t.toLongNameString()); } if (taxonomyCCB.getCheckModel().getCheckedItems().size() > 0) { taxonomySetOperationCB.setDisable(false); } else { taxonomySetOperationCB.getSelectionModel().select(0); taxonomySetOperationCB.setDisable(true); } changing = true; logger.info(String.format("Selected taxonomy: %s", StringUtils.join(checkedItems, ","))); } } }; taxonomyCCB.getCheckModel().clearChecks(); taxonomyCCB.getItems().removeAll(); taxonomyCCB.getItems().setAll(corpus.getObservableListTaxonomy()); taxonomyCCB.getCheckModel().getCheckedItems().addListener(taxonomyListener); displayTaxonomy = false; displayTaxonomyChB.setSelected(false); // set if (Tax.getCorpusTypesWithTaxonomy().contains(corpus.getCorpusType()) && corpus.getObservableListTaxonomy().size() > 0) { displayTaxonomyChB.setDisable(false); displayTaxonomyChB.selectedProperty().addListener((observable, oldValue, newValue) -> { displayTaxonomy = newValue; if (displayTaxonomy) { minimalTaxonomyTF.setDisable(false); } else { minimalTaxonomyTF.setDisable(true); minimalTaxonomyTF.setText("1"); minimalTaxonomy = 1; } logger.info("display taxonomy: ", displayTaxonomy); }); displayTaxonomyChB.setTooltip(new Tooltip(I18N.get("message.TOOLTIP_readDisplayTaxonomyChB"))); } else { displayTaxonomyChB.setDisable(true); } // cvv calculatecvvCB.selectedProperty().addListener((observable, oldValue, newValue) -> { calculateCvv = newValue; logger.info("calculate cvv: " + calculateCvv); }); // string length stringLengthTF.focusedProperty().addListener((observable, oldValue, newValue) -> { if (!newValue) { // focus lost String value = stringLengthTF.getText(); if (!ValidationUtil.isEmpty(value)) { if (!ValidationUtil.isNumber(value)) { logAlert("stringlengthTf: " + I18N.get("message.WARNING_ONLY_NUMBERS_ALLOWED")); showAlert(Alert.AlertType.ERROR, I18N.get("message.WARNING_ONLY_NUMBERS_ALLOWED")); } stringLength = Integer.parseInt(value); } else { showAlert(Alert.AlertType.ERROR, I18N.get("message.WARNING_MISSING_STRING_LENGTH")); stringLengthTF.setText("1"); logAlert(I18N.get("message.WARNING_MISSING_STRING_LENGTH")); } } }); // set default values minimalOccurrencesTF.setText("1"); minimalOccurrences = 1; minimalTaxonomyTF.setText("1"); minimalTaxonomy = 1; minimalTaxonomyTF.setDisable(true); if (minimalOccurrencesListener != null){ minimalOccurrencesTF.focusedProperty().removeListener(minimalOccurrencesListener); } // msd minimalOccurrencesListener = new ChangeListener() { @Override public void changed(ObservableValue observable, Boolean oldValue, Boolean newValue) { if (!newValue) { // focus lost String value = minimalOccurrencesTF.getText(); if (!ValidationUtil.isEmpty(value)) { if (!ValidationUtil.isNumber(value)) { logAlert("minimalOccurrencesTF: " + I18N.get("message.WARNING_ONLY_NUMBERS_ALLOWED")); showAlert(Alert.AlertType.ERROR, I18N.get("message.WARNING_ONLY_NUMBERS_ALLOWED")); } else { minimalOccurrences = Integer.parseInt(value); } } else { minimalOccurrencesTF.setText("1"); minimalOccurrences = 1; } } } }; minimalOccurrencesTF.focusedProperty().addListener(minimalOccurrencesListener); if (minimalTaxonomyListener != null){ minimalTaxonomyTF.focusedProperty().removeListener(minimalTaxonomyListener); } minimalTaxonomyListener = new ChangeListener() { @Override public void changed(ObservableValue observable, Boolean oldValue, Boolean newValue) { if (!newValue) { // focus lost String value = minimalTaxonomyTF.getText(); if (!ValidationUtil.isEmpty(value)) { if (!ValidationUtil.isNumber(value)) { logAlert("minimalTaxonomyTF: " + I18N.get("message.WARNING_ONLY_NUMBERS_ALLOWED")); showAlert(Alert.AlertType.ERROR, I18N.get("message.WARNING_ONLY_NUMBERS_ALLOWED")); } else { minimalTaxonomy = Integer.parseInt(value); } } else { minimalTaxonomyTF.setText("1"); minimalTaxonomy = 1; } } } }; minimalTaxonomyTF.focusedProperty().addListener(minimalTaxonomyListener); computeNgramsB.setOnAction(e -> { compute(); logger.info("compute button"); }); changeLanguageB.setOnAction(e -> { if (I18N.getLocale() == new Locale.Builder().setLanguage("sl").setRegion("SI").build()){ I18N.setLocale(Locale.ENGLISH); } else { I18N.setLocale(new Locale.Builder().setLanguage("sl").setRegion("SI").build()); } Messages.reload(); Messages.updateChooseCorpusL(); logger.info("change language"); }); // helpH.setOnAction(e -> openHelpWebsite()); cancel.setVisible(false); } private void addTooltipToImage(ImageView image, StringBinding stringBinding){ Tooltip tooltip = new Tooltip(); tooltip.textProperty().bind(stringBinding); Tooltip.install(image, tooltip); } private void manageTranslations(){ // helpH.textProperty().bind(I18N.createStringBinding("hyperlink.help")); changeLanguageB.textProperty().bind(I18N.createStringBinding("button.language")); computeNgramsB.textProperty().bind(I18N.createStringBinding("button.computeNgrams")); cancel.textProperty().bind(I18N.createStringBinding("button.cancel")); stringLengthL.textProperty().bind(I18N.createStringBinding("label.stringLength")); calculateForL.textProperty().bind(I18N.createStringBinding("label.calculateFor")); displayTaxonomyL.textProperty().bind(I18N.createStringBinding("label.displayTaxonomy")); dataLimitL.textProperty().bind(I18N.createStringBinding("label.dataLimit")); msdL.textProperty().bind(I18N.createStringBinding("label.msd")); taxonomyL.textProperty().bind(I18N.createStringBinding("label.taxonomy")); minimalOccurrencesL.textProperty().bind(I18N.createStringBinding("label.minimalOccurrences")); minimalTaxonomyL.textProperty().bind(I18N.createStringBinding("label.minimalTaxonomy")); taxonomySetOperationL.textProperty().bind(I18N.createStringBinding("label.taxonomySetOperation")); addTooltipToImage(stringLengthI, I18N.createStringBinding("label.letter.stringLengthH")); addTooltipToImage(calculateForI, I18N.createStringBinding("label.letter.calculateForH")); addTooltipToImage(displayTaxonomyI, I18N.createStringBinding("label.letter.displayTaxonomyH")); addTooltipToImage(msdI, I18N.createStringBinding("label.letter.msdH")); addTooltipToImage(taxonomyI, I18N.createStringBinding("label.letter.taxonomyH")); addTooltipToImage(minimalOccurrencesI, I18N.createStringBinding("label.letter.minimalOccurrencesH")); addTooltipToImage(minimalTaxonomyI, I18N.createStringBinding("label.letter.minimalTaxonomyH")); addTooltipToImage(taxonomySetOperationI, I18N.createStringBinding("label.letter.taxonomySetOperationH")); solarFilters.textProperty().bind(I18N.createStringBinding("label.solarFilters")); calculateForCB.itemsProperty().bind(I18N.createObjectBinding(N_GRAM_COMPUTE_FOR_LETTERS)); taxonomySetOperationCB.itemsProperty().bind(I18N.createObjectBinding(TAXONOMY_SET_OPERATION)); } /** * Toggles visibility for panes which hold fields for skipgram value (not applicable when calculating for letters) etc., * sets combobox values to what is applicable ... * * @param mode */ public void toggleMode(MODE mode) { if (mode == null) { mode = currentMode; } logger.info("mode: ", mode.toString()); if (mode == MODE.LETTER) { paneLetters.setVisible(true); // populate with default cvv length value if (stringLength == null) { stringLengthTF.setText("1"); stringLength = 1; } else { stringLengthTF.setText(String.valueOf(stringLength)); } // // if calculateFor was selected for something other than a word or a lemma -> reset // if (!(calculateFor == CalculateFor.WORD || calculateFor == CalculateFor.LEMMA || calculateFor == CalculateFor.LOWERCASE_WORD)) { // // if the user selected something else before selecting ngram for letters, reset that choice // calculateFor = CalculateFor.WORD; // // calculateForCB.getSelectionModel().select(0); // } if (corpus.getCorpusType() == CorpusType.GOS) { calculateForCB.itemsProperty().unbind(); calculateForCB.itemsProperty().bind(I18N.createObjectBinding(N_GRAM_COMPUTE_FOR_WORDS_GOS)); } else { calculateForCB.itemsProperty().unbind(); calculateForCB.itemsProperty().bind(I18N.createObjectBinding(N_GRAM_COMPUTE_FOR_LETTERS)); } } // override if orth mode, allow only word if (corpus.isGosOrthMode()) { // TODO change to // varietyRB.setDisable(true); msdTF.setDisable(true); } else { msdTF.setDisable(false); // varietyRB.setDisable(false); } } private void compute() { Filter filter = new Filter(); filter.setNgramValue(0); filter.setCalculateFor(calculateFor); filter.setMultipleKeys(new ArrayList<>()); filter.setMsd(msd); filter.setTaxonomy(taxonomy); filter.setDisplayTaxonomy(displayTaxonomy); filter.setAl(AnalysisLevel.STRING_LEVEL); filter.setSkipValue(0); filter.setIsCvv(calculateCvv); filter.setSolarFilters(solarFiltersMap); filter.setStringLength(stringLength); filter.setMinimalOccurrences(minimalOccurrences); filter.setMinimalTaxonomy(minimalTaxonomy); filter.setTaxonomySetOperation(taxonomySetOperation); String message = Validation.validateForStringLevel(filter); if (message == null) { // no errors logger.info("Executing: ", filter.toString()); StatisticsNew statistic = new StatisticsNew(corpus, filter, useDb); execute(statistic); } else { logAlert(message); showAlert(Alert.AlertType.ERROR, "Prosim izpolnite polja:", message); } } private void openHelpWebsite(){ hostService.showDocument(Messages.HELP_URL); } private void logAlert(String alert) { logger.info("alert: " + alert); } public Corpus getCorpus() { return corpus; } public void setCorpus(Corpus corpus) { this.corpus = corpus; if (corpus.getCorpusType() != CorpusType.SOLAR) { setSelectedFiltersLabel(null); } else { setSelectedFiltersLabel("/"); } } public void setSelectedFiltersLabel(String content) { if (content != null) { solarFilters.setVisible(true); selectedFiltersTextArea.setVisible(true); selectedFiltersTextArea.setText(content); } else { solarFilters.setVisible(false); selectedFiltersTextArea.setVisible(false); } } private void execute(StatisticsNew statistic) { logger.info("Started execution: ", statistic.getFilter()); Collection corpusFiles = statistic.getCorpus().getDetectedCorpusFiles(); final Task task = new Task() { @SuppressWarnings("Duplicates") @Override protected Void call() throws Exception { final boolean multipleFiles = CorpusType.multipleFilesCorpuses().contains(statistic.getCorpus().getCorpusType()); if(multipleFiles){ cancel.setVisible(true); } int i = 0; Date startTime = new Date(); Date previousTime = new Date(); int remainingSeconds = -1; for (File f : corpusFiles) { final int iFinal = i; XML_processing xml_processing = new XML_processing(); xml_processing.isCancelled = false; i++; if (isCancelled()) { updateMessage(I18N.get("message.CANCELING_NOTIFICATION")); break; } if(xml_processing.progressBarListener != null) { xml_processing.progressProperty().removeListener(xml_processing.progressBarListener); } if (multipleFiles) { if ((new Date()).getTime() - previousTime.getTime() > 500 || remainingSeconds == -1){ remainingSeconds = (int) (((new Date()).getTime() - startTime.getTime()) * (1.0/i) * (corpusFiles.size() - i) / 1000); previousTime = new Date(); } this.updateProgress(i, corpusFiles.size()); this.updateMessage(String.format(I18N.get("message.ONGOING_NOTIFICATION_ANALYZING_FILE_X_OF_Y"), i, corpusFiles.size(), f.getName(), remainingSeconds)); } else { xml_processing.progressBarListener = new InvalidationListener() { int remainingSeconds = -1; Date previousTime = new Date(); @Override public void invalidated(Observable observable) { cancel.setVisible(true); if ((new Date()).getTime() - previousTime.getTime() > 500 || remainingSeconds == -1){ remainingSeconds = (int) (((new Date()).getTime() - xml_processing.startTime.getTime()) * (1.0/(iFinal * 100 + ((ReadOnlyDoubleWrapper) observable).get() + 1)) * ((corpusFiles.size() - iFinal - 1) * 100 + 100 - ((ReadOnlyDoubleWrapper) observable).get()) / 1000); previousTime = new Date(); } xml_processing.isCancelled = isCancelled(); updateProgress((iFinal * 100) + ((ReadOnlyDoubleWrapper) observable).get() + 1, corpusFiles.size() * 100); updateMessage(String.format(I18N.get("message.ONGOING_NOTIFICATION_ANALYZING_FILE_X_OF_Y"), iFinal + 1, corpusFiles.size(), f.getName(), remainingSeconds)); } }; xml_processing.progressProperty().addListener(xml_processing.progressBarListener); } xml_processing.readXML(f.toString(), statistic); if (isCancelled()) { updateMessage(I18N.get("message.CANCELING_NOTIFICATION")); break; } } return null; } }; ngramProgressBar.progressProperty().bind(task.progressProperty()); progressLabel.textProperty().bind(task.messageProperty()); task.setOnSucceeded(e -> { try { boolean successullySaved = statistic.saveResultToDisk(); if (successullySaved) { showAlert(Alert.AlertType.INFORMATION, I18N.get("message.NOTIFICATION_ANALYSIS_COMPLETED")); } else { showAlert(Alert.AlertType.INFORMATION, I18N.get("message.NOTIFICATION_ANALYSIS_COMPLETED_NO_RESULTS")); } } catch (UnsupportedEncodingException e1) { showAlert(Alert.AlertType.ERROR, I18N.get("message.ERROR_WHILE_SAVING_RESULTS_TO_CSV")); logger.error("Error while saving", e1); } ngramProgressBar.progressProperty().unbind(); progressLabel.textProperty().unbind(); progressLabel.setText(""); cancel.setVisible(false); }); task.setOnFailed(e -> { showAlert(Alert.AlertType.ERROR, I18N.get("message.ERROR_WHILE_EXECUTING")); logger.error("Error while executing", e); ngramProgressBar.progressProperty().unbind(); ngramProgressBar.setProgress(0.0); progressLabel.textProperty().unbind(); progressLabel.setText(""); cancel.setVisible(false); }); task.setOnCancelled(e -> { showAlert(Alert.AlertType.INFORMATION, I18N.get("message.NOTIFICATION_ANALYSIS_CANCELED")); ngramProgressBar.progressProperty().unbind(); ngramProgressBar.setProgress(0.0); progressLabel.textProperty().unbind(); progressLabel.setText(""); cancel.setVisible(false); }); // When cancel button is pressed cancel analysis cancel.setOnAction(e -> { task.cancel(); logger.info("cancel button"); }); final Thread thread = new Thread(task, "task"); thread.setDaemon(true); thread.start(); } public void setSolarFiltersMap(HashMap> solarFiltersMap) { this.solarFiltersMap = solarFiltersMap; } public void setHostServices(HostServices hostServices){ this.hostService = hostServices; } }