List of usage examples for com.google.common.collect Table get
V get(@Nullable Object rowKey, @Nullable Object columnKey);
From source file:com.ggvaidya.scinames.complexquery.ComplexQueryViewController.java
public void updateTableWithNameClusters(Project project, List<NameCluster> nameClusters, List<Dataset> datasets) { Table<NameCluster, String, Set<String>> precalc = HashBasedTable.create(); if (nameClusters == null) { dataTableView.setItems(FXCollections.emptyObservableList()); return;//from ww w. j av a 2 s. c o m } boolean flag_nameClustersAreTaxonConcepts = false; if (nameClusters.size() > 0 && TaxonConcept.class.isAssignableFrom(nameClusters.get(0).getClass())) flag_nameClustersAreTaxonConcepts = true; dataTableView.setItems(FXCollections.observableList(nameClusters)); // Precalculate. List<String> existingColNames = new ArrayList<>(); existingColNames.add("id"); existingColNames.add("name"); existingColNames.add("names_in_dataset"); existingColNames.add("all_names_in_cluster"); // If these are taxon concepts, there's three other columns we want // to emit. if (flag_nameClustersAreTaxonConcepts) { existingColNames.add("name_cluster_id"); existingColNames.add("starts_with"); existingColNames.add("ends_with"); existingColNames.add("is_ongoing"); } else { existingColNames.add("taxon_concept_count"); existingColNames.add("taxon_concepts"); } // Set<Name> recognizedNamesInDataset = namesDataset.getRecognizedNames(project).collect(Collectors.toSet()); for (NameCluster cluster : nameClusters) { precalc.put(cluster, "id", getOneElementSet(cluster.getId().toString())); // Okay, here's what we need to do: // - If names is ALL, then we can't do better than cluster.getName(). // if(namesDataset == ALL) { precalc.put(cluster, "names_in_dataset", cluster.getNames().stream().map(n -> n.getFullName()).collect(Collectors.toSet())); precalc.put(cluster, "name", getOneElementSet(cluster.getName().getFullName())); //} else { /* // hey, here's something cool we can do: figure out which name(s) // this dataset uses from this cluster! List<String> namesInDataset = cluster.getNames().stream() .filter(n -> recognizedNamesInDataset.contains(n)) .map(n -> n.getFullName()) .collect(Collectors.toList()); String firstName = ""; if(namesInDataset.size() > 0) firstName = namesInDataset.get(0); precalc.put(cluster, "names_in_dataset", new HashSet<>(namesInDataset)); precalc.put(cluster, "name", getOneElementSet(firstName)); }*/ precalc.put(cluster, "all_names_in_cluster", cluster.getNames().stream().map(n -> n.getFullName()).collect(Collectors.toSet())); // If it's a taxon concept, precalculate a few more columns. if (flag_nameClustersAreTaxonConcepts) { TaxonConcept tc = (TaxonConcept) cluster; precalc.put(cluster, "name_cluster_id", getOneElementSet(tc.getNameCluster().getId().toString())); precalc.put(cluster, "starts_with", tc.getStartsWith().stream().map(ch -> ch.toString()).collect(Collectors.toSet())); precalc.put(cluster, "ends_with", tc.getEndsWith().stream().map(ch -> ch.toString()).collect(Collectors.toSet())); precalc.put(cluster, "is_ongoing", getOneElementSet(tc.isOngoing(project) ? "yes" : "no")); } else { // If it's a true name cluster, then perhaps people will want // to know what taxon concepts are in here? Maybe for some sort // of PhD? List<TaxonConcept> tcs = cluster.getTaxonConcepts(project); precalc.put(cluster, "taxon_concept_count", getOneElementSet(String.valueOf(tcs.size()))); precalc.put(cluster, "taxon_concepts", tcs.stream().map(tc -> tc.toString()).collect(Collectors.toSet())); } // Okay, here's where we reconcile! for (Name n : cluster.getNames()) { // TODO: there's probably an optimization here, in which we should // loop on the smaller set (either loop on 'datasets' and compare // to cluster, or loop on cluster.foundIn and compare to 'datasets'). for (Dataset ds : datasets) { Map<Name, Set<DatasetRow>> rowsByName = ds.getRowsByName(); // Are we included in this name cluster? If not, skip! if (!cluster.getFoundIn().contains(ds)) continue; // Check to see if we have any rows for this name; if not, skip. if (!rowsByName.containsKey(n)) continue; Set<DatasetRow> matched = rowsByName.get(n); LOGGER.log(Level.FINER, "Adding {0} rows under name ''{1}''", new Object[] { matched.size(), n.getFullName() }); Map<Set<DatasetColumn>, List<DatasetRow>> rowsByCols = matched.stream() .collect(Collectors.groupingBy((DatasetRow row) -> row.getColumns())); for (Set<DatasetColumn> cols : rowsByCols.keySet()) { for (DatasetColumn col : cols) { String colName = col.getName(); if (existingColNames.contains(colName)) colName = "datasets." + colName; if (!precalc.contains(cluster, colName)) precalc.put(cluster, colName, new HashSet()); for (DatasetRow row : rowsByCols.get(cols)) { if (!row.hasColumn(col)) continue; precalc.get(cluster, colName).add(row.get(col)); } LOGGER.log(Level.FINER, "Added {0} rows under name ''{1}''", new Object[] { rowsByCols.get(cols).size(), n.getFullName() }); } } } } } dataTableView.getColumns().clear(); for (String colName : existingColNames) { dataTableView.getColumns().add(createColumnFromPrecalc(colName, precalc)); } // Get distinct column names. Stream<String> colNames = precalc.cellSet().stream().map(set -> set.getColumnKey()); // Eliminate columns that are in the existingColNames. colNames = colNames.filter(colName -> !existingColNames.contains(colName)); // And add tablecolumns for the rest. List<TableColumn<NameCluster, String>> cols = colNames.distinct().sorted() .map(colName -> createColumnFromPrecalc(colName, precalc)).collect(Collectors.toList()); dataTableView.getColumns().addAll(cols); dataTableView.refresh(); // Fill in status text field. statusTextField .setText(dataTableView.getItems().size() + " rows across " + cols.size() + " reconciled columns"); }
From source file:com.ggvaidya.scinames.ui.DataReconciliatorController.java
private void reconcileDataFromOneDataset() { Project project = dataReconciliatorView.getProjectView().getProject(); String reconciliationMethod = reconcileUsingComboBox.getValue(); Table<String, String, Set<String>> precalc = HashBasedTable.create(); Dataset namesDataset = useNamesFromComboBox.getSelectionModel().getSelectedItem(); List<NameCluster> nameClusters = null; List<Name> namesInDataset = null; // Set up namesInDataset. switch (namesToUseComboBox.getValue()) { case USE_NAMES_IN_DATASET_ROWS: if (namesDataset == ALL) { namesInDataset = project.getDatasets().stream().flatMap(ds -> ds.getNamesInAllRows().stream()) .distinct().sorted().collect(Collectors.toList()); } else {//from w w w. ja va2 s. co m namesInDataset = namesDataset.getNamesInAllRows().stream().sorted().distinct() .collect(Collectors.toList()); } break; case USE_ALL_REFERENCED_NAMES: if (namesDataset == ALL) { namesInDataset = project.getDatasets().stream().flatMap(ds -> ds.getReferencedNames()).distinct() .sorted().collect(Collectors.toList()); } else { namesInDataset = namesDataset.getReferencedNames().sorted().collect(Collectors.toList()); } break; case USE_ALL_RECOGNIZED_NAMES: if (namesDataset == ALL) { namesInDataset = project.getDatasets().stream() .flatMap(ds -> project.getRecognizedNames(ds).stream()).distinct().sorted() .collect(Collectors.toList()); } else { namesInDataset = project.getRecognizedNames(namesDataset).stream().sorted() .collect(Collectors.toList()); } break; } // IMPORTANT NOTE // This algorithm now relies on nameClusters and namesInDataset // having EXACTLY the same size. So please make sure every combination // of logic here lines up exactly. boolean flag_nameClustersAreTaxonConcepts = false; switch (reconciliationMethod) { case RECONCILE_BY_NAME: // namesInDataset already has all the names we want. nameClusters = createSingleNameClusters(namesDataset, namesInDataset); break; case RECONCILE_BY_SPECIES_NAME: namesInDataset = namesInDataset.stream().filter(n -> n.hasSpecificEpithet()) .flatMap(n -> n.asBinomial()).distinct().sorted().collect(Collectors.toList()); nameClusters = createSingleNameClusters(namesDataset, namesInDataset); break; case RECONCILE_BY_SPECIES_NAME_CLUSTER: // nameClusters = project.getNameClusterManager().getSpeciesClustersAfterFiltering(project).collect(Collectors.toList()); namesInDataset = namesInDataset.stream().filter(n -> n.hasSpecificEpithet()) .flatMap(n -> n.asBinomial()).distinct().sorted().collect(Collectors.toList()); nameClusters = project.getNameClusterManager().getClusters(namesInDataset); break; case RECONCILE_BY_NAME_CLUSTER: // Note that this includes genus name clusters! nameClusters = project.getNameClusterManager().getClusters(namesInDataset); break; case RECONCILE_BY_SPECIES_TAXON_CONCEPT: /* * WARNING: untested! Please test before using! */ List<NameCluster> nameClustersByName = project.getNameClusterManager().getClusters(namesInDataset); List<Name> namesInDatasetCorresponding = new LinkedList<>(); List<NameCluster> nameClustersCorresponding = new LinkedList<>(); for (int x = 0; x < namesInDataset.size(); x++) { Name name = namesInDataset.get(0); NameCluster nameCluster = nameClustersByName.get(0); List<TaxonConcept> taxonConcepts; if (nameCluster == null) { taxonConcepts = new ArrayList<>(); } else { taxonConcepts = nameCluster.getTaxonConcepts(project); } // Now we need to unwind this data structure: each entry in nameClusters // should have a corresponding entry in namesInDataset. for (TaxonConcept tc : taxonConcepts) { namesInDatasetCorresponding.add(name); nameClustersCorresponding.add((NameCluster) tc); } } // All good? Let's swap in those variables to replace their actual counterparts. namesInDataset = namesInDatasetCorresponding; nameClusters = nameClustersCorresponding; // This is special, at least for now. Maybe some day it won't? flag_nameClustersAreTaxonConcepts = true; break; default: LOGGER.log(Level.SEVERE, "Reconciliation method ''{0}'' has not yet been implemented!", reconciliationMethod); return; } if (nameClusters == null) { dataTableView.setItems(FXCollections.emptyObservableList()); return; } LOGGER.info("Name clusters ready to display: " + nameClusters.size() + " clusters"); LOGGER.info("Based on " + namesInDataset.size() + " names from " + namesDataset + ": " + namesInDataset); // What columns do we have from the other dataset? Dataset dataDataset = includeDataFromComboBox.getSelectionModel().getSelectedItem(); List<Dataset> datasets = null; if (dataDataset == ALL) datasets = project.getDatasets(); else if (dataDataset == NONE) datasets = new ArrayList<>(); else datasets = Arrays.asList(dataDataset); // Precalculate. List<String> existingColNames = new ArrayList<>(); existingColNames.add("id"); existingColNames.add("name"); existingColNames.add("names_in_dataset"); existingColNames.add("all_names_in_cluster"); existingColNames.add("dataset_rows_for_name"); existingColNames.add("name_cluster_id"); // existingColNames.add("distinct_dataset_rows_for_name"); // If these are taxon concepts, there's three other columns we want // to emit. if (flag_nameClustersAreTaxonConcepts) { existingColNames.add("starts_with"); existingColNames.add("ends_with"); existingColNames.add("is_ongoing"); } else { existingColNames.add("taxon_concept_count"); existingColNames.add("taxon_concepts"); existingColNames.add("trajectory"); existingColNames.add("trajectory_without_renames"); existingColNames.add("trajectory_lumps_splits"); } existingColNames.add("first_added_dataset"); existingColNames.add("first_added_year"); existingColNames.add("reconciliation_duplicate_of"); // Precalculate all dataset rows. Map<Name, Set<DatasetRow>> datasetRowsByName = new HashMap<>(); for (Dataset ds : datasets) { Map<Name, Set<DatasetRow>> rowsByName = ds.getRowsByName(); // Merge into the main list. for (Name n : rowsByName.keySet()) { Set<DatasetRow> rows = rowsByName.get(n); if (!reconciliationMethod.equals(RECONCILE_BY_NAME)) { // If we're reconciling by binomial names, then // we should include binomial names for each row, too. Optional<Name> binomialName = n.asBinomial().findAny(); if (binomialName.isPresent()) { Set<DatasetRow> rowsForBinomial = rowsByName.get(binomialName.get()); if (rowsForBinomial != null) rows.addAll(rowsForBinomial); // Don't write this to the sub-binomial name, // just write to the binomial name. n = binomialName.get(); } } if (!datasetRowsByName.containsKey(n)) datasetRowsByName.put(n, new HashSet<>()); datasetRowsByName.get(n).addAll(rows); } } LOGGER.info("Precalculating all dataset rows"); // Finally, come up with unique names for every dataset we might have. Map<DatasetColumn, String> datasetColumnMap = new HashMap<>(); existingColNames.addAll(datasets.stream().flatMap(ds -> ds.getColumns().stream()).distinct().map(col -> { String colName = col.getName(); String baseName = colName; int uniqueCounter = 0; while (existingColNames.contains(colName)) { // Duplicate column name! Map it elsewhere. uniqueCounter++; colName = baseName + "." + uniqueCounter; } // Where did we map it to? datasetColumnMap.put(col, colName); // Okay, now return the new column name we need to create. return colName; }).collect(Collectors.toList())); LOGGER.info("Precalculating " + nameClusters.size() + " name clusters"); // Make sure names and name clusters are unique, otherwise bail. // Earlier this was being ensured by keeping namesInDataset as a // Set, but since it's a List now, duplicates might sneak in. assert (namesInDataset.size() == new HashSet<>(namesInDataset).size()); // Since it's a list, we can set it up so that it always corresponds to // the correct name cluster. assert (namesInDataset.size() == nameClusters.size()); // Now, nameClusters should NOT be de-duplicated: we might have the same // cluster appear multiple times! If so, we'll set // "reconciliation_duplicate_of" to point to the first reconciliation, // so we don't duplicate reconciliations. // Let's track which IDs we use for duplicated name clusters. Map<NameCluster, List<String>> idsForNameClusters = new HashMap<>(); if (nameClusters.size() != new HashSet<>(nameClusters).size()) { LOGGER.warning("Clusters not unique: " + nameClusters.size() + " clusters found, but only " + new HashSet<>(nameClusters).size() + " are unique."); } // Track duplicates. Map<NameCluster, List<String>> clusterIDsPerNameCluster = new HashMap<>(); int totalClusterCount = nameClusters.size(); int currentClusterCount = 0; List<String> nameClusterIDs = new LinkedList<>(); for (NameCluster cluster : nameClusters) { currentClusterCount++; // Probably don't need GUIDs here, right? String clusterID = String.valueOf(currentClusterCount); nameClusterIDs.add(clusterID); LOGGER.info("(" + currentClusterCount + "/" + totalClusterCount + ") Precalculating name cluster: " + cluster); precalc.put(clusterID, "id", getOneElementSet(clusterID)); precalc.put(clusterID, "name_cluster_id", getOneElementSet(cluster.getId().toString())); // The 'name' should come from namesInDataset. precalc.put(clusterID, "name", getOneElementSet(namesInDataset.get(currentClusterCount - 1).getFullName())); // Okay, here's what we need to do: // - If names is ALL, then we can't do better than cluster.getName(). if (namesDataset == ALL) { precalc.put(clusterID, "names_in_dataset", cluster.getNames().stream().map(n -> n.getFullName()).collect(Collectors.toSet())); } else { // hey, here's something cool we can do: figure out which name(s) // this dataset uses from this cluster! Set<Name> namesToFilterTo = new HashSet<>(namesInDataset); List<String> namesInCluster = cluster.getNames().stream().filter(n -> namesToFilterTo.contains(n)) .map(n -> n.getFullName()).collect(Collectors.toList()); precalc.put(clusterID, "names_in_dataset", new HashSet<>(namesInCluster)); } precalc.put(clusterID, "all_names_in_cluster", cluster.getNames().stream().map(n -> n.getFullName()).collect(Collectors.toSet())); // Is this a duplicate? if (clusterIDsPerNameCluster.containsKey(cluster)) { List<String> duplicatedRows = clusterIDsPerNameCluster.get(cluster); // Only the first one should have the actual data. precalc.put(clusterID, "reconciliation_duplicate_of", getOneElementSet(duplicatedRows.get(0))); duplicatedRows.add(clusterID); // Okay, do no other work on this cluster, since all the actual information is // in the other entry. continue; } else { precalc.put(clusterID, "reconciliation_duplicate_of", getOneElementSet("NA")); List<String> clusterIds = new LinkedList<>(); clusterIds.add(clusterID); clusterIDsPerNameCluster.put(cluster, clusterIds); } LOGGER.fine("Cluster calculation began for " + cluster); // If it's a taxon concept, precalculate a few more columns. if (flag_nameClustersAreTaxonConcepts) { TaxonConcept tc = (TaxonConcept) cluster; precalc.put(clusterID, "starts_with", tc.getStartsWith().stream().map(ch -> ch.toString()).collect(Collectors.toSet())); precalc.put(clusterID, "ends_with", tc.getEndsWith().stream().map(ch -> ch.toString()).collect(Collectors.toSet())); precalc.put(clusterID, "is_ongoing", getOneElementSet(tc.isOngoing(project) ? "yes" : "no")); } else { // If it's a true name cluster, then perhaps people will want // to know what taxon concepts are in here? Maybe for some sort // of PhD? List<TaxonConcept> tcs = cluster.getTaxonConcepts(project); precalc.put(clusterID, "taxon_concept_count", getOneElementSet(String.valueOf(tcs.size()))); precalc.put(clusterID, "taxon_concepts", tcs.stream().map(tc -> tc.toString()).collect(Collectors.toSet())); } LOGGER.fine("Cluster calculation ended for " + cluster); // When was this first added? List<Dataset> foundInSorted = cluster.getFoundInSortedWithDates(); if (!foundInSorted.isEmpty()) { precalc.put(clusterID, "first_added_dataset", getOneElementSet(foundInSorted.get(0).getCitation())); precalc.put(clusterID, "first_added_year", getOneElementSet(foundInSorted.get(0).getDate().getYearAsString())); } LOGGER.fine("Trajectory began for " + cluster); // For name clusters we can also figure out trajectories! if (!flag_nameClustersAreTaxonConcepts) { List<String> trajectorySteps = cluster.getFoundInSortedWithDates().stream().map(dataset -> { String changes = dataset.getChanges(project).filter(ch -> cluster.containsAny(ch.getAllNames())) .map(ch -> ch.getType().toString()).collect(Collectors.joining("|")); if (!changes.isEmpty()) return changes; // This can happen when a change is referenced without an explicit addition. if (cluster.containsAny(dataset.getReferencedNames().collect(Collectors.toList()))) return "referenced"; else return "missing"; }).collect(Collectors.toList()); precalc.put(clusterID, "trajectory", getOneElementSet(String.join(" -> ", trajectorySteps))); precalc.put(clusterID, "trajectory_without_renames", getOneElementSet(trajectorySteps.stream() .filter(ch -> !ch.contains("rename")).collect(Collectors.joining(" -> ")))); precalc.put(clusterID, "trajectory_lumps_splits", getOneElementSet( trajectorySteps.stream().filter(ch -> ch.contains("split") || ch.contains("lump")) .collect(Collectors.joining(" -> ")))); } LOGGER.fine("Trajectory ended for " + cluster); // Okay, here's where we reconcile! LOGGER.fine("Reconciliation began for " + cluster); // Now we need to actually reconcile the data from these unique row objects. Set<DatasetRow> allDatasetRowsCombined = new HashSet<>(); for (Name name : cluster.getNames()) { // We don't have to convert cluster names to binomial, // because the cluster formation -- or the hacky thing we do // for RECONCILE_SPECIES_NAME -- should already have done that! // // Where necessary, the previous code will automatically // set up datasetRowsByName so it matched binomial names. Set<DatasetRow> rowsToReconcile = datasetRowsByName.get(name); if (rowsToReconcile == null) continue; allDatasetRowsCombined.addAll(rowsToReconcile); Set<DatasetColumn> columns = rowsToReconcile.stream().flatMap(row -> row.getColumns().stream()) .collect(Collectors.toSet()); for (DatasetColumn col : columns) { // We've precalculated column names. String colName = datasetColumnMap.get(col); // Make sure we get this column down into 'precalc'. if (!precalc.contains(clusterID, colName)) precalc.put(clusterID, colName, new HashSet<>()); // Add all values for all rows in this column. Set<String> vals = rowsToReconcile.stream().flatMap(row -> { if (!row.hasColumn(col)) return Stream.empty(); else return Stream.of(row.get(col)); }).collect(Collectors.toSet()); precalc.get(clusterID, colName).addAll(vals); LOGGER.fine("Added " + vals.size() + " rows under name cluster '" + cluster + "'"); } } LOGGER.info("(" + currentClusterCount + "/" + totalClusterCount + ") Reconciliation completed for " + cluster); precalc.put(clusterID, "dataset_rows_for_name", getOneElementSet(allDatasetRowsCombined.size())); } // Set up table items. dataTableView.setItems(FXCollections.observableList(nameClusterIDs)); LOGGER.info("Setting up columns: " + existingColNames); dataTableView.getColumns().clear(); for (String colName : existingColNames) { dataTableView.getColumns().add(createColumnFromPrecalc(colName, precalc)); } // Get distinct column names. Stream<String> colNames = precalc.cellSet().stream().map(set -> set.getColumnKey()); // Eliminate columns that are in the existingColNames. colNames = colNames.filter(colName -> !existingColNames.contains(colName)); // And add tablecolumns for the rest. List<TableColumn<String, String>> cols = colNames.distinct().sorted() .map(colName -> createColumnFromPrecalc(colName, precalc)).collect(Collectors.toList()); dataTableView.getColumns().addAll(cols); dataTableView.refresh(); // Fill in status text field. long distinctNameCount = precalc.cellSet().stream().map(cluster -> precalc.get(cluster, "name")).distinct() .count(); String str_duplicates = ""; if (distinctNameCount != dataTableView.getItems().size()) { str_duplicates = " for " + distinctNameCount + " distinct names"; } statusTextField.setText(dataTableView.getItems().size() + " rows across " + cols.size() + " reconciled columns" + str_duplicates); LOGGER.info("All done!"); }