List of usage examples for com.google.common.collect ImmutableSortedMap get
V get(Object key);
From source file:com.yahoo.spaclu.data.index.IndexFeatureValueSpark.java
public static void main(String[] args) throws IOException { IndexFeatureValueOptions optionsFormatRawToDatabase = new IndexFeatureValueOptions(args); String inputPathString = optionsFormatRawToDatabase.getInputPath(); String outputPathString = optionsFormatRawToDatabase.getOutputPath(); String indexPathString = optionsFormatRawToDatabase.getIndexPath(); int numberOfPartitions = optionsFormatRawToDatabase.getNumberOfPartitions(); int maxCutoffThreshold = optionsFormatRawToDatabase.getMaximumCutoffThreshold(); int minCutoffThreshold = optionsFormatRawToDatabase.getMinimumCutoffThreshold(); /*/*from ww w . j a v a 2s .c o m*/ * Set<String> excludingFeatureNames = new HashSet<String>(); * excludingFeatureNames.add("login"); * excludingFeatureNames.add("time"); excludingFeatureNames.add("day"); * excludingFeatureNames.add("hms"); excludingFeatureNames.add("fail"); */ sLogger.info("Tool: " + IndexFeatureValueSpark.class.getSimpleName()); sLogger.info(" - input path: " + inputPathString); sLogger.info(" - output path: " + outputPathString); sLogger.info(" - index path: " + indexPathString); sLogger.info(" - number of partitions: " + numberOfPartitions); sLogger.info(" - maximum cutoff: " + maxCutoffThreshold); sLogger.info(" - minimum cutoff: " + minCutoffThreshold); // Create a default hadoop configuration Configuration conf = new Configuration(); // Parse created config to the HDFS FileSystem fs = FileSystem.get(conf); Path outputPath = new Path(outputPathString); if (fs.exists(outputPath)) { fs.delete(outputPath, true); } SparkConf sparkConf = new SparkConf().setAppName(optionsFormatRawToDatabase.toString()); JavaSparkContext sc = new JavaSparkContext(sparkConf); Map<Integer, String> featureIndices = getFeatureIndices(sc.textFile(indexPathString)); List<Integer> listOfAllFeatureIndices = new LinkedList<Integer>(); List<String> listOfAllFeatureInfo = new LinkedList<String>(); Iterator<Integer> indexIter = featureIndices.keySet().iterator(); while (indexIter.hasNext()) { Integer tempKey = indexIter.next(); listOfAllFeatureIndices.add(tempKey); listOfAllFeatureInfo.add(featureIndices.get(tempKey)); } /* * * * * * * * */ JavaRDD<String> rawLines = sc.textFile(inputPathString).repartition(numberOfPartitions); JavaRDD<String[]> tokenizedLines = rawLines.map(new LineFilter(listOfAllFeatureIndices)); JavaPairRDD<Entry<Integer, String>, Long> featureValuesCounts = tokenizedLines .flatMapToPair(new FeatureValueMapper()).reduceByKey(new FeatureValueReducer()); Map<Integer, Builder<String, Long>> featureValueMapping = new Hashtable<Integer, Builder<String, Long>>(); Iterator<Tuple2<Entry<Integer, String>, Long>> iter = featureValuesCounts.collect().iterator(); while (iter.hasNext()) { Tuple2<Entry<Integer, String>, Long> temp = iter.next(); Entry<Integer, String> featureValueEntry = temp._1; int featureIndex = featureValueEntry.getKey(); String featureValue = featureValueEntry.getValue(); long featureValueCount = temp._2; if (!featureValueMapping.containsKey(featureIndex)) { Builder<String, Long> mapBuilder = new Builder<String, Long>(Ordering.natural()); featureValueMapping.put(featureIndex, mapBuilder); } featureValueMapping.get(featureIndex).put(featureValue, featureValueCount); } Preconditions.checkArgument(featureValueMapping.size() == listOfAllFeatureIndices.size()); String outputFeaturePathString = outputPathString + "feature" + Settings.SEPERATOR; fs.mkdirs(new Path(outputFeaturePathString)); String outputFeatureNamePathString = outputPathString + "feature.dat"; Path outputFeatureNamePath = new Path(outputFeatureNamePathString); PrintWriter featureNamePrinterWriter = new PrintWriter(fs.create(outputFeatureNamePath), true); List<Integer> listOfFeatureIndicesToKeep = new LinkedList<Integer>(); Map<Integer, Map<String, Integer>> featureValueIndex = new Hashtable<Integer, Map<String, Integer>>(); for (int d = 0; d < featureValueMapping.size(); d++) { Map<String, Integer> valueToIndex = new Hashtable<String, Integer>(); Map<Integer, String> indexToValue = new Hashtable<Integer, String>(); ImmutableSortedMap<String, Long> immutableSortedMap = featureValueMapping.get(d).build(); for (String keyString : immutableSortedMap.keySet()) { valueToIndex.put(keyString, valueToIndex.size()); indexToValue.put(indexToValue.size(), keyString); } if (valueToIndex.size() <= minCutoffThreshold || valueToIndex.size() > maxCutoffThreshold) { sLogger.info("Feature (" + listOfAllFeatureInfo.get(d) + ") contains " + valueToIndex.size() + " values, skip..."); continue; } else { sLogger.info("Feature (" + listOfAllFeatureInfo.get(d) + ") contains " + valueToIndex.size() + " values."); listOfFeatureIndicesToKeep.add(listOfAllFeatureIndices.get(d)); featureNamePrinterWriter.println(listOfAllFeatureInfo.get(d)); } String outputFeatureIndexPathString = outputFeaturePathString + "index" + Settings.UNDER_SCORE + featureValueIndex.size() + ".dat"; Path outputIndexPath = new Path(outputFeatureIndexPathString); featureValueIndex.put(featureValueIndex.size(), valueToIndex); PrintWriter featureValueIndexPrinterWriter = new PrintWriter(fs.create(outputIndexPath), true); for (int i = 0; i < indexToValue.size(); i++) { featureValueIndexPrinterWriter.println("" + i + Settings.TAB + indexToValue.get(i) + Settings.TAB + immutableSortedMap.get(indexToValue.get(i))); } featureValueIndexPrinterWriter.close(); } featureNamePrinterWriter.close(); JavaRDD<String[]> filteredLines = rawLines.map(new LineFilter(listOfFeatureIndicesToKeep)); JavaRDD<FeatureIntegerVector> indexedData = filteredLines.map(new FeatureValueIndexer(featureValueIndex)); String outputDataPathString = outputPathString + "data"; Path outputDataPath = new Path(outputDataPathString); if (fs.exists(outputDataPath)) { fs.delete(outputDataPath, true); } indexedData.saveAsTextFile(outputDataPathString); sc.stop(); }
From source file:org.gradle.internal.execution.impl.OutputFilterUtil.java
private static FileCollectionFingerprint getFingerprintForProperty( @Nullable ImmutableSortedMap<String, FileCollectionFingerprint> fingerprinters, String propertyName) { if (fingerprinters != null) { FileCollectionFingerprint afterPreviousExecution = fingerprinters.get(propertyName); if (afterPreviousExecution != null) { return afterPreviousExecution; }//from w ww. ja v a2 s .co m } return FileCollectionFingerprint.EMPTY; }
From source file:org.gradle.api.internal.changedetection.changes.Util.java
private static FileCollectionFingerprint getFingerprintAfterPreviousExecution( @Nullable ImmutableSortedMap<String, FileCollectionFingerprint> previous, String propertyName) { if (previous != null) { FileCollectionFingerprint afterPreviousExecution = previous.get(propertyName); if (afterPreviousExecution != null) { return afterPreviousExecution; }//from www . ja v a 2 s . c o m } return FileCollectionFingerprint.EMPTY; }
From source file:org.gradle.api.internal.tasks.execution.ResolveBeforeExecutionStateTaskExecuter.java
private static ImmutableSortedMap<String, ValueSnapshot> snapshotTaskInputProperties(TaskInternal task, TaskProperties taskProperties, ImmutableSortedMap<String, ValueSnapshot> previousInputProperties, ValueSnapshotter valueSnapshotter) { ImmutableSortedMap.Builder<String, ValueSnapshot> builder = ImmutableSortedMap.naturalOrder(); Map<String, Object> inputPropertyValues = taskProperties.getInputPropertyValues().create(); assert inputPropertyValues != null; for (Map.Entry<String, Object> entry : inputPropertyValues.entrySet()) { String propertyName = entry.getKey(); Object value = entry.getValue(); try {/* w w w . ja v a2s.c om*/ ValueSnapshot previousSnapshot = previousInputProperties.get(propertyName); if (previousSnapshot == null) { builder.put(propertyName, valueSnapshotter.snapshot(value)); } else { builder.put(propertyName, valueSnapshotter.snapshot(value, previousSnapshot)); } } catch (Exception e) { throw new UncheckedIOException(String.format( "Unable to store input properties for %s. Property '%s' with value '%s' cannot be serialized.", task, propertyName, value), e); } } return builder.build(); }
From source file:org.gradle.internal.execution.impl.OutputFilterUtil.java
public static ImmutableSortedMap<String, CurrentFileCollectionFingerprint> filterOutputFingerprints( final @Nullable ImmutableSortedMap<String, FileCollectionFingerprint> outputsAfterPreviousExecution, ImmutableSortedMap<String, CurrentFileCollectionFingerprint> outputsBeforeExecution, final ImmutableSortedMap<String, CurrentFileCollectionFingerprint> outputsAfterExecution) { return ImmutableSortedMap.copyOfSorted(Maps.transformEntries(outputsBeforeExecution, new Maps.EntryTransformer<String, CurrentFileCollectionFingerprint, CurrentFileCollectionFingerprint>() { @Override//from www . j ava 2 s .c om @SuppressWarnings("NullableProblems") public CurrentFileCollectionFingerprint transformEntry(String propertyName, CurrentFileCollectionFingerprint outputBeforeExecution) { CurrentFileCollectionFingerprint outputAfterExecution = outputsAfterExecution .get(propertyName); FileCollectionFingerprint outputAfterPreviousExecution = getFingerprintForProperty( outputsAfterPreviousExecution, propertyName); return filterOutputFingerprint(outputAfterPreviousExecution, outputBeforeExecution, outputAfterExecution); } })); }
From source file:org.apache.kylin.query.util.ConvertToComputedColumn.java
static String replaceComputedColumn(String inputSql, ImmutableSortedMap<String, String> computedColumn) { if (inputSql == null) { return ""; }//from w ww . j a va2 s . c om if (computedColumn == null || computedColumn.isEmpty()) { return inputSql; } String result = inputSql; String[] lines = inputSql.split("\n"); List<Pair<String, String>> toBeReplacedExp = new ArrayList<>(); //{"alias":"expression"}, like {"t1":"t1.a+t1.b+t1.c"} for (String ccExp : computedColumn.keySet()) { List<SqlNode> matchedNodes = new ArrayList<>(); try { matchedNodes = getMatchedNodes(inputSql, computedColumn.get(ccExp)); } catch (SqlParseException e) { logger.error("Convert to computedColumn Fail,parse sql fail ", e.getMessage()); } for (SqlNode node : matchedNodes) { Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, lines); int start = startEndPos.getLeft(); int end = startEndPos.getRight(); //add table alias like t1.column,if exists alias String alias = getTableAlias(node); toBeReplacedExp.add(Pair.of(alias, inputSql.substring(start, end))); } logger.debug("Computed column: " + ccExp + "'s matched list:" + toBeReplacedExp); //replace user's input sql for (Pair<String, String> toBeReplaced : toBeReplacedExp) { result = result.replace(toBeReplaced.getRight(), toBeReplaced.getLeft() + ccExp); } } return result; }
From source file:com.facebook.buck.apple.AppleDescriptions.java
/** * Convert {@link SourcePath} to a mapping of {@code include path -> file path}. * <p/>/*from w ww . j av a2s .c om*/ * {@code include path} is the path that can be referenced in {@code #include} directives. * {@code file path} is the actual path to the file on disk. * * @throws HumanReadableException when two {@code SourcePath} yields the same IncludePath. */ @VisibleForTesting static ImmutableSortedMap<String, SourcePath> convertToFlatCxxHeaders(Path headerPathPrefix, Function<SourcePath, Path> sourcePathResolver, Set<SourcePath> headerPaths) { Set<String> includeToFile = new HashSet<String>(headerPaths.size()); ImmutableSortedMap.Builder<String, SourcePath> builder = ImmutableSortedMap.naturalOrder(); for (SourcePath headerPath : headerPaths) { Path fileName = sourcePathResolver.apply(headerPath).getFileName(); String key = headerPathPrefix.resolve(fileName).toString(); if (includeToFile.contains(key)) { ImmutableSortedMap<String, SourcePath> result = builder.build(); throw new HumanReadableException( "The same include path maps to multiple files:\n" + " Include path: %s\n" + " Conflicting files:\n" + " %s\n" + " %s", key, headerPath, result.get(key)); } includeToFile.add(key); builder.put(key, headerPath); } return builder.build(); }
From source file:org.gradle.api.internal.changedetection.changes.Util.java
public static ImmutableSortedMap<String, CurrentFileCollectionFingerprint> fingerprintAfterOutputsGenerated( final @Nullable ImmutableSortedMap<String, FileCollectionFingerprint> previous, ImmutableSortedMap<String, CurrentFileCollectionFingerprint> current, SortedSet<? extends TaskFilePropertySpec> outputProperties, boolean hasOverlappingOutputs, TaskInternal task, FileCollectionFingerprinterRegistry fingerprinterRegistry) { final ImmutableSortedMap<String, CurrentFileCollectionFingerprint> outputFilesAfter = fingerprintTaskFiles( task, outputProperties, fingerprinterRegistry); if (!hasOverlappingOutputs) { return outputFilesAfter; } else {//from w ww .ja v a2s . co m return ImmutableSortedMap.copyOfSorted(Maps.transformEntries(current, new Maps.EntryTransformer<String, CurrentFileCollectionFingerprint, CurrentFileCollectionFingerprint>() { @Override @SuppressWarnings("NullableProblems") public CurrentFileCollectionFingerprint transformEntry(String propertyName, CurrentFileCollectionFingerprint beforeExecution) { CurrentFileCollectionFingerprint afterExecution = outputFilesAfter.get(propertyName); FileCollectionFingerprint afterPreviousExecution = Util .getFingerprintAfterPreviousExecution(previous, propertyName); return Util.filterOutputFingerprint(afterPreviousExecution, beforeExecution, afterExecution); } })); } }
From source file:com.amazonaws.services.kinesis.stormspout.KinesisShardGetterBuilder.java
@Override public ImmutableList<IShardGetter> buildGetters(ImmutableList<String> shardAssignment) { ImmutableList.Builder<IShardGetter> builder = new ImmutableList.Builder<>(); ImmutableSortedMap<String, ShardInfo> shards = helper.getShardList(); for (String shardId : shardAssignment) { KinesisShardGetter getter = new KinesisShardGetter(streamName, shards.get(shardId), helper.getSharedkinesisClient()); builder.add(new BufferedGetter(getter, maxRecordsPerCall, emptyRecordListBackoffMillis)); }// w ww. j a v a 2 s.c o m return builder.build(); }
From source file:org.geogit.repository.DepthSearch.java
/** * @param parent/*from www.j av a 2 s. c o m*/ * @param directChildName * @return */ public Optional<Node> getDirectChild(RevTree parent, String directChildName, final int subtreesDepth) { if (parent.isEmpty()) { return Optional.absent(); } if (parent.trees().isPresent() || parent.features().isPresent()) { if (parent.trees().isPresent()) { ImmutableList<Node> refs = parent.trees().get(); for (int i = 0; i < refs.size(); i++) { if (directChildName.equals(refs.get(i).getName())) { return Optional.of(refs.get(i)); } } } if (parent.features().isPresent()) { ImmutableList<Node> refs = parent.features().get(); for (int i = 0; i < refs.size(); i++) { if (directChildName.equals(refs.get(i).getName())) { return Optional.of(refs.get(i)); } } } return Optional.absent(); } Integer bucket = refOrder.bucket(directChildName, subtreesDepth); ImmutableSortedMap<Integer, Bucket> buckets = parent.buckets().get(); Bucket subtreeBucket = buckets.get(bucket); if (subtreeBucket == null) { return Optional.absent(); } RevTree subtree = objectDb.get(subtreeBucket.id(), RevTree.class); return getDirectChild(subtree, directChildName, subtreesDepth + 1); }