List of usage examples for com.google.common.hash Hashing murmur3_128
public static HashFunction murmur3_128()
From source file:org.apache.kylin.engine.spark.SparkCubing.java
private Map<Long, HLLCounter> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName, String segmentId) throws Exception { CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()) .reloadCubeLocal(cubeName);/*from w ww . j a v a 2 s.co m*/ CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); CubeDesc cubeDesc = cubeInstance.getDescriptor(); CuboidScheduler cuboidScheduler = new CuboidScheduler(cubeDesc); List<Long> allCuboidIds = cuboidScheduler.getAllCuboidIds(); final HashMap<Long, HLLCounter> zeroValue = Maps.newHashMap(); for (Long id : allCuboidIds) { zeroValue.put(id, new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision())); } CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich( EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); final int[] rowKeyColumnIndexes = flatDesc.getRowKeyColumnIndexes(); final int nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length; final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMapWithExpectedSize(allCuboidIds.size()); final ByteArray[] row_hashcodes = new ByteArray[nRowKey]; for (Long cuboidId : allCuboidIds) { Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)]; long mask = Long.highestOneBit(baseCuboidId); int position = 0; for (int i = 0; i < nRowKey; i++) { if ((mask & cuboidId) > 0) { cuboidBitSet[position] = i; position++; } mask = mask >> 1; } allCuboidsBitSet.put(cuboidId, cuboidBitSet); } for (int i = 0; i < nRowKey; ++i) { row_hashcodes[i] = new ByteArray(); } final HashMap<Long, HLLCounter> samplingResult = rowJavaRDD.aggregate(zeroValue, new Function2<HashMap<Long, HLLCounter>, List<String>, HashMap<Long, HLLCounter>>() { final HashFunction hashFunction = Hashing.murmur3_128(); @Override public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, List<String> v2) throws Exception { for (int i = 0; i < nRowKey; i++) { Hasher hc = hashFunction.newHasher(); String colValue = v2.get(rowKeyColumnIndexes[i]); if (colValue != null) { row_hashcodes[i].set(hc.putString(colValue).hash().asBytes()); } else { row_hashcodes[i].set(hc.putInt(0).hash().asBytes()); } } for (Map.Entry<Long, Integer[]> entry : allCuboidsBitSet.entrySet()) { Hasher hc = hashFunction.newHasher(); HLLCounter counter = v1.get(entry.getKey()); final Integer[] cuboidBitSet = entry.getValue(); for (int position = 0; position < cuboidBitSet.length; position++) { hc.putBytes(row_hashcodes[cuboidBitSet[position]].array()); } counter.add(hc.hash().asBytes()); } return v1; } }, new Function2<HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>>() { @Override public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, HashMap<Long, HLLCounter> v2) throws Exception { Preconditions.checkArgument(v1.size() == v2.size()); Preconditions.checkArgument(v1.size() > 0); for (Map.Entry<Long, HLLCounter> entry : v1.entrySet()) { final HLLCounter counter1 = entry.getValue(); final HLLCounter counter2 = v2.get(entry.getKey()); counter1.merge(Preconditions.checkNotNull(counter2, "counter cannot be null")); } return v1; } }); return samplingResult; }
From source file:org.apache.giraph.block_app.library.prepare_graph.PrepareGraphPieces.java
/** * isSymmetricBlock using a sensible default HashFunction * * @see Hashing#murmur3_128()/*from w w w.jav a 2 s .co m*/ * @see #isSymmetricBlock(Funnel, Consumer, HashFunction) */ public static <I extends WritableComparable> Block isSymmetricBlock(Funnel<I> idHasher, Consumer<Boolean> consumer) { return isSymmetricBlock(idHasher, consumer, Hashing.murmur3_128()); }
From source file:org.apache.beam.sdk.nexmark.NexmarkUtils.java
/** * Return a transform to reduce a stream to a single, order-invariant long hash. *//*from w w w .ja va2 s .com*/ public static <T> PTransform<PCollection<T>, PCollection<Long>> hash(final long numEvents, String name) { return new PTransform<PCollection<T>, PCollection<Long>>(name) { @Override public PCollection<Long> expand(PCollection<T> input) { return input .apply(Window.<T>into(new GlobalWindows()) .triggering(AfterPane.elementCountAtLeast((int) numEvents)) .withAllowedLateness(Duration.standardDays(1)).discardingFiredPanes()) .apply(name + ".Hash", ParDo.of(new DoFn<T, Long>() { @ProcessElement public void processElement(ProcessContext c) { long hash = Hashing.murmur3_128().newHasher().putLong(c.timestamp().getMillis()) .putString(c.element().toString(), StandardCharsets.UTF_8).hash().asLong(); c.output(hash); } })) .apply(Combine.globally(new Combine.BinaryCombineFn<Long>() { @Override public Long apply(Long left, Long right) { return left ^ right; } })); } }; }
From source file:org.apache.beam.sdk.nexmark.NexmarkUtils.java
/** * Return a transform to keep the CPU busy for given milliseconds on every record. *///from w w w .j av a 2 s. com public static <T> ParDo.SingleOutput<T, T> cpuDelay(String name, final long delayMs) { return ParDo.of(new DoFn<T, T>() { @ProcessElement public void processElement(ProcessContext c) { long now = System.currentTimeMillis(); long end = now + delayMs; while (now < end) { // Find plaintext which hashes to HASH in lowest MASK bits. // Values chosen to roughly take 1ms on typical workstation. long p = INIT_PLAINTEXT; while (true) { long t = Hashing.murmur3_128().hashLong(p).asLong(); if ((t & MASK) == (HASH & MASK)) { break; } p++; } now = System.currentTimeMillis(); } c.output(c.element()); } }); }
From source file:com.caspida.plugins.graph.EntityGraphNeo4jPlugin.java
public static Long hash(String text) { HashCode hc = Hashing.murmur3_128().hashBytes(text.getBytes()); return hc.asLong(); }
From source file:org.eclipse.epp.internal.logging.aeri.ui.model.Reports.java
public static String exactIdentityHash(ErrorReport report) { final Hasher hasher = Hashing.murmur3_128().newHasher(); ModelSwitch<Hasher> s = new ModelSwitch<Hasher>() { @Override/*from w w w . jav a 2 s . c o m*/ public Hasher caseErrorReport(ErrorReport object) { hasher.putString(stripToEmpty(object.getEclipseProduct()), UTF_8); hasher.putString(stripToEmpty(object.getEclipseBuildId()), UTF_8); hasher.putString(stripToEmpty(object.getJavaRuntimeVersion()), UTF_8); hasher.putString(stripToEmpty(object.getOsgiOs()), UTF_8); hasher.putString(stripToEmpty(object.getOsgiOsVersion()), UTF_8); hasher.putString(stripToEmpty(object.getOsgiArch()), UTF_8); hasher.putString(stripToEmpty(object.getOsgiWs()), UTF_8); return null; }; @Override public Hasher caseStatus(Status object) { hasher.putString(stripToEmpty(object.getPluginId()), UTF_8); hasher.putString(stripToEmpty(object.getPluginVersion()), UTF_8); hasher.putString(stripToEmpty(object.getMessage()), UTF_8); hasher.putInt(object.getSeverity()); hasher.putInt(object.getCode()); return null; } @Override public Hasher caseBundle(org.eclipse.epp.internal.logging.aeri.ui.model.Bundle object) { hasher.putString(stripToEmpty(object.getName()), UTF_8); hasher.putString(stripToEmpty(object.getVersion()), UTF_8); return null; } @Override public Hasher caseStackTraceElement(StackTraceElement object) { hasher.putString(stripToEmpty(object.getClassName()), UTF_8); hasher.putString(stripToEmpty(object.getMethodName()), UTF_8); hasher.putInt(object.getLineNumber()); return null; } @Override public Hasher caseThrowable(Throwable object) { hasher.putString(stripToEmpty(object.getClassName()), UTF_8); hasher.putString(stripToEmpty(object.getMessage()), UTF_8); return null; } }; visit(report, s); String hash = hasher.hash().toString(); return hash; }
From source file:org.eclipse.epp.internal.logging.aeri.ui.model.Reports.java
public static String traceIdentityHash(ErrorReport report) { final Hasher hasher = Hashing.murmur3_128().newHasher(); visit(report, new ModelSwitch<Hasher>() { @Override/*from www. ja va 2 s. c o m*/ public Hasher caseStackTraceElement(StackTraceElement element) { hasher.putString(element.getClassName(), UTF_8); hasher.putString(element.getMethodName(), UTF_8); hasher.putInt(element.getLineNumber()); return null; } }); String hash = hasher.hash().toString(); return hash; }
From source file:com.palantir.atlasdb.keyvalue.jdbc.JdbcKeyValueService.java
String hashTableName(String rawName) {
// Take 40 bits from the raw name
byte[] bytes = Hashing.murmur3_128().hashString(rawName, StandardCharsets.UTF_8).asBytes();
return BaseEncoding.base32().omitPadding().encode(bytes);
}