Example usage for com.google.common.hash Hashing murmur3_128

Introduction

In this page you can find the example usage for com.google.common.hash Hashing murmur3_128.

Prototype

public static HashFunction murmur3_128() 

Source Link

Document

Returns a hash function implementing the <a href="http://smhasher.googlecode.com/svn/trunk/MurmurHash3.cpp"> 128-bit murmur3 algorithm, x64 variant</a> (little-endian variant), using a seed value of zero.

Usage

From source file:org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.java

@Override
protected void setup(Context context) throws IOException {
    super.setup(context);
    tmpbuf = ByteBuffer.allocate(4096);
    collectStatistics = Boolean//from   w w w . j  a v a  2  s.co m
            .parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
    if (collectStatistics) {
        samplingPercentage = Integer
                .parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
        cuboidScheduler = new CuboidScheduler(cubeDesc);
        nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;

        List<Long> cuboidIdList = Lists.newArrayList();
        List<Integer[]> allCuboidsBitSetList = Lists.newArrayList();
        addCuboidBitSet(baseCuboidId, allCuboidsBitSetList, cuboidIdList);

        allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[cuboidIdList.size()][]);
        cuboidIds = cuboidIdList.toArray(new Long[cuboidIdList.size()]);

        allCuboidsHLL = new HLLCounter[cuboidIds.length];
        for (int i = 0; i < cuboidIds.length; i++) {
            allCuboidsHLL[i] = new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision(),
                    RegisterType.DENSE);
        }

        TblColRef partitionColRef = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
        if (partitionColRef != null) {
            partitionColumnIndex = intermediateTableDesc.getColumnIndex(partitionColRef);
        }

        // check whether need fetch the partition col values
        if (partitionColumnIndex < 0) {
            // if partition col not on cube, no need
            needFetchPartitionCol = false;
        } else {
            needFetchPartitionCol = true;
        }
        //for KYLIN-2518 backward compatibility
        if (KylinVersion.isBefore200(cubeDesc.getVersion())) {
            isUsePutRowKeyToHllNewAlgorithm = false;
            row_hashcodes = new ByteArray[nRowKey];
            for (int i = 0; i < nRowKey; i++) {
                row_hashcodes[i] = new ByteArray();
            }
            hf = Hashing.murmur3_32();
            logger.info("Found KylinVersion : {}. Use old algorithm for cuboid sampling.",
                    cubeDesc.getVersion());
        } else {
            isUsePutRowKeyToHllNewAlgorithm = true;
            rowHashCodesLong = new long[nRowKey];
            hf = Hashing.murmur3_128();
            logger.info(
                    "Found KylinVersion : {}. Use new algorithm for cuboid sampling. About the details of the new algorithm, please refer to KYLIN-2518",
                    cubeDesc.getVersion());
        }
    }

}

---

From source file:org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.java

static String createJobId(String prefix, TableDestination tableDestination, int partition) {
    // Job ID must be different for each partition of each table.
    String destinationHash = Hashing.murmur3_128().hashUnencodedChars(tableDestination.toString()).toString();
    if (partition >= 0) {
        return String.format("%s_%s_%05d", prefix, destinationHash, partition);
    } else {/*from  w w w . j a  va 2s .c o m*/
        return String.format("%s_%s", prefix, destinationHash);
    }
}

---

From source file:org.apache.beam.sdk.io.PubsubGrpcClient.java

@Override
public int publish(TopicPath topic, Iterable<OutgoingMessage> outgoingMessages) throws IOException {
    PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(topic.getPath());
    for (OutgoingMessage outgoingMessage : outgoingMessages) {
        PubsubMessage.Builder message = PubsubMessage.newBuilder()
                .setData(ByteString.copyFrom(outgoingMessage.elementBytes));

        if (timestampLabel != null) {
            message.getMutableAttributes().put(timestampLabel,
                    String.valueOf(outgoingMessage.timestampMsSinceEpoch));
        }/*  w  ww  .j  av  a  2 s . c o m*/

        if (idLabel != null) {
            message.getMutableAttributes().put(idLabel,
                    Hashing.murmur3_128().hashBytes(outgoingMessage.elementBytes).toString());
        }

        request.addMessages(message);
    }

    PublishResponse response = publisherStub().publish(request.build());
    return response.getMessageIdsCount();
}

---

From source file:io.soabase.admin.details.IndexServlet.java

private void doReplacements(ComponentManager componentManager, Map<String, Entry> files) throws IOException {
    int currentYear = Calendar.getInstance().get(Calendar.YEAR);

    String firstId = "";
    StringBuilder tabsBuilder = new StringBuilder();
    StringBuilder idsBuilder = new StringBuilder();
    StringBuilder cssBuilder = new StringBuilder();
    StringBuilder jsBuilder = new StringBuilder();
    StringBuilder tabContentBuilder = new StringBuilder();
    StringBuilder metricsBuilder = new StringBuilder();

    for (TabComponent tab : componentManager.getTabs()) {
        if (firstId.length() == 0) {
            firstId = tab.getId();//from   w  w w.  j a v  a2  s.co m
        }

        String id = SOA_TAB_PREFIX + tab.getId();
        tabsBuilder.append("<li id='").append(id).append("-li").append("'><a href=\"#").append(tab.getId())
                .append("\">").append(tab.getName()).append("</a></li>\n");
        idsBuilder.append("soaTabIds.push('").append(id).append("');\n");

        for (String cssFile : tab.getCssUris()) {
            cssBuilder.append("<link rel=\"stylesheet\" href=\"").append(cssFile).append("\">\n");
        }
        for (String jssFile : tab.getJavascriptUris()) {
            jsBuilder.append("<script src=\"").append(jssFile).append("\"></script>\n");
        }

        String tabContent = Resources.toString(Resources.getResource(tab.getContentResourcePath()),
                Charsets.UTF_8);
        tabContentBuilder.append("<div class=\"soa-hidden\" id=\"" + SOA_TAB_PREFIX).append(tab.getId())
                .append("\">").append(tabContent).append("</div>\n");
    }

    for (MetricComponent metric : componentManager.getMetrics()) {
        String obj = mapper.writeValueAsString(metric);
        metricsBuilder.append("vmMetrics.push(").append(obj).append(");\n");
    }

    String tabs = tabsBuilder.toString();
    String metrics = metricsBuilder.toString();
    String tabContent = tabContentBuilder.toString();
    String ids = idsBuilder.toString();
    String css = cssBuilder.toString();
    String js = jsBuilder.toString();

    for (Mapping mapping : mappings) {
        Entry entry = files.get(mapping.key);
        String content = entry.content;

        content = content.replace("$SOA_TABS$", tabs);
        content = content.replace("$SOA_TABS_CONTENT$", tabContent);
        content = content.replace("$SOA_METRICS$", metrics);
        content = content.replace("$SOA_DEFAULT_TAB_ID$", firstId);
        content = content.replace("$SOA_TAB_IDS$", ids);
        content = content.replace("$SOA_CSS$", css);
        content = content.replace("$SOA_JS$", js);
        content = content.replace("$SOA_NAME$", componentManager.getAppName());
        content = content.replace("$SOA_COPYRIGHT$",
                "" + currentYear + " " + componentManager.getCompanyName());
        content = content.replace("$SOA_FOOTER_MESSAGE$", "" + componentManager.getFooterMessage());

        String eTag = '"' + Hashing.murmur3_128().hashBytes(content.getBytes()).toString() + '"';
        files.put(mapping.key, new Entry(content, eTag));
    }
}

---

From source file:com.cinchapi.concourse.server.storage.cache.LoggingBloomFilter.java

/**
 * Return the hashes for the {@code composite} object based on the ideal
 * {@link #numHashFunctions} and the size of the underlying {@link #bitSet}.
 * //from  w w  w.  j a  va 2 s  .  c om
 * @param composite
 * @return the hashes for {@code composite}.
 */
private int[] hash(Composite composite) {
    long hash64 = Hashing.murmur3_128().hashBytes(ByteBuffers.toByteArray(composite.getBytes())).asLong();
    int hash1 = (int) hash64;
    int hash2 = (int) (hash64 >>> 32);
    int[] hashes = new int[numHashFunctions];
    for (int i = 1; i <= numHashFunctions; ++i) {
        hashes[i - 1] = Math.abs((hash1 + i * hash2) % numBits);
    }
    return hashes;
}

---

From source file:fr.inria.eventcloud.api.Quadruple.java

/**
 * Returns a 128 bits hash value for the current quadruple.
 * /*from  w  ww. jav a 2 s .c  o m*/
 * @return a 128 bits hash value for the current quadruple.
 */
public HashCode hashValue() {
    Hasher hasher = Hashing.murmur3_128().newHasher();

    for (int i = 0; i < this.nodes.length; i++) {
        hasher.putString(this.nodes[i].toString(), Charsets.UTF_8);
    }

    if (this.publicationSource != null) {
        hasher.putUnencodedChars(this.publicationSource);
    }
    hasher.putLong(this.publicationTime);

    return hasher.hash();
}

---

From source file:com.google.javascript.jscomp.RandomNameGenerator.java

/**
 * Returns the {@code nameIdx}-th short name. This might be a reserved name.
 * A user-requested prefix is not included, but the first returned character
 * is supposed to go at position {@code position} in the final name
 *//*from w w w  . java 2 s  .  c o m*/
private String generateSuffix(int position, int nameIdx) {
    StringBuilder name = new StringBuilder();
    int length = getNameLength(position, nameIdx);
    nameIdx++;
    do {
        nameIdx--;
        String alphabet;
        if (position == 0) {
            alphabet = shuffledFirst;
        } else {
            Hasher hasher = Hashing.murmur3_128().newHasher();
            hasher.putInt(length);
            hasher.putUnencodedChars(name);
            int alphabetIdx = (hasher.hash().asInt() & 0x7fffffff) % NUM_SHUFFLES;
            alphabet = shuffledNonFirst.get(alphabetIdx);
        }
        int alphabetSize = alphabet.length();
        char character = alphabet.charAt(nameIdx % alphabetSize);
        name.append(character);

        nameIdx /= alphabetSize;
        position++;
    } while (nameIdx > 0);
    return name.toString();
}

---

From source file:org.syncany.plugins.transfer.features.PathAwareFeatureTransferManager.java

/**
 * Returns the subpath identifier for this file. For {@link MultichunkRemoteFile}s, this is the
 * hex string of the multichunk identifier. For all other files, this is the 128-bit murmur3 hash
 * of the full filename (fast algorithm!).
 *//*from   www .j a  va2s .c om*/
private String getSubPathId(RemoteFile remoteFile) {
    if (remoteFile.getClass() == MultichunkRemoteFile.class) {
        return StringUtil.toHex(((MultichunkRemoteFile) remoteFile).getMultiChunkId());
    } else {
        return StringUtil
                .toHex(Hashing.murmur3_128().hashString(remoteFile.getName(), Charsets.UTF_8).asBytes());
    }
}

---

From source file:com.facebook.buck.cxx.PreprocessorDelegate.java

public String hashCommand(ImmutableList<String> flags) {
    Hasher hasher = Hashing.murmur3_128().newHasher();
    String workingDirString = workingDir.toString();
    // Skips the executable argument (the first one) as that is not sanitized.
    for (String part : sanitizer.sanitizeFlags(Iterables.skip(flags, 1))) {
        // TODO(#10251354): find a better way of dealing with getting a project dir normalized hash
        if (part.startsWith(workingDirString)) {
            part = "<WORKINGDIR>" + part.substring(workingDirString.length());
        }//w ww.  j a v a  2  s.  co m
        hasher.putString(part, Charsets.UTF_8);
        hasher.putBoolean(false); // separator
    }
    return hasher.hash().toString();
}

---

From source file:org.dcache.chimera.nfsv41.door.MonitoringVfs.java

@Override
public boolean move(Inode src, String oldName, Inode dest, String newName) throws IOException {
    boolean changed = super.move(src, oldName, dest, newName);

    if (changed) {
        try {//from   ww w . j  a  va 2 s .  co  m
            Inode targetInode = super.lookup(dest, newName);
            FileType type = asFileType(super.getattr(targetInode).type());
            long generation = super.getattr(dest).getGeneration();
            Optional<PnfsId> srcParent = toPnfsId(src);
            Optional<PnfsId> destParent = toPnfsId(dest);
            Optional<PnfsId> target = toPnfsId(targetInode);

            // The cookie is a unique identifier for this move operation.  It
            // is completely unrelated to the NFS-protocol cookie for
            // directory listing
            Hasher hasher = Hashing.murmur3_128().newHasher().putLong(generation);
            destParent.ifPresent(id -> hasher.putString(id.toString(), StandardCharsets.UTF_8));
            String cookie = BaseEncoding.base64().omitPadding().encode(hasher.hash().asBytes());

            srcParent.ifPresent(id -> eventReceiver.notifyMovedEvent(IN_MOVED_FROM, id, oldName, cookie, type));
            destParent.ifPresent(id -> eventReceiver.notifyMovedEvent(IN_MOVED_TO, id, newName, cookie, type));
            target.ifPresent(id -> eventReceiver.notifySelfEvent(IN_MOVE_SELF, id, type));
        } catch (IOException e) {
            LOGGER.error("Failed establishing move of {} to {} in dir {}: {}", oldName, newName, dest,
                    e.toString());
        }
    }

    return changed;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »