List of usage examples for com.google.common.hash Hashing md5
public static HashFunction md5()
From source file:org.jclouds.examples.blobstore.BlobUploader.java
@Override public void run() { /**//from w w w. j a v a 2 s . c om * Instantiate the ThreadLocal variables when this thread runs for the first time. * Instantiating this in the constructor will not work (different thread). */ if (blobStore.get() == null) { // It is usually a good idea to include the currentThread when logging parallel tasks. System.out.println("Creating connection for thread " + Thread.currentThread()); /** * In some cases, especially when running very large jobs with many parallel threads, some connections will * break. In that case, we need to be able to obtain a new connection (and socket) to the service, which is * why this is factored out. */ resetBlobstore(username, password, provider, region); } if (container.get() == null) { container.set(UUID.randomUUID().toString()); Location location = getOnlyElement(blobStore.get().listAssignableLocations()); blobStore.get().createContainerInLocation(location, container.get()); System.out.println("Created container " + container.get() + " for thread " + Thread.currentThread() + " in " + location.toString()); } // The md5 as returned by the service, and as calculated locally. String md5Local; String md5Remote; Blob blob; try { md5Local = BaseEncoding.base16().encode(Files.hash(file, Hashing.md5()).asBytes()).toLowerCase(); } catch (java.io.IOException e) { e.printStackTrace(); /** * The file is no longer available on the local FS. * In some application cases, you might also want to retry this instead of finishing the unit of work. */ return; } ByteSourcePayload bsp = new ByteSourcePayload(Files.asByteSource(file)); /** * Uploading a file over a network is an inherently fragile operation. Over thousands of files, especially in * highly parallel jobs that tax upload bandwidth, a small percent of uploads are guaranteed to fail. */ do { System.out.println("Uploading " + file.getName() + " ; " + FileUtils.sizeOf(file)); blob = blobStore.get().blobBuilder(file.getName()).payload(bsp).build(); md5Remote = blobStore.get().putBlob(container.get(), blob).toLowerCase(); if (md5Local.equals(md5Remote)) { long total = BlobUploaderMain.bytesUploaded.addAndGet(FileUtils.sizeOf(file)); System.out.println("Uploaded MB: " + (int) total / FileUtils.ONE_MB + "MB ; " + (int) ((float) BlobUploaderMain.bytesUploaded.get() / BlobUploaderMain.totalBytes) * 100 + "%"); bsp.release(); return; } else { System.out.printf("md5 mismatch %s vs %s, retrying %s", md5Local, md5Remote, file.getName()); } } while (true); }
From source file:com.google.devtools.build.lib.collect.nestedset.NestedSetCodec.java
private void serializeOneNestedSet(Object children, CodedOutputStream codedOut, Map<Object, byte[]> childToDigest) throws IOException, SerializationException { // Serialize nested set into an inner byte array so we can take its digest ByteArrayOutputStream childOutputStream = new ByteArrayOutputStream(); HashingOutputStream hashingOutputStream = new HashingOutputStream(Hashing.md5(), childOutputStream); CodedOutputStream childCodedOut = CodedOutputStream.newInstance(hashingOutputStream); if (children instanceof Object[]) { serializeMultiItemChildArray((Object[]) children, childToDigest, childCodedOut); } else if (children != NestedSet.EMPTY_CHILDREN) { serializeSingleItemChildArray(children, childCodedOut); } else {//from ww w .ja v a 2 s . com // Empty set childCodedOut.writeInt32NoTag(0); } childCodedOut.flush(); byte[] digest = hashingOutputStream.hash().asBytes(); codedOut.writeByteArrayNoTag(digest); byte[] childBytes = childOutputStream.toByteArray(); codedOut.writeByteArrayNoTag(childBytes); childToDigest.put(children, digest); }
From source file:io.prestosql.operator.scalar.RowToRowCast.java
private static Class<?> generateRowCast(Type fromType, Type toType, FunctionRegistry functionRegistry) { List<Type> toTypes = toType.getTypeParameters(); List<Type> fromTypes = fromType.getTypeParameters(); CallSiteBinder binder = new CallSiteBinder(); // Embed the MD5 hash code of input and output types into the generated class name instead of the raw type names, // which could prevent the class name from hitting the length limitation and invalid characters. byte[] md5Suffix = Hashing.md5().hashBytes((fromType + "$" + toType).getBytes()).asBytes(); ClassDefinition definition = new ClassDefinition(a(PUBLIC, FINAL), makeClassName(Joiner.on("$").join("RowCast", BaseEncoding.base16().encode(md5Suffix))), type(Object.class)); Parameter session = arg("session", ConnectorSession.class); Parameter value = arg("value", Block.class); MethodDefinition method = definition.declareMethod(a(PUBLIC, STATIC), "castRow", type(Block.class), session, value);/*from ww w. j a v a2 s . c o m*/ Scope scope = method.getScope(); BytecodeBlock body = method.getBody(); Variable wasNull = scope.declareVariable(boolean.class, "wasNull"); Variable blockBuilder = scope.createTempVariable(BlockBuilder.class); Variable singleRowBlockWriter = scope.createTempVariable(BlockBuilder.class); body.append(wasNull.set(constantBoolean(false))); CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(definition, binder); // create the row block builder body.append(blockBuilder.set(constantType(binder, toType).invoke("createBlockBuilder", BlockBuilder.class, constantNull(BlockBuilderStatus.class), constantInt(1)))); body.append(singleRowBlockWriter.set(blockBuilder.invoke("beginBlockEntry", BlockBuilder.class))); // loop through to append member blocks for (int i = 0; i < toTypes.size(); i++) { Signature signature = internalOperator(CAST.name(), toTypes.get(i).getTypeSignature(), ImmutableList.of(fromTypes.get(i).getTypeSignature())); ScalarFunctionImplementation function = functionRegistry.getScalarFunctionImplementation(signature); Type currentFromType = fromTypes.get(i); if (currentFromType.equals(UNKNOWN)) { body.append(singleRowBlockWriter.invoke("appendNull", BlockBuilder.class).pop()); continue; } BytecodeExpression fromElement = constantType(binder, currentFromType).getValue(value, constantInt(i)); BytecodeExpression toElement = invokeFunction(scope, cachedInstanceBinder, signature.getName(), function, fromElement); IfStatement ifElementNull = new IfStatement("if the element in the row type is null..."); ifElementNull.condition(value.invoke("isNull", boolean.class, constantInt(i))) .ifTrue(singleRowBlockWriter.invoke("appendNull", BlockBuilder.class).pop()) .ifFalse(constantType(binder, toTypes.get(i)).writeValue(singleRowBlockWriter, toElement)); body.append(ifElementNull); } // call blockBuilder.closeEntry() and return the single row block body.append(blockBuilder.invoke("closeEntry", BlockBuilder.class).pop()); body.append(constantType(binder, toType) .invoke("getObject", Object.class, blockBuilder.cast(Block.class), constantInt(0)).cast(Block.class) .ret()); // create constructor MethodDefinition constructorDefinition = definition.declareConstructor(a(PUBLIC)); BytecodeBlock constructorBody = constructorDefinition.getBody(); Variable thisVariable = constructorDefinition.getThis(); constructorBody.comment("super();").append(thisVariable).invokeConstructor(Object.class); cachedInstanceBinder.generateInitializations(thisVariable, constructorBody); constructorBody.ret(); return defineClass(definition, Object.class, binder.getBindings(), RowToRowCast.class.getClassLoader()); }
From source file:com.github.acquized.retile.utils.Dump.java
public static Dump create() throws IOException, RetileAPIException { List<SubServer> servers = new ArrayList<>(); for (Map.Entry<String, ServerInfo> entry : ProxyServer.getInstance().getServers().entrySet()) { servers.add(new SubServer(entry.getKey(), entry.getValue().getAddress().toString())); }/*w w w. j av a 2 s .c o m*/ Map<String, PluginInfo> plugins = new HashMap<>(); for (Plugin p : ProxyServer.getInstance().getPluginManager().getPlugins()) { if (!p.getDescription().getAuthor().equalsIgnoreCase("SpigotMC")) { plugins.put(p.getDescription().getName(), new PluginInfo(p.getDescription().getVersion(), p.getDescription().getMain(), p.getDescription().getAuthor(), Files.hash(p.getDescription().getFile(), Hashing.md5()).toString())); } } Map<UUID, String> cache = new HashMap<>(); if (ProjectRetile.getInstance().getCache() instanceof McAPICanada) { cache = ((McAPICanada) ProjectRetile.getInstance().getCache()).getCache().asMap(); } else if (ProjectRetile.getInstance().getCache() instanceof Mojang) { cache = ((Mojang) ProjectRetile.getInstance().getCache()).getCache().asMap(); } else { cache.put(UUID.randomUUID(), "Empty"); } try { return new Dump( // Retile Plugin Information new RetilePlugin(ProjectRetile.getInstance().getDescription().getName(), ProjectRetile.getInstance().getDescription().getVersion(), ProjectRetile.getInstance().getDescription().getAuthor(), ProjectRetile.getInstance().getDescription().getMain(), Files.hash(ProjectRetile.getInstance().getDescription().getFile(), Hashing.md5()) .toString()), // Server Information new Server(ProxyServer.getInstance().getName(), ProxyServer.getInstance().getVersion()), // Servers available thought BungeeCord servers, // Machine on which BungeeCord is running new Machine(System.getProperty("java.version"), System.getProperty("os.name"), Utility.convertToReadableString(Runtime.getRuntime().freeMemory()), Utility.convertToReadableString(Runtime.getRuntime().maxMemory()), Utility.convertToReadableString(Runtime.getRuntime().totalMemory())), // Configuration ProjectRetile.getInstance().getConfig().to(Config.class), // Database Configuration ProjectRetile.getInstance().getDbConfig().to(DBConfig.class), // Blacklist Joiner.on(", ").join(ProjectRetile.getInstance().getBlacklist().getList("blacklist")), // Plugins plugins, // Database new Database(ProjectRetile.getInstance().getDatabase().isConnected(), ProjectRetile.getInstance().getDatabase().getClass().getSimpleName().replace(".class", ""), ProjectRetile.getInstance().getDatabase().doesTableExist("retile"), ProjectRetile.getInstance().getDatabase().doesTableExist("queue"), ProjectRetile.getInstance().getApi().getAllReports().length, ProjectRetile.getInstance().getApi().getWaitingReports().length), // Cache new Cache( ProjectRetile.getInstance().getCache().getClass().getSimpleName().replace(".class", ""), cache) ); } catch (SQLException | RetileAPIException ex) { throw new RetileAPIException("Could not create Dump", ex); } }
From source file:com.dangdang.ddframe.job.cloud.state.failover.FailoverService.java
private List<Integer> getAssignedShardingItems(final String jobName, final List<String> taskMetaInfoList, final Set<HashCode> assignedTasks) { List<Integer> result = new ArrayList<>(taskMetaInfoList.size()); for (String each : taskMetaInfoList) { TaskContext.MetaInfo metaInfo = TaskContext.MetaInfo.from(each); if (assignedTasks .add(Hashing.md5().newHasher().putString(jobName, Charsets.UTF_8) .putInt(metaInfo.getShardingItem()).hash()) && !runningService.isTaskRunning(metaInfo)) { result.add(metaInfo.getShardingItem()); }/*from w w w . j a va 2s . c o m*/ } return result; }
From source file:org.jclouds.digitalocean.ssh.DSAKeys.java
/** * Create a fingerprint per the following <a * href="http://tools.ietf.org/html/draft-friedl-secsh-fingerprint-00" * >spec</a>//from www . j a v a 2s. c o m * * @return hex fingerprint ex. * {@code 2b:a9:62:95:5b:8b:1d:61:e0:92:f7:03:10:e9:db:d9} */ public static String fingerprint(BigInteger p, BigInteger q, BigInteger g, BigInteger y) { byte[] keyBlob = keyBlob(p, q, g, y); return hexColonDelimited(Hashing.md5().hashBytes(keyBlob)); }
From source file:com.facebook.presto.operator.scalar.RowToRowCast.java
private static Class<?> generateRowCast(Type fromType, Type toType, FunctionRegistry functionRegistry) { List<Type> toTypes = toType.getTypeParameters(); List<Type> fromTypes = fromType.getTypeParameters(); CallSiteBinder binder = new CallSiteBinder(); // Embed the MD5 hash code of input and output types into the generated class name instead of the raw type names, // which could prevent the class name from hitting the length limitation and invalid characters. byte[] md5Suffix = Hashing.md5().hashBytes((fromType + "$" + toType).getBytes()).asBytes(); ClassDefinition definition = new ClassDefinition(a(PUBLIC, FINAL), CompilerUtils/*w w w . j a v a 2 s .c o m*/ .makeClassName(Joiner.on("$").join("RowCast", BaseEncoding.base16().encode(md5Suffix))), type(Object.class)); Parameter session = arg("session", ConnectorSession.class); Parameter value = arg("value", Block.class); MethodDefinition method = definition.declareMethod(a(PUBLIC, STATIC), "castRow", type(Block.class), session, value); Scope scope = method.getScope(); BytecodeBlock body = method.getBody(); Variable wasNull = scope.declareVariable(boolean.class, "wasNull"); Variable blockBuilder = scope.createTempVariable(BlockBuilder.class); body.append(wasNull.set(constantBoolean(false))); CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(definition, binder); // create the interleave block builder body.newObject(InterleavedBlockBuilder.class).dup() .append(constantType(binder, toType).invoke("getTypeParameters", List.class)) .append(newInstance(BlockBuilderStatus.class)).append(constantInt(toTypes.size())) .invokeConstructor(InterleavedBlockBuilder.class, List.class, BlockBuilderStatus.class, int.class) .putVariable(blockBuilder); // loop through to append member blocks for (int i = 0; i < toTypes.size(); i++) { Signature signature = internalOperator(CAST.name(), toTypes.get(i).getTypeSignature(), ImmutableList.of(fromTypes.get(i).getTypeSignature())); ScalarFunctionImplementation function = functionRegistry.getScalarFunctionImplementation(signature); Type currentFromType = fromTypes.get(i); if (currentFromType.equals(UNKNOWN)) { body.append(blockBuilder.invoke("appendNull", BlockBuilder.class).pop()); continue; } BytecodeExpression fromElement = constantType(binder, currentFromType).getValue(value, constantInt(i)); BytecodeExpression toElement = invokeFunction(scope, cachedInstanceBinder, signature.getName(), function, fromElement); IfStatement ifElementNull = new IfStatement("if the element in the row type is null..."); ifElementNull.condition(value.invoke("isNull", boolean.class, constantInt(i))) .ifTrue(blockBuilder.invoke("appendNull", BlockBuilder.class).pop()) .ifFalse(constantType(binder, toTypes.get(i)).writeValue(blockBuilder, toElement)); body.append(ifElementNull); } // call blockBuilder.build() body.append(blockBuilder.invoke("build", Block.class)).retObject(); // create constructor MethodDefinition constructorDefinition = definition.declareConstructor(a(PUBLIC)); BytecodeBlock constructorBody = constructorDefinition.getBody(); Variable thisVariable = constructorDefinition.getThis(); constructorBody.comment("super();").append(thisVariable).invokeConstructor(Object.class); cachedInstanceBinder.generateInitializations(thisVariable, constructorBody); constructorBody.ret(); return defineClass(definition, Object.class, binder.getBindings(), RowToRowCast.class.getClassLoader()); }
From source file:org.jclouds.jdbc.service.JdbcService.java
@Transactional(rollbackOn = IOException.class) public BlobEntity createOrModifyBlob(String containerName, Blob blob, BlobAccess blobAccess) throws IOException { List<Long> chunks; HashingInputStream his = new HashingInputStream(Hashing.md5(), blob.getPayload().openStream()); try {//w ww . ja v a 2s . c o m chunks = storeData(his); } finally { Closeables2.closeQuietly(his); } HashCode actualHashCode = his.hash(); HashCode expectedHashCode = blob.getPayload().getContentMetadata().getContentMD5AsHashCode(); if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) { throw new IOException( "MD5 hash code mismatch, actual: " + actualHashCode + " expected: " + expectedHashCode); } String key = blob.getMetadata().getName(); Date creationDate = null; BlobEntity oldBlobEntity = findBlobById(containerName, key); if (oldBlobEntity != null) { creationDate = oldBlobEntity.getCreationDate(); } BlobEntity blobEntity = blobToBlobEntity.apply(blob); blobEntity.getPayload().setChunks(chunks); blobEntity.setContainerEntity(containerRepository.findContainerByName(containerName)); blobEntity.setKey(key); blobEntity.setBlobAccess(blobAccess); blobEntity.setTier(blob.getMetadata().getTier()); blobEntity.setCreationDate(creationDate); blobEntity.setLastModified(new Date()); blobEntity.setEtag(base16().lowerCase().encode(actualHashCode.asBytes())); blobEntity.getPayload().setContentMD5(actualHashCode.asBytes()); BlobEntity result = blobRepository.save(blobEntity); return result; }
From source file:edu.cmu.lti.oaqa.ecd.phase.ProcessingStepUtils.java
/** * Execution hash is computed from JCas currentExperimentId, trace and sequenceId * @return MD5 hash corresponding to the above mentioned elements *///from w ww . ja v a 2 s.c o m public static String getExecutionIdHash(JCas jcas) { String experimentId = ProcessingStepUtils.getCurrentExperimentId(jcas); Trace trace = ProcessingStepUtils.getTrace(jcas); String sequenceId = ProcessingStepUtils.getSequenceId(jcas); HashFunction hf = Hashing.md5(); Hasher hasher = hf.newHasher(); hasher.putString(experimentId, StandardCharsets.UTF_16LE); hasher.putString(trace.getTrace(), StandardCharsets.UTF_16LE); hasher.putString(String.valueOf(sequenceId), StandardCharsets.UTF_16LE); HashCode hash = hasher.hash(); final String traceHash = hash.toString(); return traceHash; }
From source file:com.peppe130.supermaninstaller.core.CheckFile.java
@Override protected Boolean doInBackground(String... mCheck) { String mModel = (String.format(Utils.ACTIVITY.getString(R.string.device_model), Utils.GetDeviceModel())); StringBuilder sbUpdate = new StringBuilder(); updateResult((long) 1200, sbUpdate.append(mModel).toString()); if (ControlCenter.TRIAL_MODE) { isDeviceCompatible = true;//from w w w .j a v a2s .c o m } if (isDeviceCompatible) { if (ControlCenter.TRIAL_MODE) { updateResult((long) 2500, sbUpdate.append(Utils.ACTIVITY.getString(R.string.calculating_md5)) .append(" (Fake)").toString()); updateResult((long) 5000, sbUpdate.append(Utils.ACTIVITY.getString(R.string.initialized)).toString()); return true; } else { publishProgress(sbUpdate.append(Utils.ACTIVITY.getString(R.string.calculating_md5)).toString()); try { mMD5 = Files.hash(Utils.GetZipFile(), Hashing.md5()).toString(); } catch (IOException e) { e.printStackTrace(); } try { Thread.sleep(2000); } catch (InterruptedException e) { e.printStackTrace(); } if ((Utils.GetZipFile().exists()) && (Arrays.asList(ControlCenter.ROM_MD5_LIST).contains(mMD5.toUpperCase()) || Arrays.asList(ControlCenter.ROM_MD5_LIST).contains(mMD5.toLowerCase()))) { updateResult((long) 5000, sbUpdate.append(Utils.ACTIVITY.getString(R.string.initialized)).toString()); return true; } } } return false; }