List of usage examples for com.google.api.client.googleapis.batch BatchRequest size
public int size()
From source file:com.google.cloud.dataflow.sdk.util.GcsUtil.java
License:Apache License
List<BatchRequest> makeCopyBatches(List<String> srcFilenames, List<String> destFilenames) throws IOException { checkArgument(srcFilenames.size() == destFilenames.size(), "Number of source files %s must equal number of destination files %s", srcFilenames.size(), destFilenames.size());/* ww w . j a v a 2s .c o m*/ List<BatchRequest> batches = new LinkedList<>(); BatchRequest batch = storageClient.batch(); for (int i = 0; i < srcFilenames.size(); i++) { final GcsPath sourcePath = GcsPath.fromUri(srcFilenames.get(i)); final GcsPath destPath = GcsPath.fromUri(destFilenames.get(i)); enqueueCopy(sourcePath, destPath, batch); if (batch.size() >= MAX_REQUESTS_PER_BATCH) { batches.add(batch); batch = storageClient.batch(); } } if (batch.size() > 0) { batches.add(batch); } return batches; }
From source file:org.apache.beam.sdk.extensions.gcp.util.GcsUtil.java
License:Apache License
List<BatchRequest> makeCopyBatches(LinkedList<RewriteOp> rewrites) throws IOException { List<BatchRequest> batches = new ArrayList<>(); BatchRequest batch = createBatchRequest(); Iterator<RewriteOp> it = rewrites.iterator(); while (it.hasNext()) { RewriteOp rewrite = it.next();//from ww w. j av a 2 s . c o m if (!rewrite.getReadyToEnqueue()) { it.remove(); continue; } rewrite.enqueue(batch); if (batch.size() >= MAX_REQUESTS_PER_BATCH) { batches.add(batch); batch = createBatchRequest(); } } if (batch.size() > 0) { batches.add(batch); } return batches; }
From source file:org.apache.beam.sdk.util.GcsUtil.java
License:Apache License
List<BatchRequest> makeCopyBatches(Iterable<String> srcFilenames, Iterable<String> destFilenames)
throws IOException {
List<String> srcList = Lists.newArrayList(srcFilenames);
List<String> destList = Lists.newArrayList(destFilenames);
checkArgument(srcList.size() == destList.size(),
"Number of source files %s must equal number of destination files %s", srcList.size(),
destList.size());/*from ww w .j a v a 2 s . c o m*/
List<BatchRequest> batches = new LinkedList<>();
BatchRequest batch = createBatchRequest();
for (int i = 0; i < srcList.size(); i++) {
final GcsPath sourcePath = GcsPath.fromUri(srcList.get(i));
final GcsPath destPath = GcsPath.fromUri(destList.get(i));
enqueueCopy(sourcePath, destPath, batch);
if (batch.size() >= MAX_REQUESTS_PER_BATCH) {
batches.add(batch);
batch = createBatchRequest();
}
}
if (batch.size() > 0) {
batches.add(batch);
}
return batches;
}
From source file:to.lean.tools.gmail.importer.gmail.Mailbox.java
License:Open Source License
Multimap<LocalMessage, Message> mapMessageIds(Iterable<LocalMessage> localMessages) {
Multimap<LocalMessage, Message> results = MultimapBuilder.hashKeys().linkedListValues().build();
Gmail gmail = gmailService.getServiceWithRetries();
BatchRequest batch = gmail.batch();
try {//from ww w .j a va 2 s . c om
for (LocalMessage localMessage : localMessages) {
gmail.users().messages().list(user.getEmailAddress())
.setQ("rfc822msgid:" + localMessage.getMessageId()).setFields("messages(id)")
.queue(batch, new JsonBatchCallback<ListMessagesResponse>() {
@Override
public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders)
throws IOException {
System.err.println("Could not get message: " + localMessage.getMessageId());
System.err.println(" because: " + e);
}
@Override
public void onSuccess(ListMessagesResponse response, HttpHeaders responseHeaders)
throws IOException {
if (!response.isEmpty()) {
results.putAll(localMessage, response.getMessages());
System.err.println("For " + localMessage.getMessageId() + " got:");
response.getMessages().stream().forEach(
message -> System.err.println(" message id: " + message.getId()));
}
}
});
}
if (batch.size() > 0) {
batch.execute();
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return results;
}
From source file:to.lean.tools.gmail.importer.gmail.Mailbox.java
License:Open Source License
void fetchExistingLabels(Iterable<Message> messages) { Gmail gmail = gmailService.getServiceWithRetries(); BatchRequest batch = gmail.batch(); try {//ww w . j a v a 2s. c om for (Message message : messages) { gmail.users().messages().get(user.getEmailAddress(), message.getId()).setFields("id,labelIds") .queue(batch, new JsonBatchCallback<Message>() { @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException { System.err.format("For message: %s, got error: %s\n", message.getId(), e.toPrettyString()); } @Override public void onSuccess(Message responseMessage, HttpHeaders responseHeaders) throws IOException { Preconditions.checkState(message.getId().equals(responseMessage.getId()), "Message ids must be equal"); List<String> gmailMessageIds = responseMessage.getLabelIds() == null ? ImmutableList.of() : responseMessage.getLabelIds(); System.out.format("For message %s, got labels: %s\n", responseMessage.getId(), gmailMessageIds.stream() .map(id -> labelsById.getOrDefault(id, new Label().setName(id))) .map(Label::getName).collect(Collectors.joining(", "))); message.setLabelIds(gmailMessageIds); } }); } if (batch.size() > 0) { batch.execute(); } } catch (IOException e) { throw new RuntimeException(e); } }
From source file:to.lean.tools.gmail.importer.gmail.Mailbox.java
License:Open Source License
private void syncLabels(Gmail gmailApi, BiMap<String, String> labelIdToNameMap, Multimap<LocalMessage, Message> localMessageToGmailMessages) throws IOException { BatchRequest relabelBatch = gmailApi.batch(); for (Map.Entry<LocalMessage, Message> entry : localMessageToGmailMessages.entries()) { LocalMessage localMessage = entry.getKey(); Message gmailMessage = entry.getValue(); Set<String> gmailLabels = gmailMessage.getLabelIds() == null ? ImmutableSet.of() : gmailMessage.getLabelIds().stream().map(labelIdToNameMap::get).collect(Collectors.toSet()); List<String> missingLabelIds = localMessage.getFolders().stream() .map(folder -> folder.equalsIgnoreCase("Inbox") ? "INBOX" : folder) .filter(folder -> !gmailLabels.contains(folder)) .map(folder -> labelIdToNameMap.inverse().get(folder)).collect(Collectors.toList()); if (localMessage.isUnread() && !gmailLabels.contains("UNREAD")) { missingLabelIds.add("UNREAD"); }/*from ww w . j a v a 2 s. co m*/ if (localMessage.isStarred() && !gmailLabels.contains("STARRED")) { missingLabelIds.add("STARRED"); } for (String folder : localMessage.getFolders()) { if (!gmailLabels.contains(folder)) { System.out.format("Trying to add labels %s to %s\n", missingLabelIds.stream().map(labelIdToNameMap::get).collect(Collectors.joining(", ")), gmailMessage.getId()); gmailApi.users().messages() .modify(user.getEmailAddress(), gmailMessage.getId(), new ModifyMessageRequest().setAddLabelIds(missingLabelIds)) .queue(relabelBatch, new JsonBatchCallback<Message>() { @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException { System.err.format("For label ids %s, got error: %s\n", missingLabelIds, e.toPrettyString()); } @Override public void onSuccess(Message message, HttpHeaders responseHeaders) throws IOException { System.out.format( "Successfully added labels %s to %s\n", missingLabelIds.stream() .map(labelIdToNameMap::get).collect(Collectors.joining(", ")), message.getId()); } }); } } if (relabelBatch.size() > 0) { relabelBatch.execute(); } } }