List of usage examples for com.amazonaws.services.dynamodbv2.model BatchGetItemResult getUnprocessedKeys
public java.util.Map<String, KeysAndAttributes> getUnprocessedKeys()
A map of tables and their respective keys that were not processed with the current response.
From source file:com.dell.doradus.db.dynamodb.DDBRowIterator.java
License:Apache License
/** * Create a row iterator that returns specific rows and columns for the given table. A * series of batch requests are made if needed. * //from www . j ava2s .c o m * @param tableName Name for which to get rows for. * @param rowKeys Collection of row keys to get. * @param colNames Set of column names to fetch. If null or empty, all columns are * fetched. */ public DDBRowIterator(String tableName, Collection<String> rowKeys, Collection<String> colNames) { Iterator<String> rowKeyIter = rowKeys.iterator(); KeysAndAttributes keysAndAttributes = makeKeyBatch(rowKeyIter); while (true) { BatchGetItemRequest batchRequest = new BatchGetItemRequest(); batchRequest.addRequestItemsEntry(tableName, keysAndAttributes); BatchGetItemResult batchResult = DynamoDBService.instance().batchGetItem(batchRequest); Map<String, List<Map<String, AttributeValue>>> responseMap = batchResult.getResponses(); List<Map<String, AttributeValue>> itemsList = responseMap.get(tableName); if (itemsList == null || itemsList.size() == 0) { break; } for (Map<String, AttributeValue> attributeMap : itemsList) { AttributeValue rowAttr = attributeMap.get(DynamoDBService.ROW_KEY_ATTR_NAME); m_rowList.add(new DDBRow(rowAttr.getS(), new DDBColumnIterator(attributeMap, colNames))); } Map<String, KeysAndAttributes> unprocessedKeys = batchResult.getUnprocessedKeys(); if (unprocessedKeys != null && unprocessedKeys.containsKey(tableName)) { keysAndAttributes = unprocessedKeys.get(tableName); } else if (rowKeyIter.hasNext()) { keysAndAttributes = makeKeyBatch(rowKeyIter); } else { break; } } }
From source file:com.dell.doradus.db.dynamodb.DDBRowIterator.java
License:Apache License
/** * Create a row iterator that returns specific rows and column ranges for the given * table. A series of batch item requests are made if needed. * /*from www.ja v a2 s . co m*/ * @param tableName Name for which to get rows for. * @param rowKeys Collection of row keys to get. * @param startCol If non-null/empty, only column names greater than or equal to this * name are captured. * @param endCol If non-null/empty, only column names less than or equal to this * name are captured. */ public DDBRowIterator(String tableName, Collection<String> rowKeys, String startCol, String endCol) { Iterator<String> rowKeyIter = rowKeys.iterator(); KeysAndAttributes keysAndAttributes = makeKeyBatch(rowKeyIter); while (true) { BatchGetItemRequest batchRequest = new BatchGetItemRequest(); batchRequest.addRequestItemsEntry(tableName, keysAndAttributes); BatchGetItemResult batchResult = DynamoDBService.instance().batchGetItem(batchRequest); Map<String, List<Map<String, AttributeValue>>> responseMap = batchResult.getResponses(); List<Map<String, AttributeValue>> itemsList = responseMap.get(tableName); if (itemsList == null || itemsList.size() == 0) { break; } for (Map<String, AttributeValue> attributeMap : itemsList) { AttributeValue rowAttr = attributeMap.get(DynamoDBService.ROW_KEY_ATTR_NAME); m_rowList.add(new DDBRow(rowAttr.getS(), new DDBColumnIterator(attributeMap, startCol, endCol))); } Map<String, KeysAndAttributes> unprocessedKeys = batchResult.getUnprocessedKeys(); if (unprocessedKeys != null && unprocessedKeys.containsKey(tableName)) { keysAndAttributes = unprocessedKeys.get(tableName); } else if (rowKeyIter.hasNext()) { keysAndAttributes = makeKeyBatch(rowKeyIter); } else { break; } } }
From source file:com.dell.doradus.db.s3.DynamoDBService2.java
License:Apache License
@Override public List<DColumn> getColumns(String storeName, String rowKey, Collection<String> columnNames) { Timer t = new Timer(); String namespace = getTenant().getName(); String key = storeName + "_" + rowKey; List<DColumn> list = new ArrayList<>(); List<Map<String, AttributeValue>> keys = new ArrayList<>(); for (String col : columnNames) { keys.add(getPrimaryKey(key, col)); if (keys.size() >= 100) { KeysAndAttributes x = new KeysAndAttributes().withKeys(keys); Map<String, KeysAndAttributes> map = new HashMap<>(); map.put(namespace, x);// w w w . j av a2 s. c o m BatchGetItemResult result = m_client.batchGetItem(new BatchGetItemRequest(map)); if (result.getUnprocessedKeys().size() > 0) throw new RuntimeException("Could not process all items"); list.addAll(fromItems(result.getResponses().get(namespace))); keys.clear(); } } if (keys.size() > 0) { KeysAndAttributes x = new KeysAndAttributes().withKeys(keys); Map<String, KeysAndAttributes> map = new HashMap<>(); map.put(namespace, x); BatchGetItemResult result = m_client.batchGetItem(new BatchGetItemRequest(map)); if (result.getUnprocessedKeys().size() > 0) throw new RuntimeException("Could not process all items"); list.addAll(fromItems(result.getResponses().get(namespace))); keys.clear(); } m_logger.debug("get columns for {} in {}", namespace, t); return list; }
From source file:com.erudika.para.persistence.AWSDynamoDAO.java
License:Apache License
private <P extends ParaObject> void batchGet(Map<String, KeysAndAttributes> kna, Map<String, P> results) { if (kna == null || kna.isEmpty() || results == null) { return;/* w w w . j av a2s . co m*/ } try { BatchGetItemResult result = client().batchGetItem(new BatchGetItemRequest() .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withRequestItems(kna)); if (result == null) { return; } List<Map<String, AttributeValue>> res = result.getResponses().get(kna.keySet().iterator().next()); for (Map<String, AttributeValue> item : res) { P obj = fromRow(item); if (obj != null) { results.put(obj.getId(), obj); } } logger.debug("batchGet(): total {}, cc {}", res.size(), result.getConsumedCapacity()); if (result.getUnprocessedKeys() != null && !result.getUnprocessedKeys().isEmpty()) { Thread.sleep(1000); logger.warn("UNPROCESSED {}", result.getUnprocessedKeys().size()); batchGet(result.getUnprocessedKeys(), results); } } catch (Exception e) { logger.error(null, e); } }
From source file:com.vivastream.dynamodb.core.DynamoDBTemplate.java
License:Apache License
public <T> List<T> batchGet(String tableName, KeysAndAttributes keysAndAttributes, final ObjectExtractor<T> extractor) throws EmptyResultDataAccessException { Assert.notNull(tableName, "Table must not be null"); Assert.notNull(extractor, "ObjectExtractor must not be null"); if (logger.isDebugEnabled()) { logger.debug("Executing batch get on " + tableName + " for " + keysAndAttributes.toString()); }/* w w w . ja va 2 s. c o m*/ List<T> results = new ArrayList<T>(keysAndAttributes.getKeys().size()); Map<String, KeysAndAttributes> unprocessedKeys = Collections.singletonMap(tableName, keysAndAttributes); while (unprocessedKeys.size() > 0) { BatchGetItemResult result = client.batchGetItem(unprocessedKeys); List<Map<String, AttributeValue>> items = result.getResponses().get(tableName); if (items != null) { for (Map<String, AttributeValue> item : items) { results.add(extractor.extract(item)); } } unprocessedKeys = result.getUnprocessedKeys(); } if (results.size() == 0) { throw new EmptyResultDataAccessException( "No results found in " + tableName + "for " + keysAndAttributes.toString()); } return results; }
From source file:jp.classmethod.aws.dynamodb.DynamoDbRepository.java
License:Open Source License
private List<E> findAll(Iterable<AttributeValue> ids, boolean useParallelBatches) { Preconditions.checkNotNull(ids, "ids may not be null"); List<AttributeValue> idList = Lists.newArrayList(ids); if (idList.isEmpty()) { return new ArrayList<>(); }//ww w.j a v a 2s . c om List<Map<String, AttributeValue>> resultantItems = new ArrayList<>(); StreamSupport.stream(Iterables.partition(idList, 25).spliterator(), useParallelBatches).forEach(inner -> { BatchGetItemRequest req = new BatchGetItemRequest(); KeysAndAttributes keysAndAttributes = new KeysAndAttributes(); keysAndAttributes.setConsistentRead(true); keysAndAttributes.setKeys( inner.stream().map(id -> ImmutableMap.of(hashKeyName, id)).collect(Collectors.toList())); String tableName = tableName(); req.withRequestItems(ImmutableMap.of(tableName, keysAndAttributes)); BatchGetItemResult result; do { try { result = dynamoDB.batchGetItem(req); resultantItems.addAll(result.getResponses().get(tableName)); req.setRequestItems(result.getUnprocessedKeys()); } catch (AmazonClientException e) { throw this.convertDynamoDBException(e, "batch get", null /*no conditions for reads*/); } } while (false == result.getUnprocessedKeys().isEmpty()); }); return resultantItems.stream().map(legacyItem -> Item.fromMap(InternalUtils.toSimpleMapValue(legacyItem))) .map(item -> convertItemToDomain(item)).collect(Collectors.toList()); }
From source file:org.iternine.jeppetto.dao.dynamodb.iterable.BatchGetIterable.java
License:Apache License
@Override protected Iterator<Map<String, AttributeValue>> fetchItems() { // TODO: logging and catch dynamodb exception... BatchGetItemResult currentBatchGetItemResult = getDynamoDB().batchGetItem(batchGetItemRequest); Iterator<Map<String, AttributeValue>> iterator = currentBatchGetItemResult.getResponses().get(tableName) .iterator();//from w w w . j ava2 s. c om batchGetItemRequest.withRequestItems(currentBatchGetItemResult.getUnprocessedKeys()); // Prepare for next query if (logger.isDebugEnabled()) { List<ConsumedCapacity> consumedCapacities = currentBatchGetItemResult.getConsumedCapacity(); logger.debug( "Queried {} using {}. Took {} read capacity units, retrieved {} items, more items {} available.", getEnhancer().getBaseClass().getSimpleName(), batchGetItemRequest, consumedCapacities == null ? null : consumedCapacities.get(0), // Only expecting 1 table currentBatchGetItemResult.getResponses().get(tableName).size(), currentBatchGetItemResult.getUnprocessedKeys() == null ? "are not" : "are"); } return iterator; }