Example usage for com.amazonaws.services.kinesis.model Record getPartitionKey

List of usage examples for com.amazonaws.services.kinesis.model Record getPartitionKey

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.model.*
  5. Record
  6. getPartitionKey

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model Record getPartitionKey.

Prototype


public String getPartitionKey()

Source Link

Document

Identifies which shard in the stream the data record is assigned to.

Usage

From source file:SampleKinesisRecordScheme.java

License:Open Source License

@Override
public List<Object> deserialize(Record record) {
    final List<Object> l = new ArrayList<>();
    l.add(record.getPartitionKey());
    l.add(record.getSequenceNumber());//from w w w.j  a v a  2s .  c o m
    l.add(record.getData().array());
    return l;
}

From source file:AmazonKinesisApplicationSampleRecordProcessor.java

License:Open Source License

/**
 * Process a single record./*  ww  w.ja  v a  2 s .  c o  m*/
 * 
 * @param record The record to be processed.
 */
private void processSingleRecord(Record record) {
    // TODO Add your own record processing logic here

    String data = null;
    try {
        // For this app, we interpret the payload as UTF-8 chars.
        data = decoder.decode(record.getData()).toString();
        // Assume this record came from AmazonKinesisSample and log its age.
        long recordCreateTime = new Long(data.substring("testData-".length()));
        long ageOfRecordInMillis = System.currentTimeMillis() - recordCreateTime;

        LOG.info(record.getSequenceNumber() + ", " + record.getPartitionKey() + ", " + data + ", Created "
                + ageOfRecordInMillis + " milliseconds ago.");
    } catch (NumberFormatException e) {
        LOG.info("Record does not match sample record format. Ignoring record with data; " + data);
    } catch (CharacterCodingException e) {
        LOG.error("Malformed data: " + data, e);
    }
}

From source file:SampleRecordProcessor.java

License:Open Source License

/** Process records performing retries as needed. Skip "poison pill" records.
 * @param records/*from   w w  w .j  av a  2  s  . c o  m*/
 */
private void processRecordsWithRetries(List<Record> records) {
    for (Record record : records) {
        boolean processedSuccessfully = false;
        String data = null;
        for (int i = 0; i < NUM_RETRIES; i++) {
            try {
                // For this app, we interpret the payload as UTF-8 chars.
                data = decoder.decode(record.getData()).toString();
                LOG.info(record.getSequenceNumber() + ", " + record.getPartitionKey() + ", " + data);
                //
                // Logic to process record goes here.
                //
                processedSuccessfully = true;
                break;
            } catch (CharacterCodingException e) {
                LOG.error("Malformed data: " + data, e);
                break;
            } catch (Throwable t) {
                LOG.warn("Caught throwable while processing record " + record, t);
            }

            // backoff if we encounter an exception.
            try {
                Thread.sleep(BACKOFF_TIME_IN_MILLIS);
            } catch (InterruptedException e) {
                LOG.debug("Interrupted sleep", e);
            }
        }

        if (!processedSuccessfully) {
            LOG.error("Couldn't process record " + record + ". Skipping the record.");
        }
    }
}

From source file:com.alertlogic.aws.analytics.poc.RecordProcessor.java

License:Open Source License

@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    for (Record r : records) {
        // Deserialize each record as an UTF-8 encoded JSON String of the type provided
        T record;//from  w ww .j  a va  2 s  . c  o  m
        try {
            record = JSON.readValue(r.getData().array(), recordType);
        } catch (IOException e) {
            LOG.warn("Skipping record. Unable to parse record into Record. Partition Key: "
                    + r.getPartitionKey() + ". Sequence Number: " + r.getSequenceNumber(), e);
            continue;
        }
        // Increment the counter for the new record. This is synchronized because there is another thread reading from
        // the counter to compute running totals every interval.
        synchronized (counter) {
            counter.increment(record);
        }
    }

    // Checkpoint if it's time to!
    if (checkpointTimer.isTimeUp()) {
        // Obtain a lock on the counter to prevent additional counts from being calculated while checkpointing.
        synchronized (counter) {
            checkpoint(checkpointer);
            resetCheckpointAlarm();
        }
    }
}

From source file:com.alertlogic.aws.kinesis.test1.kcl.CountingRecordProcessor.java

License:Open Source License

@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    for (Record r : records) {
        // Deserialize each record as an UTF-8 encoded JSON String of the type provided
        T pair;//from ww  w .  ja v a2  s  .com
        try {
            pair = JSON.readValue(r.getData().array(), recordType);
        } catch (IOException e) {
            LOG.warn("Skipping record. Unable to parse record into HttpReferrerPair. Partition Key: "
                    + r.getPartitionKey() + ". Sequence Number: " + r.getSequenceNumber(), e);
            continue;
        }
        // Increment the counter for the new pair. This is synchronized because there is another thread reading from
        // the counter to compute running totals every interval.
        synchronized (counter) {
            counter.increment(pair);
        }
    }

    // Checkpoint if it's time to!
    if (checkpointTimer.isTimeUp()) {
        // Obtain a lock on the counter to prevent additional counts from being calculated while checkpointing.
        synchronized (counter) {
            checkpoint(checkpointer);
            resetCheckpointAlarm();
        }
    }
}

From source file:com.calamp.services.kinesis.events.processor.OrderedRecordProcessor.java

License:Open Source License

private CalAmpEvent processRecord(Record record) {
    CalAmpEvent e = CalAmpEvent.fromJsonAsBytes(record.getData().array());
    if (e == null) {
        LOG.warn("Skipping record. Unable to parse record into StockTrade. Partition Key: "
                + record.getPartitionKey());
        return null;
    }//from  w w w . ja v a  2  s.  co m
    Utils.lazyLog(record, CalAmpParameters.orderedStreamName, CalAmpParameters.readLogName);
    return e;
}

From source file:com.calamp.services.kinesis.events.processor.UnorderedRecordProcessor.java

License:Open Source License

/**
 * {@inheritDoc}//from w  ww . j a v  a2 s .c  o  m
 */
@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    System.out.println("Process Unordered Records #" + records.size());
    List<CalAmpEvent> eventsOldEnough = Collections.synchronizedList(new ArrayList<CalAmpEvent>());
    List<CalAmpEvent> eventsTooYoung = Collections.synchronizedList(new ArrayList<CalAmpEvent>());
    for (Record r : records) {
        Utils.lazyLog(r, CalAmpParameters.unorderdStreamName, CalAmpParameters.bufferLogName);
        // The bytes could be null if there is an issue with the JSON serialization by the Jackson JSON library.
        byte[] bytes = r.getData().array();
        if (bytes != null) {
            CalAmpEvent e = CalAmpEvent.fromJsonAsBytes(r.getData().array());
            if (e != null) {
                if (!(eventsOldEnough.contains(e))) {
                    if (CalAmpEventFilter.oldEnough(e)) {
                        eventsOldEnough.add(e);
                    } else if (!(eventsTooYoung.contains(e))) {
                        eventsTooYoung.add(e);
                    }
                }
            } else {
                LOG.warn("Skipping record. Unable to parse record into CalAmpEvent 2. Partition Key: "
                        + r.getPartitionKey() + " Event: " + e);
            }
        } else {
            LOG.warn("Skipping record. Unable to parse record into CalAmpEvent 1. Partition Key: "
                    + r.getPartitionKey());
        }
    }

    Collections.sort(eventsOldEnough, new CalAmpEventPriorityComparator());
    Collections.sort(eventsTooYoung, new CalAmpEventPriorityComparator());

    for (CalAmpEvent cae : eventsTooYoung) {
        LOG.info("Event too young : " + cae);

    }
    for (CalAmpEvent cae : eventsOldEnough) {
        LOG.info("Event old enough: " + cae);
    }

    Utils.putByParts(eventsTooYoung, CalAmpParameters.unorderdStreamName, kinesisClientToUnordered,
            CalAmpParameters.bufferLogName);
    Utils.putByParts(eventsOldEnough, CalAmpParameters.orderedStreamName, kinesisClientToOrdered,
            CalAmpParameters.bufferLogName);
    checkpoint(checkpointer);
}

From source file:com.hortonworks.streamline.streams.runtime.storm.spout.KinesisRecordToTupleMapper.java

License:Apache License

@Override
public List<Object> getTuple(Record record) {
    CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder();
    List<Object> tuple = new ArrayList<>();
    tuple.add(record.getPartitionKey());
    tuple.add(record.getSequenceNumber());
    try {/*from w  ww.  j  a v a  2  s  .  c o m*/
        String data = decoder.decode(record.getData()).toString();
        tuple.add(data);
    } catch (CharacterCodingException e) {
        e.printStackTrace();
        LOG.warn("Exception occured. Emitting tuple with empty string data", e);
        tuple.add("");
    }
    return tuple;
}

From source file:com.kinesis.datavis.kcl.processor.CountingRecordProcessor.java

License:Open Source License

@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    for (Record r : records) {
        // Deserialize each record as an UTF-8 encoded JSON String of the type provided
        T rec;//from   ww w  .  j  a  v a2  s.c  o  m
        try {
            rec = JSON.readValue(r.getData().array(), recordType);

            rec = typeProcessor.process(r, rec);

        } catch (IOException e) {
            LOG.warn("Skipping record. Unable to parse record into HttpReferrerPair. Partition Key: "
                    + r.getPartitionKey() + ". Sequence Number: " + r.getSequenceNumber(), e);
            continue;
        }

        synchronized (windowCounter) {
            windowCounter.increment(rec);
        }
    }

    if (buffer.shouldFlush()) {
        emit(buffer.getRecords());
    }

    // Checkpoint if it's time to!
    if (checkpointTimer.isTimeUp()) {
        // Obtain a lock on the windowCounter to prevent additional counts from being calculated while checkpointing.
        synchronized (windowCounter) {
            checkpoint(checkpointer);
            resetCheckpointAlarm();
        }
    }
}

From source file:com.kinesis.processor.StockTradeRecordProcessor.java

License:Open Source License

private void processRecord(Record record) {
    StockTrade trade = StockTrade.fromJsonAsBytes(record.getData().array());
    if (trade == null) {
        LOG.warn("Skipping record. Unable to parse record into StockTrade. Partition Key: "
                + record.getPartitionKey());
        return;// ww  w .java2 s . c  o  m
    }
    stockStats.addStockTrade(trade);
}