Example usage for com.amazonaws.services.kinesis.model Record getData

List of usage examples for com.amazonaws.services.kinesis.model Record getData

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.model.*
  5. Record
  6. getData

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model Record getData.

Prototype


public java.nio.ByteBuffer getData()

Source Link

Document

The data blob.

Usage

From source file:AmazonKinesisAuditRecordProcessor.java

License:Open Source License

/**
 * Process a single record./*from ww w . j a  va2  s  .co m*/
 * 
 * @param record The record to be processed.
 */
private void processSingleRecord(Record record) {
    // TODO Add your own record processing logic here

    String data = null;
    String App_uuid = "4aaf4ac1-b0fd-45c2-8b8f-88aaf3a6248f";
    try {
        // For this app, we interpret the payload as UTF-8 chars.
        data = decoder.decode(record.getData()).toString();
        Boolean Verify = false;
        String str = data;

        String output = new String();
        String[] delims = { "\"audit_uuid\":", "\"where_uri\":", "\"who_uri\":", "\"what_uri\":",
                "\"which_changed\":", "\"why\":", "\"when_audited\":", "\"when_persisted\":",
                "\"corrected_audit_uuid\":" };
        String[] words = new String[delims.length + 1];

        for (int i = 1; i < delims.length; i++) {
            StringTokenizer st = new StringTokenizer(str, delims[i]);
            String[] tokens = str.split(delims[i]);
            String phrase = new String();
            int tokenCount = tokens.length;
            for (int j = 0; j < tokenCount; j++) {
                if (j == 0) {
                    tokens[j] = tokens[j].replace("{\"audit_uuid\":", "");
                    tokens[j] = tokens[j].replace("\"where_uri\":", "");
                    tokens[j] = tokens[j].replace("\"who_uri\":", "");
                    tokens[j] = tokens[j].replace("\"what_uri\":", "");
                    tokens[j] = tokens[j].replace("\"which_changed\":", "");
                    tokens[j] = tokens[j].replace("\"why\":", "");
                    tokens[j] = tokens[j].replace("\"when_audited\":", "");
                    tokens[j] = tokens[j].replace("\"when_persisted\":", "");
                    tokens[j] = tokens[j].replace("\"corrected_audit_uuid\":", "");
                    phrase = tokens[j];
                    String rephrase = null;
                    if (phrase != null && phrase.length() > 1) {
                        rephrase = phrase.substring(0, phrase.length() - 1);
                        words[i] = rephrase;
                    }
                }
                if (j == 1) {
                    str = tokens[j];
                    if (i == delims.length - 1) {
                        words[delims.length - 1] = str;
                    }
                }
            }
        }
        for (int k = 1; k < delims.length; k++) {
            if (k == 2) {
                Verify = words[k].contains(App_uuid);
                if (Verify) {
                    System.out.println("\nAudit: " + data);
                    Verify = false;
                }
            }
            if (k == 3) {
                //System.out.println("\nwho_uri :"+ words[k]);
            }

        }

    } catch (CharacterCodingException e) {
        //do something clever with the exception
        LOG.error("Malformed data: " + data, e);
    } finally {

    }
}

From source file:SampleKinesisRecordScheme.java

License:Open Source License

@Override
public List<Object> deserialize(Record record) {
    final List<Object> l = new ArrayList<>();
    l.add(record.getPartitionKey());/*  w  w  w .j ava2s .  c o m*/
    l.add(record.getSequenceNumber());
    l.add(record.getData().array());
    return l;
}

From source file:AmazonKinesisGet.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();/*from w  ww. ja v a 2  s  . c o m*/

    final String myStreamName = "philsteststream";
    final Integer myStreamSize = 1;

    // list all of my streams
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);

        streamNames.addAll(listStreamsResult.getStreamNames());

    }
    LOG.info("Printing my list of streams : ");

    // print all of my streams.
    if (!streamNames.isEmpty()) {
        System.out.println("List of my streams: ");
    }
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println(streamNames.get(i));
    }

    //System.out.println(streamNames.get(0));
    String myownstream = streamNames.get(0);

    // Retrieve the Shards from a Stream
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(myownstream);
    DescribeStreamResult describeStreamResult;
    List<Shard> shards = new ArrayList<>();
    String lastShardId = null;

    do {
        describeStreamRequest.setExclusiveStartShardId(lastShardId);
        describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        shards.addAll(describeStreamResult.getStreamDescription().getShards());
        if (shards.size() > 0) {
            lastShardId = shards.get(shards.size() - 1).getShardId();
        }
    } while (describeStreamResult.getStreamDescription().getHasMoreShards());

    // Get Data from the Shards in a Stream
    // Hard-coded to use only 1 shard
    String shardIterator;
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(myownstream);
    //get(0) shows hardcoded to 1 stream
    getShardIteratorRequest.setShardId(shards.get(0).getShardId());
    // using TRIM_HORIZON but could use alternatives
    getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");

    GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();

    // Continuously read data records from shard.
    List<Record> records;

    while (true) {
        // Create new GetRecordsRequest with existing shardIterator.
        // Set maximum records to return to 1000.

        GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(1000);

        GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest);

        // Put result into record list. Result may be empty.
        records = result.getRecords();

        // Print records
        for (Record record : records) {
            ByteBuffer byteBuffer = record.getData();
            System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(),
                    new String(byteBuffer.array())));
        }

        try {
            Thread.sleep(1000);
        } catch (InterruptedException exception) {
            throw new RuntimeException(exception);
        }

        shardIterator = result.getNextShardIterator();
    }

}

From source file:AmazonKinesisApplicationSampleRecordProcessor.java

License:Open Source License

/**
 * Process a single record.//from  ww  w.j  a v a 2 s.  com
 * 
 * @param record The record to be processed.
 */
private void processSingleRecord(Record record) {
    // TODO Add your own record processing logic here

    String data = null;
    try {
        // For this app, we interpret the payload as UTF-8 chars.
        data = decoder.decode(record.getData()).toString();
        // Assume this record came from AmazonKinesisSample and log its age.
        long recordCreateTime = new Long(data.substring("testData-".length()));
        long ageOfRecordInMillis = System.currentTimeMillis() - recordCreateTime;

        LOG.info(record.getSequenceNumber() + ", " + record.getPartitionKey() + ", " + data + ", Created "
                + ageOfRecordInMillis + " milliseconds ago.");
    } catch (NumberFormatException e) {
        LOG.info("Record does not match sample record format. Ignoring record with data; " + data);
    } catch (CharacterCodingException e) {
        LOG.error("Malformed data: " + data, e);
    }
}

From source file:SampleRecordProcessor.java

License:Open Source License

/** Process records performing retries as needed. Skip "poison pill" records.
 * @param records/*from ww  w.  jav  a2s .  c  o m*/
 */
private void processRecordsWithRetries(List<Record> records) {
    for (Record record : records) {
        boolean processedSuccessfully = false;
        String data = null;
        for (int i = 0; i < NUM_RETRIES; i++) {
            try {
                // For this app, we interpret the payload as UTF-8 chars.
                data = decoder.decode(record.getData()).toString();
                LOG.info(record.getSequenceNumber() + ", " + record.getPartitionKey() + ", " + data);
                //
                // Logic to process record goes here.
                //
                processedSuccessfully = true;
                break;
            } catch (CharacterCodingException e) {
                LOG.error("Malformed data: " + data, e);
                break;
            } catch (Throwable t) {
                LOG.warn("Caught throwable while processing record " + record, t);
            }

            // backoff if we encounter an exception.
            try {
                Thread.sleep(BACKOFF_TIME_IN_MILLIS);
            } catch (InterruptedException e) {
                LOG.debug("Interrupted sleep", e);
            }
        }

        if (!processedSuccessfully) {
            LOG.error("Couldn't process record " + record + ". Skipping the record.");
        }
    }
}

From source file:com.alertlogic.aws.analytics.poc.RecordProcessor.java

License:Open Source License

@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    for (Record r : records) {
        // Deserialize each record as an UTF-8 encoded JSON String of the type provided
        T record;/* ww w .  j  ava2s.c  o m*/
        try {
            record = JSON.readValue(r.getData().array(), recordType);
        } catch (IOException e) {
            LOG.warn("Skipping record. Unable to parse record into Record. Partition Key: "
                    + r.getPartitionKey() + ". Sequence Number: " + r.getSequenceNumber(), e);
            continue;
        }
        // Increment the counter for the new record. This is synchronized because there is another thread reading from
        // the counter to compute running totals every interval.
        synchronized (counter) {
            counter.increment(record);
        }
    }

    // Checkpoint if it's time to!
    if (checkpointTimer.isTimeUp()) {
        // Obtain a lock on the counter to prevent additional counts from being calculated while checkpointing.
        synchronized (counter) {
            checkpoint(checkpointer);
            resetCheckpointAlarm();
        }
    }
}

From source file:com.alertlogic.aws.kinesis.test1.kcl.CountingRecordProcessor.java

License:Open Source License

@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    for (Record r : records) {
        // Deserialize each record as an UTF-8 encoded JSON String of the type provided
        T pair;//from w  w  w  . j  a  va2  s .  c  o  m
        try {
            pair = JSON.readValue(r.getData().array(), recordType);
        } catch (IOException e) {
            LOG.warn("Skipping record. Unable to parse record into HttpReferrerPair. Partition Key: "
                    + r.getPartitionKey() + ". Sequence Number: " + r.getSequenceNumber(), e);
            continue;
        }
        // Increment the counter for the new pair. This is synchronized because there is another thread reading from
        // the counter to compute running totals every interval.
        synchronized (counter) {
            counter.increment(pair);
        }
    }

    // Checkpoint if it's time to!
    if (checkpointTimer.isTimeUp()) {
        // Obtain a lock on the counter to prevent additional counts from being calculated while checkpointing.
        synchronized (counter) {
            checkpoint(checkpointer);
            resetCheckpointAlarm();
        }
    }
}

From source file:com.calamp.services.kinesis.events.processor.OrderedRecordProcessor.java

License:Open Source License

private CalAmpEvent processRecord(Record record) {
    CalAmpEvent e = CalAmpEvent.fromJsonAsBytes(record.getData().array());
    if (e == null) {
        LOG.warn("Skipping record. Unable to parse record into StockTrade. Partition Key: "
                + record.getPartitionKey());
        return null;
    }//from   w  ww. j  a v a  2s.  c o  m
    Utils.lazyLog(record, CalAmpParameters.orderedStreamName, CalAmpParameters.readLogName);
    return e;
}

From source file:com.calamp.services.kinesis.events.processor.UnorderedRecordProcessor.java

License:Open Source License

/**
 * {@inheritDoc}//from  w  ww.  jav  a 2s. c o m
 */
@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    System.out.println("Process Unordered Records #" + records.size());
    List<CalAmpEvent> eventsOldEnough = Collections.synchronizedList(new ArrayList<CalAmpEvent>());
    List<CalAmpEvent> eventsTooYoung = Collections.synchronizedList(new ArrayList<CalAmpEvent>());
    for (Record r : records) {
        Utils.lazyLog(r, CalAmpParameters.unorderdStreamName, CalAmpParameters.bufferLogName);
        // The bytes could be null if there is an issue with the JSON serialization by the Jackson JSON library.
        byte[] bytes = r.getData().array();
        if (bytes != null) {
            CalAmpEvent e = CalAmpEvent.fromJsonAsBytes(r.getData().array());
            if (e != null) {
                if (!(eventsOldEnough.contains(e))) {
                    if (CalAmpEventFilter.oldEnough(e)) {
                        eventsOldEnough.add(e);
                    } else if (!(eventsTooYoung.contains(e))) {
                        eventsTooYoung.add(e);
                    }
                }
            } else {
                LOG.warn("Skipping record. Unable to parse record into CalAmpEvent 2. Partition Key: "
                        + r.getPartitionKey() + " Event: " + e);
            }
        } else {
            LOG.warn("Skipping record. Unable to parse record into CalAmpEvent 1. Partition Key: "
                    + r.getPartitionKey());
        }
    }

    Collections.sort(eventsOldEnough, new CalAmpEventPriorityComparator());
    Collections.sort(eventsTooYoung, new CalAmpEventPriorityComparator());

    for (CalAmpEvent cae : eventsTooYoung) {
        LOG.info("Event too young : " + cae);

    }
    for (CalAmpEvent cae : eventsOldEnough) {
        LOG.info("Event old enough: " + cae);
    }

    Utils.putByParts(eventsTooYoung, CalAmpParameters.unorderdStreamName, kinesisClientToUnordered,
            CalAmpParameters.bufferLogName);
    Utils.putByParts(eventsOldEnough, CalAmpParameters.orderedStreamName, kinesisClientToOrdered,
            CalAmpParameters.bufferLogName);
    checkpoint(checkpointer);
}

From source file:com.datatorrent.apps.KinesisByteArrayInputOperator.java

License:Apache License

/**
 * Implement abstract method of AbstractKinesisInputOperator
 *//*from   w  ww  .j  a  v  a 2 s  . c om*/
@Override
public byte[] getTuple(Record record) {
    try {
        ByteBuffer bb = record.getData();
        byte[] bytes = new byte[bb.remaining()];
        bb.get(bytes);
        return bytes;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}