Example usage for org.apache.cassandra.hadoop ConfigHelper getInputKeyRange

Introduction

In this page you can find the example usage for org.apache.cassandra.hadoop ConfigHelper getInputKeyRange.

Prototype

public static Pair<String, String> getInputKeyRange(Configuration conf) 

Source Link

Document

The start and end token of the input key range as a pair.

Usage

From source file:com.clojurewerkz.cascading.cassandra.hadoop.ColumnFamilyInputFormat.java

License:Apache License

public List<InputSplit> getSplits(JobContext context) throws IOException {
    Configuration conf = context.getConfiguration();

    validateConfiguration(conf);/*www .ja v a2s.com*/

    // cannonical ranges and nodes holding replicas
    List<TokenRange> masterRangeNodes = getRangeMap(conf);

    keyspace = ConfigHelper.getInputKeyspace(context.getConfiguration());
    cfName = ConfigHelper.getInputColumnFamily(context.getConfiguration());
    partitioner = ConfigHelper.getInputPartitioner(context.getConfiguration());
    logger.debug("partitioner is " + partitioner);

    // cannonical ranges, split into pieces, fetching the splits in parallel
    ExecutorService executor = Executors.newCachedThreadPool();
    List<InputSplit> splits = new ArrayList<InputSplit>();

    try {
        List<Future<List<InputSplit>>> splitfutures = new ArrayList<Future<List<InputSplit>>>();
        KeyRange jobKeyRange = ConfigHelper.getInputKeyRange(conf);
        Range<Token> jobRange = null;
        if (jobKeyRange != null && jobKeyRange.start_token != null) {
            assert partitioner
                    .preservesOrder() : "ConfigHelper.setInputKeyRange(..) can only be used with a order preserving paritioner";
            assert jobKeyRange.start_key == null : "only start_token supported";
            assert jobKeyRange.end_key == null : "only end_token supported";
            jobRange = new Range<Token>(partitioner.getTokenFactory().fromString(jobKeyRange.start_token),
                    partitioner.getTokenFactory().fromString(jobKeyRange.end_token), partitioner);
        }

        for (TokenRange range : masterRangeNodes) {
            if (jobRange == null) {
                // for each range, pick a live owner and ask it to compute bite-sized splits
                splitfutures.add(executor.submit(new SplitCallable(range, conf)));
            } else {
                Range<Token> dhtRange = new Range<Token>(
                        partitioner.getTokenFactory().fromString(range.start_token),
                        partitioner.getTokenFactory().fromString(range.end_token), partitioner);

                if (dhtRange.intersects(jobRange)) {
                    for (Range<Token> intersection : dhtRange.intersectionWith(jobRange)) {
                        range.start_token = partitioner.getTokenFactory().toString(intersection.left);
                        range.end_token = partitioner.getTokenFactory().toString(intersection.right);
                        // for each range, pick a live owner and ask it to compute bite-sized splits
                        splitfutures.add(executor.submit(new SplitCallable(range, conf)));
                    }
                }
            }
        }

        // wait until we have all the results back
        for (Future<List<InputSplit>> futureInputSplits : splitfutures) {
            try {
                splits.addAll(futureInputSplits.get());
            } catch (Exception e) {
                throw new IOException("Could not get input splits", e);
            }
        }
    } finally {
        executor.shutdownNow();
    }

    assert splits.size() > 0;
    Collections.shuffle(splits, new Random(System.nanoTime()));
    return splits;
}

---

From source file:com.clojurewerkz.cascading.cassandra.hadoop.ColumnFamilyRecordReader.java

License:Apache License

public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
    this.split = (ColumnFamilySplit) split;
    Configuration conf = context.getConfiguration();
    KeyRange jobRange = ConfigHelper.getInputKeyRange(conf);
    filter = jobRange == null ? null : jobRange.row_filter;
    predicate = ConfigHelper.getInputSlicePredicate(conf);
    boolean widerows = ConfigHelper.getInputIsWide(conf);
    isEmptyPredicate = isEmptyPredicate(predicate);
    totalRowCount = ConfigHelper.getInputSplitSize(conf);
    batchSize = ConfigHelper.getRangeBatchSize(conf);
    cfName = ConfigHelper.getInputColumnFamily(conf);
    consistencyLevel = ConsistencyLevel.valueOf(ConfigHelper.getReadConsistencyLevel(conf));

    keyspace = ConfigHelper.getInputKeyspace(conf);

    try {//from   www  .j  a  va  2  s . co  m
        // only need to connect once
        if (socket != null && socket.isOpen())
            return;

        // create connection using thrift
        String location = getLocation();
        socket = new TSocket(location, ConfigHelper.getInputRpcPort(conf));
        TTransport transport = ConfigHelper.getInputTransportFactory(conf).openTransport(socket);
        TBinaryProtocol binaryProtocol = new TBinaryProtocol(transport);
        client = new Cassandra.Client(binaryProtocol);

        // log in
        client.set_keyspace(keyspace);
        if (ConfigHelper.getInputKeyspaceUserName(conf) != null) {
            Map<String, String> creds = new HashMap<String, String>();
            creds.put(IAuthenticator.USERNAME_KEY, ConfigHelper.getInputKeyspaceUserName(conf));
            creds.put(IAuthenticator.PASSWORD_KEY, ConfigHelper.getInputKeyspacePassword(conf));
            AuthenticationRequest authRequest = new AuthenticationRequest(creds);
            client.login(authRequest);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    iter = widerows ? new WideRowIterator() : new StaticRowIterator();
    logger.debug("created {}", iter);
}

---