Example usage for org.apache.cassandra.hadoop ConfigHelper getOutputPartitioner

Introduction

In this page you can find the example usage for org.apache.cassandra.hadoop ConfigHelper getOutputPartitioner.

Prototype

public static IPartitioner getOutputPartitioner(Configuration conf) 

Source Link

Usage

From source file:com.buzzinate.dm.cassandra.ColumnFamilyOutputFormat.java

License:Apache License

private void checkOutputSpecs(Configuration conf) {
    if (ConfigHelper.getOutputKeyspace(conf) == null || ConfigHelper.getOutputColumnFamily(conf) == null) {
        throw new UnsupportedOperationException(
                "you must set the keyspace and columnfamily with setColumnFamily()");
    }//from   www .j a  v a 2  s. c om
    if (ConfigHelper.getOutputPartitioner(conf) == null)
        throw new UnsupportedOperationException(
                "You must set the output partitioner to the one used by your Cassandra cluster");
    if (ConfigHelper.getOutputInitialAddress(conf) == null)
        throw new UnsupportedOperationException("You must set the initial output address to a Cassandra node");
}

---

From source file:com.dse.pig.udfs.CqlStorage.java

License:Apache License

/** set store configuration settings */
public void setStoreLocation(String location, Job job) throws IOException {
    conf = job.getConfiguration();/*  w w w.j a v  a  2  s.  co m*/
    setLocationFromUri(location);

    if (username != null && password != null)
        ConfigHelper.setOutputKeyspaceUserNameAndPassword(conf, username, password);
    if (splitSize > 0)
        ConfigHelper.setInputSplitSize(conf, splitSize);
    if (partitionerClass != null)
        ConfigHelper.setOutputPartitioner(conf, partitionerClass);
    if (rpcPort != null) {
        ConfigHelper.setOutputRpcPort(conf, rpcPort);
        ConfigHelper.setInputRpcPort(conf, rpcPort);
    }
    if (initHostAddress != null) {
        ConfigHelper.setOutputInitialAddress(conf, initHostAddress);
        ConfigHelper.setInputInitialAddress(conf, initHostAddress);
    }

    ConfigHelper.setOutputColumnFamily(conf, keyspace, column_family);
    CqlConfigHelper.setOutputCql(conf, outputQuery);

    setConnectionInformation();

    if (ConfigHelper.getOutputRpcPort(conf) == 0)
        throw new IOException("PIG_OUTPUT_RPC_PORT or PIG_RPC_PORT environment variable not set");
    if (ConfigHelper.getOutputInitialAddress(conf) == null)
        throw new IOException("PIG_OUTPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set");
    if (ConfigHelper.getOutputPartitioner(conf) == null)
        throw new IOException("PIG_OUTPUT_PARTITIONER or PIG_PARTITIONER environment variable not set");

    initSchema(storeSignature);
}

---

From source file:com.spotify.hdfs2cass.cassandra.cql.CrunchCqlBulkRecordWriter.java

License:Apache License

private void prepareWriter() {
    try {//from   ww w.j  a  v  a  2s.c om
        if (writer == null) {
            writer = CQLSSTableWriter.builder().forTable(schema).using(insertStatement)
                    .withPartitioner(ConfigHelper.getOutputPartitioner(conf)).inDirectory(outputDir).sorted()
                    .build();
        }
        if (loader == null) {
            CrunchExternalClient externalClient = new CrunchExternalClient(conf);
            externalClient.addKnownCfs(keyspace, schema);
            this.loader = new SSTableLoader(outputDir, externalClient,
                    new BulkRecordWriter.NullOutputHandler());
        }
    } catch (Exception e) {
        throw new CrunchRuntimeException(e);
    }
}

---

From source file:com.spotify.hdfs2cass.cassandra.thrift.CrunchBulkRecordWriter.java

License:Apache License

private void prepareWriter() {
    String columnFamily = CrunchConfigHelper.getOutputColumnFamily(conf);
    String keyspace = ConfigHelper.getOutputKeyspace(conf);

    if (outputdir == null) {
        // dir must be named by ks/cf for the loader
        outputdir = Paths.get(getOutputLocation(), keyspace, columnFamily).toFile();
        outputdir.mkdirs();//from   w  w  w  .ja v  a 2s  . c  o  m
    }

    if (writer == null) {
        AbstractType<?> subcomparator = null;

        if (cfType == CFType.SUPER)
            subcomparator = BytesType.instance;

        int bufferSizeInMB = Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64"));
        this.writer = new SSTableSimpleUnsortedWriter(outputdir, ConfigHelper.getOutputPartitioner(conf),
                keyspace, columnFamily, BytesType.instance, subcomparator, bufferSizeInMB,
                ConfigHelper.getOutputCompressionParamaters(conf));

        ExternalSSTableLoaderClient externalClient = new ExternalSSTableLoaderClient(
                ConfigHelper.getOutputInitialAddress(conf), ConfigHelper.getOutputRpcPort(conf),
                ConfigHelper.getOutputKeyspaceUserName(conf), ConfigHelper.getOutputKeyspacePassword(conf));

        this.loader = new SSTableLoader(outputdir, externalClient, new OutputHandler.SystemOutput(true, true));
    }
}

---

From source file:de.hpi.isg.mdms.hadoop.cassandra.CqlBulkRecordWriter.java

License:Apache License

private void prepareWriter() throws IOException {
    try {//from   ww  w .j av  a 2 s .  c o  m
        if (writer == null) {
            writer = CQLSSTableWriter.builder().forTable(schema).using(insertStatement)
                    .withPartitioner(ConfigHelper.getOutputPartitioner(conf)).inDirectory(outputDir)
                    .withBufferSizeInMB(Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64"))).build();
        }
        if (loader == null) {
            ExternalClient externalClient = new ExternalClient(conf);

            this.loader = new SSTableLoader(outputDir, externalClient,
                    new BulkRecordWriter.NullOutputHandler()) {
                @Override
                public void onSuccess(StreamState finalState) {
                    if (deleteSrc)
                        FileUtils.deleteRecursive(outputDir);
                }
            };
        }
    } catch (Exception e) {
        throw new IOException(e);
    }
}

---

From source file:uk.co.pinpointlabs.io.CqlConcurrentBulkRecordWriter.java

License:Apache License

private void prepareWriter() throws IOException {
    try {//from   w  ww .j a v  a2s  .c  o  m
        if (writer == null) {
            writer = CQLSSTableWriter.builder().forTable(schema).using(insertStatement)
                    .withPartitioner(ConfigHelper.getOutputPartitioner(conf)).inDirectory(outputDir)
                    .withBufferSizeInMB(Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64"))).build();
        }
        if (loader == null) {
            ExternalClient externalClient = new ExternalClient(conf);

            externalClient.addKnownCfs(keyspace, schema);

            this.loader = new SSTableLoader(outputDir, externalClient,
                    new BulkRecordWriter.NullOutputHandler()) {
                @Override
                public void onSuccess(StreamState finalState) {
                    if (deleteSrc)
                        FileUtils.deleteRecursive(outputDir);
                }
            };
        }
    } catch (Exception e) {
        throw new IOException(e);
    }
}

---