Example usage for org.apache.cassandra.config Config setClientMode

Introduction

In this page you can find the example usage for org.apache.cassandra.config Config setClientMode.

Prototype

@Deprecated
public static void setClientMode(boolean clientMode) 

Source Link

Document

Client mode means that the process is a pure client, that uses C* code base but does not read or write local C* database files.

Usage

From source file:biggraphite.BgGenerateCassandraSSTables.java

License:Apache License

public static void main(String[] args) throws IOException {
    if (args.length != 4) {
        System.out/*from w  w  w  .  j a v  a2  s  .co m*/
                .println("usage: java biggraphite.BgGenerateCassandraSSTables <KEYSPACE> <TABLE> <CQL> <CSV>");
        return;
    }
    final String keyspace = args[0];
    final String table = args[1];
    final String schema = new String(Files.readAllBytes(Paths.get(args[2])), StandardCharsets.UTF_8);
    final String data = args[3];
    final String insert_stmt = String.format(INSERT_STMT, keyspace, table);

    // magic!
    Config.setClientMode(true);

    // Create output directory that has keyspace and table name in the path
    File outputDir = new File(DEFAULT_OUTPUT_DIR + File.separator + keyspace + File.separator + table);
    if (!outputDir.exists() && !outputDir.mkdirs()) {
        throw new RuntimeException("Cannot create output directory: " + outputDir);
    }

    // Prepare SSTable writer
    CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder();
    // set output directory
    builder.inDirectory(outputDir).forTable(schema).using(insert_stmt)
            .withPartitioner(new Murmur3Partitioner());
    CQLSSTableWriter writer = builder.build();

    try (BufferedReader reader = new BufferedReader(new FileReader(data));
            CsvListReader csvReader = new CsvListReader(reader, CsvPreference.STANDARD_PREFERENCE)) {
        csvReader.getHeader(true);

        // Write to SSTable while reading data
        List<String> line;
        while ((line = csvReader.read()) != null) {
            // We use Java types here based on
            // http://www.datastax.com/drivers/java/2.0/com/datastax/driver/core/DataType.Name.html#asJavaClass%28%29
            writer.addRow(UUID.fromString(line.get(0)), // metric uuid
                    Long.parseLong(line.get(1)), // time_start_ms
                    Short.parseShort(line.get(2)), // offset
                    Double.parseDouble(line.get(3)), // value
                    Integer.parseInt(line.get(4))); // count
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

    try {
        writer.close();
    } catch (IOException ignore) {
    }
}

---

From source file:bulkload.CreateInvertedIndices.java

License:Apache License

public static void main(String[] args) throws IOException {

    if (args.length != 1) {
        System.out.println("usage: java bulkload.CreateInvertedIndices /path/to/free_base/data/");
        return;//from ww w  . j a  v  a2s  .co m
    }

    String Freebase_base = args[0];

    // magic!
    Config.setClientMode(true);

    stopWords = new StopWords(); // setup stop words

    // create inverted indexes
    Map<Long, IntArrayList> map = new HashMap<>();

    // load vocab table
    Map<Integer, String> vocabMap = UploadVocab.loadVocab(Freebase_base);

    // create main table
    {
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(
                new FileInputStream(Freebase_base + File.separator + "freebase_data.txt")))) {

            // Write to SSTable while reading data
            int id_counter = 1;
            String line;
            while ((line = reader.readLine()) != null) {
                String[] parts = line.split("\\|");

                if (parts.length == 3) {

                    List<Integer> lhs = toIntList(parts[0]);
                    List<Integer> rhs = toIntList(parts[2]);
                    int predicate = Integer.parseInt(parts[1]);

                    Set<Integer> set = new HashSet<>();
                    set.addAll(lhs);
                    set.addAll(rhs);
                    for (int id : set) {
                        addIndex(id, predicate, id_counter, vocabMap, map);
                    }

                    id_counter += 1;

                    if ((id_counter % 10_000_000) == 0) {
                        System.out.println(id_counter + ", size: " + map.size());
                    }

                }

            } // while loop

            System.out.println("saving map");
            save_hashmap(map);

        } catch (InvalidRequestException | IOException e) {
            e.printStackTrace();
        }

    }

    System.out.println("done");
    System.exit(0);

}

---

From source file:bulkload.DVDsLoad.java

License:Apache License

public static void main(String[] args) {
    if (args.length == 0) {
        System.out.println("usage: java bulkload.DVDsLoad <list of ticker symbols>");
        return;//from  w  w w.  j a v  a  2  s .  co m
    }

    // magic!
    Config.setClientMode(true);

    // Create output directory that has keyspace and table name in the path
    File outputDir = new File(DEFAULT_OUTPUT_DIR + File.separator + KEYSPACE + File.separator + TABLE);
    if (!outputDir.exists() && !outputDir.mkdirs()) {
        throw new RuntimeException("Cannot create output directory: " + outputDir);
    }

    // Prepare SSTable writer
    CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder();
    // set output directory
    builder.inDirectory(outputDir)
            // set target schema
            .forTable(SCHEMA)
            // set CQL statement to put data
            .using(INSERT_STMT)
            // set partitioner if needed
            // default is Murmur3Partitioner so set if you use different one.
            .withPartitioner(new Murmur3Partitioner());
    CQLSSTableWriter writer = builder.build();

    for (String ticker : args) {
        URLConnection conn;
        try {
            URL url = new URL(String.format(CSV_URL, ticker));
            conn = url.openConnection();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        try (BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
                CsvListReader csvReader = new CsvListReader(reader, CsvPreference.STANDARD_PREFERENCE)) {
            csvReader.getHeader(true);

            // Write to SSTable while reading data
            List<String> line;
            while ((line = csvReader.read()) != null) {
                // We use Java types here based on
                // http://www.datastax.com/drivers/java/2.0/com/datastax/driver/core/DataType.Name.html#asJavaClass%28%29
                writer.addRow(line.get(0) == null ? null : new String(line.get(0)),
                        line.get(1) == null ? null : new String(line.get(1)),
                        line.get(2) == null ? null : new String(line.get(2)),
                        line.get(3) == null ? null : new String(line.get(3)),
                        line.get(4) == null ? null : new String(line.get(4)),
                        line.get(5) == null ? null : new String(line.get(5)),
                        line.get(6) == null ? null : new String(line.get(6)),
                        line.get(7) == null ? null : new String(line.get(7)),
                        line.get(8) == null ? null : new String(line.get(8)),
                        line.get(9) == null ? null : new String(line.get(9)),
                        line.get(10) == null ? null : new String(line.get(10)),
                        line.get(11) == null ? null : new String(line.get(11)),
                        line.get(12) == null ? "0001-01-01" : new String(line.get(12)),
                        line.get(13) == null ? null : new Integer(line.get(13)),
                        line.get(14) == null ? null : new String(line.get(14)));
            }
        } catch (InvalidRequestException | IOException e) {
            e.printStackTrace();
        }
    }

    try {
        writer.close();
    } catch (IOException ignore) {
    }
}

---

From source file:bulkload.UploadIndexes.java

License:Apache License

public static void main(String[] args) {
    if (args.length != 1) {
        System.out.println(/*from ww w .jav  a 2s.  com*/
                "usage: java bulkload.UploadIndexes /path/to/inverted_indexes.txt (see bulkload.CreateInvertedIndices)");
        return;
    }

    String inverted_indexes_file = args[0];

    // magic!
    Config.setClientMode(true);

    // create inverted indexes
    {
        File outputDir2 = new File(Constants.KEYSPACE + File.separator + Constants.CF_INDEX);
        if (!outputDir2.exists() && !outputDir2.mkdirs()) {
            throw new RuntimeException("Cannot create output directory: " + outputDir2);
        }

        // Prepare SSTable writer
        CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder();
        // set output directory
        builder.inDirectory(outputDir2)
                // set target schema
                .forTable(SCHEMA_2)
                // set CQL statement to put data
                .using(INSERT_STMT_2)
                // set partitioner if needed
                // default is Murmur3Partitioner so set if you use different one.
                .withPartitioner(new Murmur3Partitioner());

        CQLSSTableWriter writer = builder.build();

        try (BufferedReader reader = new BufferedReader(
                new InputStreamReader(new FileInputStream(inverted_indexes_file)))) {

            int counter = 0;
            String line;
            while ((line = reader.readLine()) != null) {
                String[] parts = line.split(",");

                if (parts.length > 1) {

                    long id = Long.parseLong(parts[0]);
                    long l_word_id = id >> 32L;
                    int word_id = (int) l_word_id;
                    int predicate = (int) id;

                    List<Integer> intList = new ArrayList<>();
                    for (int i = 1; i < parts.length; i++) {
                        intList.add(Integer.parseInt(parts[i]));
                    }

                    Collections.sort(intList);

                    if (intList.size() > 0) {
                        try {
                            writer.addRow(predicate, word_id, intList);
                        } catch (InvalidRequestException | IOException e) {
                            e.printStackTrace();
                        }
                    }
                    counter += 1;
                    if ((counter % 1_000_000) == 0) {
                        System.out.println(counter);
                    }

                }

            } // while loop

        } catch (InvalidRequestException | IOException e) {
            e.printStackTrace();
        }

        try {
            writer.close();
        } catch (IOException ignore) {
        }
    }

    System.out.println("done");
    String path = Constants.KEYSPACE + File.separator + Constants.CF_INDEX;
    System.out.println("you can upload these files to Cassandra: sstableloader -d host " + path);
    System.exit(0);

}

---

From source file:bulkload.UploadTuples.java

License:Apache License

public static void main(String[] args) {
    if (args.length != 1) {
        System.out.println("usage: java bulkload.UploadTuples /path/to/free_base/data/");
        return;//from  w  ww .j  a va2 s  .com
    }

    String Freebase_base = args[0];

    // magic!
    Config.setClientMode(true);

    // create main table
    {
        // Create output directory that has keyspace and table name in the path
        File outputDir1 = new File(Constants.KEYSPACE + File.separator + Constants.CF_TUPLE);
        if (!outputDir1.exists() && !outputDir1.mkdirs()) {
            throw new RuntimeException("Cannot create output directory: " + outputDir1);
        }

        // Prepare SSTable writer
        CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder();
        // set output directory
        builder.inDirectory(outputDir1)
                // set target schema
                .forTable(SCHEMA_1)
                // set CQL statement to put data
                .using(INSERT_STMT_1)
                // set partitioner if needed
                // default is Murmur3Partitioner so set if you use different one.
                .withPartitioner(new Murmur3Partitioner());

        CQLSSTableWriter writer = builder.build();

        try (BufferedReader reader = new BufferedReader(new InputStreamReader(
                new FileInputStream(Freebase_base + File.separator + "freebase_data.txt")))) {

            // Write to SSTable while reading data
            int id_counter = 1;
            int hm_counter = 1;
            String line;
            while ((line = reader.readLine()) != null) {
                String[] parts = line.split("\\|");
                if (parts.length == 3) {
                    List<Integer> lhs = toIntList(parts[0]);
                    List<Integer> rhs = toIntList(parts[2]);
                    int predicate = Integer.parseInt(parts[1]);

                    writer.addRow(id_counter, lhs, predicate, rhs);

                    id_counter += 1;

                    if ((id_counter % 10_000_000) == 0) {
                        System.out.println(id_counter);
                    }

                }

            } // while loop

        } catch (InvalidRequestException | IOException e) {
            e.printStackTrace();
        }

        try {
            writer.close();
        } catch (IOException ignore) {
        }

    }

    System.out.println("done");
    String path = Constants.KEYSPACE + File.separator + Constants.CF_TUPLE;
    System.out.println("you can upload these files to Cassandra: sstableloader -d host " + path);
    System.exit(0);

}

---

From source file:bulkload.UploadVocab.java

License:Apache License

public static void main(String[] args) throws IOException {

    if (args.length != 1) {
        System.out.println("usage: java bulkload.UploadVocab /path/to/free_base/output/");
        return;//from w w w .ja v a 2  s . co m
    }

    String Freebase_base = args[0];

    // magic!
    Config.setClientMode(true);

    // create main table
    {
        // Create output directory that has Constants.KEYSPACE and table name in the path
        File outputDir1 = new File(Constants.KEYSPACE + File.separator + Constants.CF_VOCAB);
        if (!outputDir1.exists() && !outputDir1.mkdirs()) {
            throw new RuntimeException("Cannot create output directory: " + outputDir1);
        }

        File outputDir2 = new File(Constants.KEYSPACE + File.separator + Constants.CF_WORD);
        if (!outputDir2.exists() && !outputDir2.mkdirs()) {
            throw new RuntimeException("Cannot create output directory: " + outputDir2);
        }

        // read the predicate vocab map
        Map<Integer, String> predicateMap = new HashMap<>();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(
                new FileInputStream(Freebase_base + File.separator + "freebase_predicate_vocab.txt")))) {
            String line;
            // predicate_str|predicate_id
            while ((line = reader.readLine()) != null) {
                String[] parts = line.split("\\|");
                if (parts.length == 2) {
                    String word = parts[0];
                    int word_id = Integer.parseInt(parts[1]);
                    predicateMap.put(word_id, word);
                }
            } // while loop

        } catch (InvalidRequestException | IOException e) {
            e.printStackTrace();
        }

        // Prepare SSTable writer
        CQLSSTableWriter.Builder builder_1 = CQLSSTableWriter.builder();
        // set output directory
        builder_1.inDirectory(outputDir1)
                // set target schema
                .forTable(SCHEMA_1)
                // set CQL statement to put data
                .using(INSERT_STMT_1)
                // set partitioner if needed
                // default is Murmur3Partitioner so set if you use different one.
                .withPartitioner(new Murmur3Partitioner());

        CQLSSTableWriter writer_1 = builder_1.build();

        // Prepare SSTable writer
        CQLSSTableWriter.Builder builder_2 = CQLSSTableWriter.builder();
        // set output directory
        builder_2.inDirectory(outputDir2)
                // set target schema
                .forTable(SCHEMA_2)
                // set CQL statement to put data
                .using(INSERT_STMT_2)
                // set partitioner if needed
                // default is Murmur3Partitioner so set if you use different one.
                .withPartitioner(new Murmur3Partitioner());

        CQLSSTableWriter writer_2 = builder_2.build();

        Map<Integer, String> vocabMap = loadVocab(Freebase_base);
        for (int word_id : vocabMap.keySet()) {
            String word = vocabMap.get(word_id);
            writer_1.addRow(word_id, word, predicateMap.containsKey(word_id));
            writer_2.addRow(word, word_id, predicateMap.containsKey(word_id));
        }

        try {
            writer_1.close();
        } catch (IOException ignore) {
        }

        try {
            writer_2.close();
        } catch (IOException ignore) {
        }

    }

    System.out.println("done");
    String path_1 = Constants.KEYSPACE + File.separator + Constants.CF_VOCAB;
    System.out.println("you can upload these files to Cassandra:\nsstableloader -d host " + path_1);

    String path_2 = Constants.KEYSPACE + File.separator + Constants.CF_WORD;
    System.out.println("sstableloader -d host " + path_2);

    System.exit(0);

}

---

From source file:com.criteo.biggraphite.BgGenerateCassandraSSTables.java

License:Apache License

/**
 * Utility to write Cassandra SSTables./*from w w  w .  j  a v  a 2  s. c om*/
 *
 * @param args <KEYSPACE> <TABLE> <CQL> <CSV>
 * @throws IOException if an I/O error occurs reading from the stream
 */
public static void main(String[] args) throws IOException {

    if (args.length != 4) {
        System.out
                .println("usage: java biggraphite.BgGenerateCassandraSSTables <KEYSPACE> <TABLE> <CQL> <CSV>");
        return;
    }
    final String keyspace = args[0];
    final String table = args[1];
    final String schema = new String(Files.readAllBytes(Paths.get(args[2])), StandardCharsets.UTF_8);
    final String data = args[3];
    final String insert_stmt = String.format(INSERT_STMT, keyspace, table);

    // magic!
    Config.setClientMode(true);

    // Create output directory that has keyspace and table name in the path
    final File outputDir = Paths.get(DEFAULT_OUTPUT_DIR, keyspace, table).toFile();
    if (!outputDir.exists() && !outputDir.mkdirs()) {
        throw new RuntimeException("Cannot create output directory: " + outputDir);
    }

    // Prepare SSTable writer
    final CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder().inDirectory(outputDir) // the directory where to write the sstables
            .forTable(schema) // the schema (CREATE TABLE statement) for the table for which sstable are to be created
            .using(insert_stmt) // the INSERT statement defining the order of the values to add for a given CQL row
            .withPartitioner(new Murmur3Partitioner());

    try (CQLSSTableWriter writer = builder.build();
            BufferedReader reader = new BufferedReader(new FileReader(data));
            CsvListReader csvReader = new CsvListReader(reader, CsvPreference.STANDARD_PREFERENCE)) {
        // import_whisper don't generate any header, so we should NOT skip the first line
        //csvReader.getHeader(true);

        // Write to SSTable while reading data
        List<String> line;
        while ((line = csvReader.read()) != null) {
            // We use Java types here based on
            // http://www.datastax.com/drivers/java/2.0/com/datastax/driver/core/DataType.Name.html#asJavaClass%28%29
            writer.addRow(UUID.fromString(line.get(0)), // metric uuid
                    Long.parseLong(line.get(1)), // time_start_ms
                    Short.parseShort(line.get(2)), // offset
                    parseDouble(line.get(3)), // value
                    Integer.parseInt(line.get(4))); // count
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}

---

From source file:com.scylladb.tools.BulkLoader.java

License:Apache License

public static void main(String args[]) {
    Config.setClientMode(true);
    LoaderOptions options = LoaderOptions.parseArgs(args);

    try {//ww w  .  j a va2 s.c  om
        File dir = options.directory;
        if (dir.isFile()) {
            dir = dir.getParentFile();
        }

        String keyspace = dir.getParentFile().getName();

        CQLClient client = new CQLClient(options, keyspace);
        SSTableToCQL ssTableToCQL = new SSTableToCQL(keyspace, client);
        ssTableToCQL.stream(options.directory);
        System.exit(0);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(1);
    }
}

---

From source file:com.spotify.cassandra.opstools.CountTombstones.java

License:Apache License

/**
 * Counts the number of tombstones, per row, in a given SSTable
 *
 * Assumes RandomPartitioner, standard columns and UTF8 encoded row keys
 *
 * Does not require a cassandra.yaml file or system tables.
 *
 * @param args command lines arguments//from   ww  w.ja va2  s.c om
 *
 * @throws java.io.IOException on failure to open/read/write files or output streams
 */
public static void main(String[] args) throws IOException, ParseException {
    String usage = String.format("Usage: %s [-l] <sstable> [<sstable> ...]%n", CountTombstones.class.getName());

    final Options options = new Options();
    options.addOption("l", "legend", false, "Include column name explanation");
    options.addOption("p", "partitioner", true, "The partitioner used by database");

    CommandLineParser parser = new BasicParser();
    CommandLine cmd = parser.parse(options, args);

    if (cmd.getArgs().length < 1) {
        System.err.println("You must supply at least one sstable");
        System.err.println(usage);
        System.exit(1);
    }

    // Fake DatabaseDescriptor settings so we don't have to load cassandra.yaml etc
    Config.setClientMode(true);
    String partitionerName = String.format("org.apache.cassandra.dht.%s",
            cmd.hasOption("p") ? cmd.getOptionValue("p") : "RandomPartitioner");
    try {
        Class<?> clazz = Class.forName(partitionerName);
        IPartitioner partitioner = (IPartitioner) clazz.newInstance();
        DatabaseDescriptor.setPartitioner(partitioner);
    } catch (Exception e) {
        throw new RuntimeException("Can't instantiate partitioner " + partitionerName);
    }

    PrintStream out = System.out;

    for (String arg : cmd.getArgs()) {
        String ssTableFileName = new File(arg).getAbsolutePath();

        Descriptor descriptor = Descriptor.fromFilename(ssTableFileName);

        run(descriptor, cmd, out);
    }

    System.exit(0);
}

---

From source file:com.spotify.hdfs2cass.cassandra.thrift.CrunchBulkRecordWriter.java

License:Apache License

public CrunchBulkRecordWriter(TaskAttemptContext context) {
    Config.setClientMode(true);
    Config.setOutboundBindAny(true);/*  www  .  j  a  v a 2 s  .c om*/
    this.conf = HadoopCompat.getConfiguration(context);
    this.context = context;
    int megabitsPerSec = Integer.parseInt(conf.get(STREAM_THROTTLE_MBITS, "0"));
    DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(megabitsPerSec);
    heartbeat = new ProgressHeartbeat(context, 120);
}

---

• «
• 1
• 2
• »