Example usage for org.apache.spark.api.java.function MapFunction MapFunction

Introduction

In this page you can find the example usage for org.apache.spark.api.java.function MapFunction MapFunction.

Prototype

MapFunction

Source Link

Usage

From source file:sql.JavaSQLDataSourceExample.java

License:Apache License

private static void runBasicParquetExample(SparkSession spark) {
    // $example on:basic_parquet_example$
    Dataset<Row> peopleDF = spark.read()
            .json("/home/paul/spark/spark-2.1.0-bin-hadoop2.7/examples/src/main/resources/people.json");

    // DataFrames can be saved as Parquet files, maintaining the schema information
    peopleDF.write().parquet("people.parquet");

    // Read in the Parquet file created above.
    // Parquet files are self-describing so the schema is preserved
    // The result of loading a parquet file is also a DataFrame
    Dataset<Row> parquetFileDF = spark.read().parquet("people.parquet");

    // Parquet files can also be used to create a temporary view and then used in SQL statements
    parquetFileDF.createOrReplaceTempView("parquetFile");
    Dataset<Row> namesDF = spark.sql("SELECT name FROM parquetFile WHERE age BETWEEN 13 AND 19");
    Dataset<String> namesDS = namesDF.map(new MapFunction<Row, String>() {
        public String call(Row row) {
            return "Name: " + row.getString(0);
        }/*  w  w  w.  j a  va2 s.  c o m*/
    }, Encoders.STRING());
    namesDS.show();
    // +------------+
    // |       value|
    // +------------+
    // |Name: Justin|
    // +------------+
    // $example off:basic_parquet_example$
}