List of usage examples for org.apache.hadoop.mapreduce.lib.output TextOutputFormat SEPERATOR
String SEPERATOR
To view the source code for org.apache.hadoop.mapreduce.lib.output TextOutputFormat SEPERATOR.
Click Source Link
From source file:co.cask.cdap.internal.app.runtime.spark.SparkAppUsingFileSet.java
License:Apache License
@Override public void configure() { try {// w w w. j av a 2 s . c o m createDataset("fs", FileSet.class, FileSetProperties.builder().setInputFormat(MyTextInputFormat.class) .setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); createDataset("pfs", PartitionedFileSet.class, PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder().addStringField("x").build()) .setInputFormat(MyTextInputFormat.class).setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); createDataset("tpfs", TimePartitionedFileSet.class, FileSetProperties.builder().setInputFormat(MyTextInputFormat.class) .setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); createDataset("myfs", MyFileSet.class, FileSetProperties.builder().setInputFormat(MyTextInputFormat.class) .setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); addSpark(new JavaCharCount()); addSpark(new ScalaCharCount()); } catch (Throwable t) { throw Throwables.propagate(t); } }
From source file:co.cask.cdap.partitioned.AppWithMapReduceConsumingPartitions.java
License:Apache License
@Override public void configure() { setName("AppWithMapReduceConsumingPartitions"); setDescription("Application with MapReduce job consuming partitions of a PartitionedFileSet Dataset"); createDataset("consumingState", KeyValueTable.class); createDataset("counts", IncrementingKeyValueTable.class); addMapReduce(new WordCount()); addService(new DatasetService()); // Create the "lines" partitioned file set, configure it to work with MapReduce createDataset("lines", PartitionedFileSet.class, PartitionedFileSetProperties.builder() // Properties for partitioning .setPartitioning(Partitioning.builder().addLongField("time").build()) // Properties for file set .setInputFormat(TextInputFormat.class).setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ",").build()); }
From source file:co.cask.cdap.partitioned.AppWithPartitionConsumers.java
License:Apache License
@Override public void configure() { setName("AppWithPartitionConsumers"); setDescription(/*from w w w. ja v a2 s . c om*/ "Application with MapReduce job and Worker consuming partitions of a PartitionedFileSet Dataset"); createDataset("consumingState", KeyValueTable.class); createDataset("counts", IncrementingKeyValueTable.class); addMapReduce(new WordCountMapReduce()); addWorker(new WordCountWorker()); addService(new DatasetService()); // Create the "lines" partitioned file set, configure it to work with MapReduce createDataset("lines", PartitionedFileSet.class, PartitionedFileSetProperties.builder() // Properties for partitioning .setPartitioning(Partitioning.builder().addLongField("time").build()) // Properties for file set .setInputFormat(TextInputFormat.class).setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ",").build()); // Create the "outputLines" partitioned file set, configure it to work with MapReduce createDataset("outputLines", PartitionedFileSet.class, PartitionedFileSetProperties.builder() // Properties for partitioning .setPartitioning(Partitioning.builder().addLongField("time").build()) // Properties for file set .setInputFormat(TextInputFormat.class).setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ",") // enable explore .setEnableExploreOnCreate(true).setExploreFormat("text").setExploreFormatProperty("delimiter", "\n") .setExploreSchema("record STRING").build()); }
From source file:co.cask.cdap.security.PartitionTestApp.java
License:Apache License
@Override public void configure() { addService(new PartitionService()); // Create a partitioned file set, configure it to work with MapReduce and with Explore createDataset("pfs", PartitionedFileSet.class, PartitionedFileSetProperties.builder() // Properties for partitioning .setPartitioning(/* w w w . ja va 2 s . c o m*/ Partitioning.builder().addStringField("partition").addIntField("sub-partition").build()) // Properties for file set .setInputFormat(TextInputFormat.class).setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ",") // Properties for Explore (to create a partitioned Hive table) .setEnableExploreOnCreate(true).setExploreFormat("csv").setExploreSchema("f1 STRING, f2 INT") .setDescription("App for testing authorization in partitioned filesets.").build()); }
From source file:co.cask.cdap.spark.app.SparkAppUsingFileSet.java
License:Apache License
@Override public void configure() { try {// w ww . j a va 2 s .co m createDataset("fs", FileSet.class, FileSetProperties.builder().setInputFormat(MyTextInputFormat.class) .setOutputFormat(MyTextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); createDataset("pfs", PartitionedFileSet.class, PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder().addStringField("x").build()) .setInputFormat(MyTextInputFormat.class).setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); createDataset("tpfs", TimePartitionedFileSet.class, FileSetProperties.builder().setInputFormat(MyTextInputFormat.class) .setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); createDataset("myfs", MyFileSet.class, FileSetProperties.builder().setInputFormat(MyTextInputFormat.class) .setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, ":").build()); addSpark(new FileCountSparkProgram()); addSpark(new ScalaFileCountSparkProgram()); } catch (Throwable t) { throw Throwables.propagate(t); } }
From source file:com.intel.hibench.DFSIO.java
License:Apache License
@Override public void configure() { setName("DFSIO"); setDescription("Benchmark Application with workload DFSIO"); createDataset("lines", FileSet.class, FileSetProperties.builder().setInputFormat(TextInputFormat.class) .setOutputFormat(TextOutputFormat.class).setOutputProperty(TextOutputFormat.SEPERATOR, " ") .build());//from ww w . j av a2 s. c om createDataset("benchData", Table.class); addMapReduce(new DFSIOWriter()); addService(new BenchService()); addService(new BenchUI()); }
From source file:com.intel.hibench.WordCountApp.java
License:Apache License
@Override public void configure() { setName("WordCount"); setDescription("Benchmark Application with workload wordCount"); createDataset("lines", FileSet.class, FileSetProperties.builder().setInputFormat(TextInputFormat.class) .setOutputFormat(TextOutputFormat.class).setOutputProperty(TextOutputFormat.SEPERATOR, " ") .build());/*from w w w . ja va 2 s. c o m*/ createDataset("counts", FileSet.class, FileSetProperties.builder().setInputFormat(TextInputFormat.class) .setOutputFormat(TextOutputFormat.class).setOutputProperty(TextOutputFormat.SEPERATOR, " ") .build()); createDataset("benchData", Table.class); addService(new FileSetService()); addMapReduce(new RandomTextWriter()); addMapReduce(new WordCount()); addService(new BenchService()); addService(new BenchUI()); }
From source file:com.toddbodnar.simpleHadoop.distributedHadoopDriver.java
/** * Runs a job//from w ww . ja v a2 s .co m * * @param theJob the MapReduceJob to be run * @param verbose if true, output progress information */ public static void run(MapReduceJob theJob, boolean verbose) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = GetConfiguration.get(); Job job = Job.getInstance(conf, theJob.toString()); job.setJarByClass(distributedHadoopDriver.class); job.setMapperClass(theJob.getMapper().getClass()); job.setReducerClass(theJob.getReducer().getClass()); job.setMapOutputKeyClass(theJob.getKeyType()); job.setMapOutputValueClass(theJob.getValueType()); theJob.writeConfig(job.getConfiguration()); hdfsFile input = hdfsFile.transferToHDFS(theJob.getInput().getFile()); if (!input.equals(theJob.getInput().getFile())) { garbage_collector.noteCreated(input); } if (theJob.getClass().equals(join.class)) { join jobLeftJoin = (join) theJob; hdfsFile input2 = hdfsFile.transferToHDFS(jobLeftJoin.getOtherInput().getFile()); if (!input2.equals(jobLeftJoin.getOtherInput().getFile())) { garbage_collector.noteCreated(input2); } Mapper maps[] = jobLeftJoin.getMapperPairs(); MultipleInputs.addInputPath(job, input.getPath(), TextInputFormat.class, maps[0].getClass()); MultipleInputs.addInputPath(job, input2.getPath(), TextInputFormat.class, maps[1].getClass()); } else { MultipleInputs.addInputPath(job, input.getPath(), TextInputFormat.class); } job.getConfiguration().set(TextOutputFormat.SEPERATOR, ""); job.setOutputFormatClass(TextOutputFormat.class); //FileInputFormat.setInputPaths(job, new Path(theJob.getInput().getFile().getLocation())); Path out = new Path(settings.hdfs_prefix + "/TMP_TABLE_" + theJob.hashCode()); FileOutputFormat.setOutputPath(job, out); boolean success = job.waitForCompletion(true); if (!success) { System.err.println("Error processing " + theJob); return; } FileSystem fs = FileSystem.get(GetConfiguration.get()); fs.delete(new Path(out, "_SUCCESS"), false); table output = new table(new hdfsFile(out), theJob.getOutput().getColNames()); output.setSeperator(theJob.getOutput().getSeperator()); theJob.setOutput(output); garbage_collector.noteCreated(output.getFile()); }
From source file:org.shaf.core.io.emulator.KeyValueTextReader.java
License:Apache License
/** * Reads the next line of text and forms a {@link Record record}. *///from ww w . j a va 2s .com @Override public Record<Text, Text> readRecord() throws IOException { String line = this.br.readLine(); if (line != null) { this.pos += (line.length() + System.lineSeparator().length()); String[] pair = line.split(super.config.get(TextOutputFormat.SEPERATOR, "\t")); return new Record<Text, Text>(new Text(pair[0].trim()), new Text(pair[1].trim())); } else { return null; } }
From source file:org.shaf.core.io.emulator.TextWriter.java
License:Apache License
/** * Writes a {@link Record record} as a text. *//*from www. j a v a 2s . co m*/ @Override public synchronized void writeRecord(Record<Writable, Writable> record) throws IOException { this.bw.append(record.getKey().toString()); this.bw.append(super.config.get(TextOutputFormat.SEPERATOR, "\t")); this.bw.append(record.getValue().toString()); this.bw.append(System.lineSeparator()); this.bw.flush(); }