List of usage examples for org.apache.hadoop.util Progressable Progressable
Progressable
From source file:batch.BatchScan2Html.java
License:Apache License
public static void writeAccumuloTableToHdfsAsHtml() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); //TODO add options for URI and output Path FileSystem hdfs = FileSystem.get(new URI("hdfs://n001:54310"), configuration); Path file = new Path("hdfs://n001:54310/s2013/batch/table.html"); //TODO add option to override file default: true if (hdfs.exists(file)) { hdfs.delete(file, true);/*from w w w .j a va 2 s . c om*/ } startTime = System.currentTimeMillis(); OutputStream os = hdfs.create(file, new Progressable() { public void progress() { // TODO add a better progress descriptor crudeRunTime = System.currentTimeMillis() - startTime; out.println("...bytes written: [ " + bytesWritten + " ]"); out.println("...bytes / second: [ " + (bytesWritten / crudeRunTime) * 1000 + " ]"); } }); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); // TODO add option for table id { example } writeHtmlTableHeader(br, "example", new ArrayList<String>(Arrays.asList("Row ID", "Column Family", "Column Qualifier", "Column Visibility", "Timestamp", "Value"))); writeHtmlTableBody(br); out.println("Total bytes written: " + bytesWritten); out.println("Total crude time: " + crudeRunTime / 1000); br.close(); hdfs.close(); }
From source file:cn.lhfei.hadoop.ch03.FileCopyWithProgress.java
License:Apache License
public static void main(String[] args) { String localSrc = args[0];//from w ww .j a v a 2 s . c o m String dst = args[1]; FileSystem fs = null; InputStream in = null; OutputStream out = null; try { Configuration conf = new Configuration(); fs = FileSystem.get(URI.create(localSrc), conf); in = new BufferedInputStream(new FileInputStream(localSrc)); out = fs.create(new Path(dst), new Progressable() { @Override public void progress() { log.info("... ..."); } }); IOUtils.copyBytes(in, out, 4096, true); } catch (FileNotFoundException e) { // e.printStackTrace(); log.error(e.getMessage(), e); } catch (IOException e) { // e.printStackTrace(); log.error(e.getMessage(), e); } finally { IOUtils.closeStream(in); IOUtils.closeStream(out); } }
From source file:com.bah.culvert.CulvertOutputFormatTest.java
License:Apache License
@Ignore(value = "Filed as bug. #375") @Test/* w ww .j a v a 2s . co m*/ public void testBasicOperation() throws Throwable { CulvertOutputFormat format = new CulvertOutputFormat(); // the client only works with configurations JobConf conf = new JobConf(); Client.setDatabaseAdapter(conf, InMemoryDB.class); /* * most of the stuff we set in the table properties because we use the * jobconf earlier for input stuff */ Properties tblProps = CulvertHiveUtils.confToProps(conf); InMemoryDB db = new InMemoryDB(); CColumn col = new CColumn("foo".getBytes(), "bar".getBytes()); db.create("baz", Arrays.asList(col)); CulvertHiveUtils.setCulvertConfigurationIsEmbedded(tblProps, true); CulvertHiveUtils.setCulvertTable(tblProps, "baz"); final int[] i = { 0 }; Progressable progress = new Progressable() { @Override public void progress() { i[0]++; } }; RecordWriter writer = format.getHiveRecordWriter(conf, null, Put.class, true, tblProps, progress); writer.write(new Put(new CKeyValue("a".getBytes(), "b".getBytes(), "c".getBytes(), "d".getBytes()))); Assert.assertEquals(1, i[0]); SeekingCurrentIterator it = db.getTableAdapter("baz").get(new Get(new CRange("a".getBytes()))); // this is failing - looks like stuff has been put but isn't coming out of // the get Assert.assertTrue("Iterator should have a next value", it.hasNext()); Result next = it.next(); Assert.assertTrue("Result row should be 'a' byte equivalent", Arrays.equals("a".getBytes(), next.getRecordId())); Assert.assertTrue("Result should be ", Arrays.equals("d".getBytes(), next.getValue("b".getBytes(), "c".getBytes()).getValue())); }
From source file:com.digitalpebble.behemoth.solr.TestSOLRWriter.java
License:Apache License
@Test public void testFieldMappings() throws IOException { JobConf conf = new JobConf(); conf.set("solr.server.url", "http://example.org"); conf.set("solr.f.person", "Person.string"); conf.set("solr.f.personTitle", "Person.title"); conf.set("solr.f.location", "Location"); Progressable progress = new Progressable() { @Override//from w w w . jav a2 s.co m public void progress() { } }; SOLRWriter writer = new SOLRWriter(progress); writer.open(conf, "test"); assertEquals(writer.getFieldMapping().size(), 2); assertNotNull(writer.getFieldMapping().get("Person")); assertEquals(writer.getFieldMapping().get("Person").size(), 2); assertEquals(writer.getFieldMapping().get("Person").get("string"), "person"); assertEquals(writer.getFieldMapping().get("Person").get("title"), "personTitle"); assertNotNull(writer.getFieldMapping().get("Location")); assertEquals(writer.getFieldMapping().get("Location").size(), 1); assertEquals(writer.getFieldMapping().get("Location").get("*"), "location"); }
From source file:com.ema.hadoop.test_hdfs.TestWrite.java
public static void main(String[] args) throws IOException, URISyntaxException { Configuration configuration = new Configuration(); FileSystem hdfs = FileSystem.get(new URI("hdfs://localhost:9000"), configuration); Path file = new Path("hdfs://localhost:9000/user/student/text_file_write.txt"); if (hdfs.exists(file)) { hdfs.delete(file, true);// w w w . j av a 2 s. c om } OutputStream os = hdfs.create(file, new Progressable() { @Override public void progress() { out.println("...bytes written"); } }); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); br.write("This is just a test to check if it is possible to write a file on HDFS using the Java API"); br.close(); hdfs.close(); }
From source file:com.ema.hadoop.wordcount.WordCount_cache.java
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: WordCount <input path> <output path>"); System.exit(-1);//from w ww . j av a 2 s . c om } // First we write the stop word list // it could also be a file manually loaded into HDFS String[] stopwords = { "the", "a" }; Configuration configuration = new Configuration(); FileSystem hdfs = FileSystem.get(new URI("hdfs://localhost:9000"), configuration); Path file = new Path("hdfs://localhost:9000/user/student/stop_words.txt"); if (hdfs.exists(file)) { hdfs.delete(file, true); } OutputStream os = hdfs.create(file, new Progressable() { @Override public void progress() { out.println("...bytes written"); } }); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); for (String w : stopwords) { br.write(w + "\n"); } br.close(); hdfs.close(); Job job = Job.getInstance(); job.addCacheFile(new Path("hdfs://localhost:9000/user/student/stop_words.txt").toUri()); job.setJarByClass(WordCount_cache.class); job.setJobName("Word count job"); FileInputFormat.setInputPaths(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setMapperClass(WCMapper_cache.class); job.setReducerClass(WCReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.facebook.presto.hive.AbstractTestHiveFileFormats.java
License:Apache License
public FileSplit createTestFile(String filePath, HiveOutputFormat<?, ?> outputFormat, @SuppressWarnings("deprecation") SerDe serDe, String compressionCodec) throws Exception { JobConf jobConf = new JobConf(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", COLUMN_NAMES_STRING); tableProperties.setProperty("columns.types", COLUMN_TYPES); serDe.initialize(new Configuration(), tableProperties); if (compressionCodec != null) { CompressionCodec codec = new CompressionCodecFactory(new Configuration()) .getCodecByName(compressionCodec); jobConf.set(COMPRESS_CODEC, codec.getClass().getName()); jobConf.set(COMPRESS_TYPE, SequenceFile.CompressionType.BLOCK.toString()); }/*w ww .j a v a2 s . com*/ RecordWriter recordWriter = outputFormat.getHiveRecordWriter(jobConf, new Path(filePath), Text.class, compressionCodec != null, tableProperties, new Progressable() { @Override public void progress() { } }); try { serDe.initialize(new Configuration(), tableProperties); SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(COLUMN_NAMES, FIELD_INSPECTORS); Object row = objectInspector.create(); List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs()); for (int rowNumber = 0; rowNumber < NUM_ROWS; rowNumber++) { for (int i = 0; i < TEST_VALUES.size(); i++) { Object key = TEST_VALUES.get(i).getKey(); if (key instanceof Slice) { key = ((Slice) key).getBytes(); } objectInspector.setStructFieldData(row, fields.get(i), key); } Writable record = serDe.serialize(row, objectInspector); recordWriter.write(record); } } finally { recordWriter.close(false); } Path path = new Path(filePath); path.getFileSystem(new Configuration()).setVerifyChecksum(true); File file = new File(filePath); return new FileSplit(path, 0, file.length(), new String[0]); }
From source file:com.facebook.presto.hive.BenchmarkHiveFileFormats.java
License:Apache License
public static RecordWriter createRecordWriter(List<? extends TpchColumn<?>> columns, File outputFile, HiveOutputFormat<?, ?> outputFormat, CompressionType compressionCodec) throws Exception { JobConf jobConf = new JobConf(); ReaderWriterProfiler.setProfilerOptions(jobConf); if (compressionCodec != CompressionType.none) { CompressionCodec codec = new CompressionCodecFactory(new Configuration()) .getCodecByName(compressionCodec.toString()); jobConf.set(COMPRESS_CODEC, codec.getClass().getName()); jobConf.set(COMPRESS_TYPE, org.apache.hadoop.io.SequenceFile.CompressionType.BLOCK.toString()); jobConf.set("parquet.compression", compressionCodec.toString()); jobConf.set("parquet.enable.dictionary", "true"); switch (compressionCodec) { case gzip: jobConf.set("hive.exec.orc.default.compress", "ZLIB"); jobConf.set("hive.exec.orc.compress", "ZLIB"); break; case snappy: jobConf.set("hive.exec.orc.default.compress", "SNAPPY"); jobConf.set("hive.exec.orc.compress", "SNAPPY"); break; default:/*from ww w .ja v a2 s.co m*/ throw new IllegalArgumentException("Unsupported compression codec: " + compressionCodec); } } else { jobConf.set("parquet.enable.dictionary", "true"); jobConf.set("hive.exec.orc.default.compress", "NONE"); jobConf.set("hive.exec.orc.compress", "NONE"); } RecordWriter recordWriter = outputFormat.getHiveRecordWriter(jobConf, new Path(outputFile.toURI()), Text.class, compressionCodec != CompressionType.none, createTableProperties(columns), new Progressable() { @Override public void progress() { } }); return recordWriter; }
From source file:com.jeffy.hdfs.HDFSWriteFile.java
License:Apache License
/** * ??hdfs/* w w w. j a v a2 s . c om*/ * * @param args */ public static void main(String[] args) { if (args.length < 2) { System.err.println("Please input two parameter!"); System.out.println("Parameter: localfile hdfsfile"); System.exit(1); } String localPath = args[0]; String hdfsPath = args[1]; //?? Configuration config = new Configuration(); //?? try (InputStream in = new BufferedInputStream(new FileInputStream(localPath))) { FileSystem fs = FileSystem.get(URI.create(hdfsPath), config); try (FSDataOutputStream out = fs.create(new Path(hdfsPath), new Progressable() { @Override public void progress() { System.out.println("."); } })) { //??OutputStream,Hadooporg.apache.commons.io.IOUtils IOUtils.copy(in, out); System.out.println("File copy finished."); } } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
From source file:com.netflix.bdp.s3mper.listing.BigTableGcsConsistentListingAspectTest.java
License:Apache License
@Test public void testFileCreateMethods() throws Throwable { System.out.println("testFileCreateMethods"); Path file = new Path(testPath + "/create-methods.test"); //create(Path) OutputStream fout = deleteFs.create(file); assertNotNull(fout);/* w ww .jav a 2 s . co m*/ fout.close(); List<FileInfo> files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(Path, Progressable)"); fout = deleteFs.create(file, new Progressable() { @Override public void progress() { } }); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(Path, boolean)"); fout = deleteFs.create(file, true); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(Path, short)"); fout = deleteFs.create(file, (short) 1); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(Path, boolean, int)"); fout = deleteFs.create(file, true, 4096); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(FileSystem, Path, FsPermission)"); fout = deleteFs.create(deleteFs, file, FsPermission.getDefault()); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(FileSystem, short, Progressable)"); fout = deleteFs.create(file, (short) 1, new Progressable() { @Override public void progress() { } }); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(FileSystem, boolean, int, Progressable)"); fout = deleteFs.create(file, true, 4096, new Progressable() { @Override public void progress() { } }); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(FileSystem, boolean, int, short, long)"); fout = deleteFs.create(file, true, 4096, (short) 1, 100000000); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); System.out.println("create(FileSystem, boolean, int, short, long, Progressable)"); fout = deleteFs.create(file, true, 4096, (short) 1, 100000000, new Progressable() { @Override public void progress() { } }); assertNotNull(fout); fout.close(); files = meta.list(Collections.singletonList(file.getParent())); assertEquals(1, files.size()); deleteFs.delete(file.getParent(), true); //janitor.clearPath(testPath); }