List of usage examples for org.apache.hadoop.io IOUtils closeStream
public static void closeStream(java.io.Closeable stream)
From source file:nl.basjes.utils.TestWritableInterface.java
License:Apache License
/** * Converts a byte[] back into an instance of the specified class (assuming it was created with the above method serialize). * @param bytes/*from w w w . j av a 2s .c om*/ * @param clazz * @return * @throws IOException */ public static <T extends Writable> T asWritable(byte[] bytes, Class<T> clazz) throws IOException { T result = null; DataInputStream dataIn = null; try { result = clazz.newInstance(); ByteArrayInputStream in = new ByteArrayInputStream(bytes); dataIn = new DataInputStream(in); result.readFields(dataIn); } catch (InstantiationException e) { // should not happen assert false; } catch (IllegalAccessException e) { // should not happen assert false; } finally { IOUtils.closeStream(dataIn); } return result; }
From source file:nl.gridline.zieook.inx.movielens.AggregateAndRecommendReducer.java
License:Apache License
@Override protected void setup(Context context) throws IOException { Configuration jobConf = context.getConfiguration(); recommendationsPerUser = jobConf.getInt(NUM_RECOMMENDATIONS, DEFAULT_NUM_RECOMMENDATIONS); booleanData = jobConf.getBoolean(RecommenderJob.BOOLEAN_DATA, false); indexItemIDMap = TasteHadoopUtils.readItemIDIndexMap(jobConf.get(ITEMID_INDEX_PATH), jobConf); FSDataInputStream in = null;//w ww . j a v a 2 s. c o m try { String itemFilePathString = jobConf.get(ITEMS_FILE); if (itemFilePathString == null) { itemsToRecommendFor = null; } else { Path unqualifiedItemsFilePath = new Path(itemFilePathString); FileSystem fs = FileSystem.get(unqualifiedItemsFilePath.toUri(), jobConf); itemsToRecommendFor = new FastIDSet(); Path itemsFilePath = unqualifiedItemsFilePath.makeQualified(fs); in = fs.open(itemsFilePath); for (String line : new FileLineIterable(in)) { itemsToRecommendFor.add(Long.parseLong(line)); } } } finally { IOUtils.closeStream(in); } }
From source file:nl.gridline.zieook.inx.movielens.UserVectorSplitterMapper.java
License:Apache License
@Override protected void setup(Context context) throws IOException { Configuration jobConf = context.getConfiguration(); maxPrefsPerUserConsidered = jobConf.getInt(MAX_PREFS_PER_USER_CONSIDERED, DEFAULT_MAX_PREFS_PER_USER_CONSIDERED); String usersFilePathString = jobConf.get(USERS_FILE); if (usersFilePathString != null) { FSDataInputStream in = null;/* w w w. ja v a 2s .c om*/ try { Path unqualifiedUsersFilePath = new Path(usersFilePathString); FileSystem fs = FileSystem.get(unqualifiedUsersFilePath.toUri(), jobConf); usersToRecommendFor = new FastIDSet(); Path usersFilePath = unqualifiedUsersFilePath.makeQualified(fs); in = fs.open(usersFilePath); for (String line : new FileLineIterable(in)) { usersToRecommendFor.add(Long.parseLong(line)); } } finally { IOUtils.closeStream(in); } } }
From source file:nl.gridline.zieook.statistics.popularity.PopularityStatistics.java
License:Apache License
public static String readData(Path path, Configuration conf) throws IOException { FileSystem fs = path.getFileSystem(conf); Path outputFile = fs.listStatus(path, PathFilters.partFilter())[0].getPath(); InputStream in = null;/*from w ww .j a v a 2s. c o m*/ try { in = fs.open(outputFile); ByteArrayOutputStream out = new ByteArrayOutputStream(); IOUtils.copyBytes(in, out, conf); return new String(out.toByteArray(), Charsets.UTF_8).trim(); } finally { IOUtils.closeStream(in); } }
From source file:org.acacia.partitioner.java.WholeFileRecordReader.java
License:Apache License
@Override public boolean next(NullWritable key, BytesWritable value) throws IOException { if (!processed) { byte[] contents = new byte[(int) fileSplit.getLength()]; Path file = fileSplit.getPath(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream in = null;/*from w w w .ja va 2s .c o m*/ try { in = fs.open(file); IOUtils.readFully(in, contents, 0, contents.length); value.set(contents, 0, contents.length); } finally { IOUtils.closeStream(in); } processed = true; return true; } return false; }
From source file:org.apache.camel.component.hdfs.HdfsInputStream.java
License:Apache License
@Override public final void close() throws IOException { if (opened) { IOUtils.closeStream(in); HdfsInfo info = HdfsInfoFactory.newHdfsInfo(actualPath); info.getFileSystem().rename(new Path(suffixedPath), new Path(actualPath + '.' + HdfsConstants.DEFAULT_READ_SUFFIX)); opened = false;/* w w w . jav a 2s . c om*/ } }
From source file:org.apache.camel.component.hdfs.HdfsOutputStream.java
License:Apache License
@Override public void close() throws IOException { if (opened) { IOUtils.closeStream(out); info.getFileSystem().rename(new Path(suffixedPath), new Path(actualPath)); opened = false;/*from ww w .j a v a2s .c o m*/ } }
From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java
License:Apache License
@Test public void testSimpleWriteFile() throws Exception { if (SKIP) {//from w w w . j a va 2 s .c o m return; } final Path file = new Path(new File("target/test/test-camel-simple-write-file").getAbsolutePath()); deleteDirectory("target/file-batch1"); context.addRoutes(new RouteBuilder() { @Override public void configure() throws Exception { from("file://target/file-batch1?sortBy=file:name") .to("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL"); } }); context.start(); NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create(); for (int i = 0; i < 10; ++i) { template.sendBodyAndHeader("file://target/file-batch1/", "CIAO", "CamelFileName", "CIAO" + i); } Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime()); context.stop(); InputStream in = null; try { in = new URL("file:///" + file.toUri()).openStream(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); IOUtils.copyBytes(in, bos, 4096, false); Assert.assertEquals(40, bos.size()); } finally { IOUtils.closeStream(in); } }
From source file:org.apache.flink.test.hadoop.mapred.HadoopIOFormatsITCase.java
License:Apache License
@Override protected void preSubmit() throws Exception { resultPath = new String[] { getTempDirPath("result0"), getTempDirPath("result1") }; File sequenceFile = createAndRegisterTempFile("seqFile"); sequenceFileInPath = sequenceFile.toURI().toString(); // Create a sequence file org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); FileSystem fs = FileSystem.get(URI.create(sequenceFile.getAbsolutePath()), conf); Path path = new Path(sequenceFile.getAbsolutePath()); // ------------------ Long / Text Key Value pair: ------------ int kvCount = 4; LongWritable key = new LongWritable(); Text value = new Text(); SequenceFile.Writer writer = null; try {/*from w w w. jav a 2s . c om*/ writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass()); for (int i = 0; i < kvCount; i++) { if (i == 1) { // write key = 0 a bit more often. for (int a = 0; a < 15; a++) { key.set(i); value.set(i + " - somestring"); writer.append(key, value); } } key.set(i); value.set(i + " - somestring"); writer.append(key, value); } } finally { IOUtils.closeStream(writer); } // ------------------ Long / Text Key Value pair: ------------ File sequenceFileNull = createAndRegisterTempFile("seqFileNullKey"); sequenceFileInPathNull = sequenceFileNull.toURI().toString(); path = new Path(sequenceFileInPathNull); LongWritable value1 = new LongWritable(); SequenceFile.Writer writer1 = null; try { writer1 = SequenceFile.createWriter(fs, conf, path, NullWritable.class, value1.getClass()); for (int i = 0; i < kvCount; i++) { value1.set(i); writer1.append(NullWritable.get(), value1); } } finally { IOUtils.closeStream(writer1); } }
From source file:org.apache.giraffa.TestLeaseManagement.java
License:Apache License
@Test public void testLeaseCreation() throws IOException { String src = "/testLeaseCreation"; Path path = new Path(src); long currentTime = Time.now(); FSDataOutputStream outputStream = grfs.create(path); try {/* w w w.ja v a 2 s. co m*/ // check lease exists after creation checkLease(src, currentTime); // check lease renew currentTime = Time.now(); grfs.grfaClient.getNamespaceService().renewLease(grfs.grfaClient.getClientName()); checkLease(src, currentTime); } finally { IOUtils.closeStream(outputStream); } INodeFile iNode = INodeFile.valueOf(nodeManager.getINode(src)); assertThat(iNode.getFileState(), is(FileState.CLOSED)); FileLease lease = iNode.getLease(); assertThat(lease, is(nullValue())); }