List of usage examples for org.apache.hadoop.fs FSDataInputStream read
@Override
public int read(ByteBuffer buf) throws IOException
From source file:org.apache.ambari.view.filebrowser.DownloadService.java
License:Apache License
/** * Concatenate files/* www. j av a 2 s . c o m*/ * @param request download request * @return response with all files concatenated */ @POST @Path("/concat") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_OCTET_STREAM) public Response concat(final DownloadRequest request) { try { StreamingOutput result = new StreamingOutput() { public void write(OutputStream output) throws IOException, ServiceFormattedException { FSDataInputStream in = null; for (String path : request.entries) { try { in = getApi(context).open(path); byte[] chunk = new byte[1024]; while (in.read(chunk) != -1) { output.write(chunk); } } catch (Exception ex) { throw new ServiceFormattedException(ex.getMessage(), ex); } finally { if (in != null) in.close(); } } } }; ResponseBuilder response = Response.ok(result); if (request.download) { response.header("Content-Disposition", "inline; filename=\"concatResult.txt\"") .type(MediaType.APPLICATION_OCTET_STREAM); } else { response.header("Content-Disposition", "filename=\"concatResult.txt\"").type(MediaType.TEXT_PLAIN); } return response.build(); } catch (WebApplicationException ex) { throw ex; } catch (Exception ex) { throw new ServiceFormattedException(ex.getMessage(), ex); } }
From source file:org.apache.ambari.view.hive.resources.files.FileServiceTest.java
License:Apache License
private void assertHDFSFileContains(String filePath, String expectedContent) throws IOException { FSDataInputStream fileInputStream = hdfsCluster.getFileSystem().open(new Path(filePath)); byte[] buffer = new byte[256]; int read = fileInputStream.read(buffer); byte[] readData = Arrays.copyOfRange(buffer, 0, read); String actualContent = new String(readData, Charset.forName("UTF-8")); Assert.assertEquals(expectedContent, actualContent); }
From source file:org.apache.apex.malhar.lib.io.fs.AbstractFileOutputOperator.java
License:Apache License
/** * Recovers a file which exists on the disk. If the length of the file is not same as the * length which the operator remembers then the file is truncated. <br/> * When always writing to a temporary file, then a file is restored even when the length is same as what the * operator remembers however this is done only for files which had open streams that weren't closed before * failure./*from www. j av a2s .com*/ * * @param filename name of the actual file. * @param partFileName name of the part file. When not rolling this is same as filename; otherwise this is the * latest open part file name. * @param filepath path of the file. When always writing to temp file, this is the path of the temp file; * otherwise path of the actual file. * @throws IOException */ private void recoverFile(String filename, String partFileName, Path filepath) throws IOException { LOG.debug("path exists {}", filepath); long offset = endOffsets.get(filename).longValue(); FSDataInputStream inputStream = fs.open(filepath); FileStatus status = fs.getFileStatus(filepath); if (status.getLen() != offset) { LOG.info("path corrupted {} {} {}", filepath, offset, status.getLen()); byte[] buffer = new byte[COPY_BUFFER_SIZE]; String recoveryFileName = partFileName + '.' + System.currentTimeMillis() + TMP_EXTENSION; Path recoveryFilePath = new Path(filePath + Path.SEPARATOR + recoveryFileName); FSDataOutputStream fsOutput = openStream(recoveryFilePath, false); while (inputStream.getPos() < offset) { long remainingBytes = offset - inputStream.getPos(); int bytesToWrite = remainingBytes < COPY_BUFFER_SIZE ? (int) remainingBytes : COPY_BUFFER_SIZE; inputStream.read(buffer); fsOutput.write(buffer, 0, bytesToWrite); } flush(fsOutput); fsOutput.close(); inputStream.close(); LOG.debug("active {} recovery {} ", filepath, recoveryFilePath); if (alwaysWriteToTmp) { //recovery file is used as the new tmp file and we cannot delete the old tmp file because when the operator //is restored to an earlier check-pointed window, it will look for an older tmp. fileNameToTmpName.put(partFileName, recoveryFileName); } else { LOG.debug("recovery path {} actual path {} ", recoveryFilePath, status.getPath()); rename(recoveryFilePath, status.getPath()); } } else { if (alwaysWriteToTmp && filesWithOpenStreams.contains(filename)) { String currentTmp = partFileName + '.' + System.currentTimeMillis() + TMP_EXTENSION; FSDataOutputStream outputStream = openStream(new Path(filePath + Path.SEPARATOR + currentTmp), false); IOUtils.copy(inputStream, outputStream); streamsCache.put(filename, new FSFilterStreamContext(outputStream)); fileNameToTmpName.put(partFileName, currentTmp); } inputStream.close(); } }
From source file:org.apache.camel.component.hdfs.integration.HdfsAppendTest.java
License:Apache License
@Test public void testAppend() throws Exception { context.addRoutes(new RouteBuilder() { @Override//from w w w . j a va 2s. c om public void configure() throws Exception { from("direct:start1").to( "hdfs://localhost:9000/tmp/test/test-camel-simple-write-file1?append=true&fileSystemType=HDFS"); } }); startCamelContext(); for (int i = 0; i < 10; ++i) { template.sendBody("direct:start1", "PIPPO"); } Configuration conf = new Configuration(); Path file = new Path("hdfs://localhost:9000/tmp/test/test-camel-simple-write-file1"); FileSystem fs = FileSystem.get(file.toUri(), conf); FSDataInputStream in = fs.open(file); byte[] buffer = new byte[5]; int ret = 0; for (int i = 0; i < 20; ++i) { ret = in.read(buffer); } ret = in.read(buffer); assertEquals(-1, ret); in.close(); }
From source file:org.apache.camel.component.hdfs2.integration.HdfsAppendTest.java
License:Apache License
@Test public void testAppend() throws Exception { context.addRoutes(new RouteBuilder() { @Override//from w ww . java2s. c o m public void configure() throws Exception { from("direct:start1").to( "hdfs2://localhost:9000/tmp/test/test-camel-simple-write-file1?append=true&fileSystemType=HDFS"); } }); startCamelContext(); for (int i = 0; i < 10; ++i) { template.sendBody("direct:start1", "PIPPQ"); } Configuration conf = new Configuration(); Path file = new Path("hdfs://localhost:9000/tmp/test/test-camel-simple-write-file1"); FileSystem fs = FileSystem.get(file.toUri(), conf); FSDataInputStream in = fs.open(file); byte[] buffer = new byte[5]; int ret = 0; for (int i = 0; i < 20; ++i) { ret = in.read(buffer); System.out.println("> " + new String(buffer)); } ret = in.read(buffer); assertEquals(-1, ret); in.close(); }
From source file:org.apache.drill.exec.work.batch.FileTest.java
License:Apache License
public static void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "sync:///"); System.out.println(FileSystem.getDefaultUri(conf)); FileSystem fs = FileSystem.get(conf); // FileSystem fs = new LocalSyncableFileSystem(conf); Path path = new Path("/tmp/testFile"); FSDataOutputStream out = fs.create(path); byte[] s = "hello world".getBytes(); out.write(s);//from www .ja va 2s. c o m out.sync(); // out.close(); FSDataInputStream in = fs.open(path); byte[] bytes = new byte[s.length]; in.read(bytes); System.out.println(new String(bytes)); File file = new File("/tmp/testFile"); FileOutputStream fos = new FileOutputStream(file); FileInputStream fis = new FileInputStream(file); fos.write(s); fos.getFD().sync(); fis.read(bytes); System.out.println(new String(bytes)); out = fs.create(new Path("/tmp/file")); for (int i = 0; i < 100; i++) { bytes = new byte[256 * 1024]; Stopwatch watch = Stopwatch.createStarted(); out.write(bytes); out.sync(); long t = watch.elapsed(TimeUnit.MILLISECONDS); System.out.printf("Elapsed: %d. Rate %d.\n", t, (long) ((long) bytes.length * 1000L / t)); } }
From source file:org.apache.hoya.api.ClusterDescription.java
License:Apache License
/** * Load from the filesystem//w w w . ja v a 2 s . co m * @param fs filesystem * @param path path * @return a loaded CD * @throws IOException IO problems */ public static ClusterDescription load(FileSystem fs, Path path) throws IOException, JsonParseException, JsonMappingException { FileStatus status = fs.getFileStatus(path); byte[] b = new byte[(int) status.getLen()]; FSDataInputStream dataInputStream = fs.open(path); int count = dataInputStream.read(b); String json = new String(b, 0, count, UTF_8); return fromJson(json); }
From source file:org.apache.hoya.core.persist.JsonSerDeser.java
License:Apache License
/** * Load from a Hadoop filesystem/*from w w w .j av a 2 s .co m*/ * @param fs filesystem * @param path path * @return a loaded CD * @throws IOException IO problems * @throws JsonParseException parse problems * @throws JsonMappingException O/J mapping problems */ public T load(FileSystem fs, Path path) throws IOException, JsonParseException, JsonMappingException { FileStatus status = fs.getFileStatus(path); byte[] b = new byte[(int) status.getLen()]; FSDataInputStream dataInputStream = fs.open(path); int count = dataInputStream.read(b); String json = new String(b, 0, count, UTF_8); return fromJson(json); }
From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java
License:Apache License
/** * Ensure that when running in multithreaded mode only one create() operation succeed. * * @throws Exception If failed.//from w w w.j av a 2 s . co m */ public void testMultithreadedCreate() throws Exception { Path dir = new Path(new Path(primaryFsUri), "/dir"); fs.mkdir(dir, FsPermission.getDefault(), true); final Path file = new Path(dir, "file"); fs.create(file, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault())) .close(); final AtomicInteger cnt = new AtomicInteger(); final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT); multithreaded(new Runnable() { @Override public void run() { int idx = cnt.getAndIncrement(); byte[] data = new byte[256]; Arrays.fill(data, (byte) idx); FSDataOutputStream os = null; try { os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE), Options.CreateOpts.perms(FsPermission.getDefault())); os.write(data); } catch (IOException ignore) { errs.add(idx); } finally { U.awaitQuiet(barrier); U.closeQuiet(os); } } }, THREAD_CNT); // Only one thread could obtain write lock on the file. assert errs.size() == THREAD_CNT - 1 : "Invalid errors count [expected=" + (THREAD_CNT - 1) + ", actual=" + errs.size() + ']'; int idx = -1; for (int i = 0; i < THREAD_CNT; i++) { if (!errs.remove(i)) { idx = i; break; } } byte[] expData = new byte[256]; Arrays.fill(expData, (byte) idx); FSDataInputStream is = fs.open(file); byte[] data = new byte[256]; is.read(data); is.close(); assert Arrays.equals(expData, data); }
From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java
License:Apache License
/** * Ensure that when running in multithreaded mode only one append() operation succeed. * * @throws Exception If failed./*from w ww . j a v a 2 s .c o m*/ */ public void testMultithreadedAppend() throws Exception { Path dir = new Path(new Path(primaryFsUri), "/dir"); fs.mkdir(dir, FsPermission.getDefault(), true); final Path file = new Path(dir, "file"); fs.create(file, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault())) .close(); final AtomicInteger cnt = new AtomicInteger(); final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT); multithreaded(new Runnable() { @Override public void run() { int idx = cnt.getAndIncrement(); byte[] data = new byte[256]; Arrays.fill(data, (byte) idx); U.awaitQuiet(barrier); FSDataOutputStream os = null; try { os = fs.create(file, EnumSet.of(CreateFlag.APPEND), Options.CreateOpts.perms(FsPermission.getDefault())); os.write(data); } catch (IOException ignore) { errs.add(idx); } finally { U.awaitQuiet(barrier); U.closeQuiet(os); } } }, THREAD_CNT); // Only one thread could obtain write lock on the file. assert errs.size() == THREAD_CNT - 1; int idx = -1; for (int i = 0; i < THREAD_CNT; i++) { if (!errs.remove(i)) { idx = i; break; } } byte[] expData = new byte[256]; Arrays.fill(expData, (byte) idx); FSDataInputStream is = fs.open(file); byte[] data = new byte[256]; is.read(data); is.close(); assert Arrays.equals(expData, data); }