List of usage examples for org.apache.hadoop.io IOUtils copyBytes
public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException
From source file:org.apache.falcon.retention.EvictionHelper.java
License:Apache License
private static void debug(final FileSystem fs, final Path outPath) throws IOException { ByteArrayOutputStream writer = new ByteArrayOutputStream(); InputStream instance = fs.open(outPath); IOUtils.copyBytes(instance, writer, 4096, true); LOG.debug("Instance Paths copied to {}", outPath); LOG.debug("Written {}", writer); }
From source file:org.apache.falcon.retention.FeedEvictorTest.java
License:Apache License
private String readLogFile(Path logFile) throws IOException { Configuration conf = cluster.getConf(); FileSystem fs = FileSystem.get(conf); ByteArrayOutputStream writer = new ByteArrayOutputStream(); InputStream date = fs.open(logFile); IOUtils.copyBytes(date, writer, 4096, true); return writer.toString(); }
From source file:org.apache.gobblin.source.extractor.SimpleJsonExtractor.java
License:Apache License
@Override public String getSchema() throws IOException { // Source is responsible to set SOURCE_SCHEMA ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); IOUtils.copyBytes(fs.open(new Path(workUnitState.getProp(ConfigurationKeys.SOURCE_SCHEMA))), outputStream, 4096, false);/*from www . ja v a 2s . co m*/ String schema = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); workUnitState.setProp((ConfigurationKeys.CONVERTER_AVRO_SCHEMA_KEY), schema); return schema; }
From source file:org.apache.hama.bsp.YarnSerializePrinting.java
License:Apache License
static void printOutput(HamaConfiguration conf) throws IOException { FileSystem fs = FileSystem.get(conf); FileStatus[] files = fs.listStatus(OUTPUT_PATH); for (FileStatus file : files) { if (file.getLen() > 0) { FSDataInputStream in = fs.open(file.getPath()); IOUtils.copyBytes(in, System.out, conf, false); in.close();/* ww w .ja va 2 s. c om*/ } } fs.delete(OUTPUT_PATH, true); }
From source file:org.apache.hama.examples.CombineExample.java
License:Apache License
static void printOutput(HamaConfiguration conf) throws IOException { FileSystem fs = FileSystem.get(conf); FileStatus[] files = fs.listStatus(TMP_OUTPUT); for (int i = 0; i < files.length; i++) { if (files[i].getLen() > 0) { FSDataInputStream in = fs.open(files[i].getPath()); IOUtils.copyBytes(in, System.out, conf, false); in.close();/*w ww .j a v a2 s . c o m*/ break; } } fs.delete(TMP_OUTPUT, true); }
From source file:org.apache.hama.examples.SuperstepPiEstimator.java
License:Apache License
private static void printOutput(HamaConfiguration conf) throws IOException { FileSystem fs = FileSystem.get(conf); FileStatus[] files = fs.listStatus(TMP_OUTPUT); for (FileStatus file : files) { if (file.getLen() > 0) { FSDataInputStream in = fs.open(file.getPath()); IOUtils.copyBytes(in, System.out, conf, false); in.close();//from w w w . j a va2 s .c om break; } } fs.delete(TMP_OUTPUT, true); }
From source file:org.apache.ivory.logging.LogMover.java
License:Apache License
private void copyOozieLog(OozieClient client, FileSystem fs, Path path, String id) throws OozieClientException, IOException { InputStream in = new StringInputStream(client.getJobLog(id)); OutputStream out = fs.create(new Path(path, "oozie.log")); IOUtils.copyBytes(in, out, 4096, true); LOG.info("Copied oozie log to " + path); }
From source file:org.apache.ivory.logging.LogMover.java
License:Apache License
private void copyTTlogs(ARGS args, FileSystem fs, Path path, WorkflowAction action) throws Exception { String ttLogURL = getTTlogURL(action.getExternalId()); if (ttLogURL != null) { LOG.info("Fetching log for action: " + action.getExternalId() + " from url: " + ttLogURL); InputStream in = getURLinputStream(new URL(ttLogURL)); OutputStream out = fs/*w w w .j a v a 2s .c o m*/ .create(new Path(path, action.getName() + "_" + getMappedStatus(action.getStatus()) + ".log")); IOUtils.copyBytes(in, out, 4096, true); LOG.info("Copied log to " + path); } }
From source file:org.apache.ivory.messaging.EntityInstanceMessage.java
License:Apache License
private static String[] getFeedPaths(CommandLine cmd) throws IOException { String topicName = cmd.getOptionValue(ARG.topicName.getArgName()); String operation = cmd.getOptionValue(ARG.operation.getArgName()); if (topicName.equals(IVORY_ENTITY_TOPIC_NAME)) { LOG.debug("Returning instance paths for Ivory Topic: " + cmd.getOptionValue(ARG.feedInstancePaths.getArgName())); return new String[] { cmd.getOptionValue(ARG.feedInstancePaths.getArgName()) }; }//from www . j a v a 2s . c om if (operation.equals(EntityOps.GENERATE.name()) || operation.equals(EntityOps.REPLICATE.name())) { LOG.debug("Returning instance paths: " + cmd.getOptionValue(ARG.feedInstancePaths.getArgName())); return cmd.getOptionValue(ARG.feedInstancePaths.getArgName()).split(","); } //else case of feed retention Path logFile = new Path(cmd.getOptionValue(ARG.logFile.getArgName())); FileSystem fs = FileSystem.get(logFile.toUri(), new Configuration()); ByteArrayOutputStream writer = new ByteArrayOutputStream(); InputStream instance = fs.open(logFile); IOUtils.copyBytes(instance, writer, 4096, true); String[] instancePaths = writer.toString().split("="); if (instancePaths.length == 1) { LOG.debug("Returning 0 instance paths for feed "); return new String[0]; } else { LOG.debug("Returning instance paths for feed " + instancePaths[1]); return instancePaths[1].split(","); } }
From source file:org.apache.ivory.retention.FeedEvictor.java
License:Apache License
private void debug(Path outPath, FileSystem fs) throws IOException { ByteArrayOutputStream writer = new ByteArrayOutputStream(); InputStream instance = fs.open(outPath); IOUtils.copyBytes(instance, writer, 4096, true); LOG.debug("Instance Paths copied to " + outPath); LOG.debug("Written " + writer); }