Example usage for org.apache.hadoop.io IOUtils copyBytes

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils copyBytes.

Prototype

public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException

Source Link

Document

Copies count bytes from one stream to another.

Usage

From source file:org.apache.falcon.lifecycle.TableStorageFeedEvictorIT.java

License:Apache License

private String readLogFile(Path logFile) throws IOException {
    ByteArrayOutputStream writer = new ByteArrayOutputStream();
    InputStream date = logFile.getFileSystem(new Configuration()).open(logFile);
    IOUtils.copyBytes(date, writer, 4096, true);
    return writer.toString();
}

From source file:org.apache.falcon.logging.JobLogMover.java

License:Apache License

private void copyOozieLog(OozieClient client, FileSystem fs, Path path, String id)
        throws OozieClientException, IOException {
    InputStream in = new ByteArrayInputStream(client.getJobLog(id).getBytes());
    OutputStream out = fs.create(new Path(path, "oozie.log"));
    IOUtils.copyBytes(in, out, 4096, true);
    LOG.info("Copied oozie log to {}", path);
}

From source file:org.apache.falcon.logging.JobLogMover.java

License:Apache License

private void copyTTlogs(FileSystem fs, Path path, WorkflowAction action) throws Exception {
    List<String> ttLogUrls = getTTlogURL(action.getExternalId());
    if (ttLogUrls != null) {
        int index = 1;
        for (String ttLogURL : ttLogUrls) {
            LOG.info("Fetching log for action: {} from url: {}", action.getExternalId(), ttLogURL);
            InputStream in = getURLinputStream(new URL(ttLogURL));
            OutputStream out = fs.create(new Path(path, action.getName() + "_" + action.getType() + "_"
                    + getMappedStatus(action.getStatus()) + "-" + index + ".log"));
            IOUtils.copyBytes(in, out, 4096, true);
            LOG.info("Copied log to {}", path);
            index++;/* w  ww  .j a va2  s  .  c  o m*/
        }
    }
}

From source file:org.apache.falcon.logging.LogMover.java

License:Apache License

private void copyTTlogs(FileSystem fs, Path path, WorkflowAction action) throws Exception {
    String ttLogURL = getTTlogURL(action.getExternalId());
    if (ttLogURL != null) {
        LOG.info("Fetching log for action: {} from url: {}", action.getExternalId(), ttLogURL);
        InputStream in = getURLinputStream(new URL(ttLogURL));
        OutputStream out = fs//from w ww.  j  a  va 2 s.c  om
                .create(new Path(path, action.getName() + "_" + getMappedStatus(action.getStatus()) + ".log"));
        IOUtils.copyBytes(in, out, 4096, true);
        LOG.info("Copied log to {}", path);
    }
}

From source file:org.apache.falcon.messaging.EntityInstanceMessage.java

License:Apache License

private static String getInstancePathsFromFile(Path path) throws FalconException {
    InputStream instance = null;//ww  w.  j  a v  a 2  s . c om
    try {
        FileSystem fs = path.getFileSystem(new Configuration());

        if (fs.exists(path)) {
            ByteArrayOutputStream writer = new ByteArrayOutputStream();
            instance = fs.open(path);
            IOUtils.copyBytes(instance, writer, 4096, true);
            String[] instancePaths = writer.toString().split("=");
            if (instancePaths.length > 1) {
                return instancePaths[1];
            }
        }
    } catch (IOException e) {
        throw new FalconException(e);
    } finally {
        org.apache.commons.io.IOUtils.closeQuietly(instance);
    }
    return null;
}

From source file:org.apache.falcon.resource.proxy.BufferedRequest.java

License:Apache License

public BufferedRequest(HttpServletRequest request) {
    try {//  w  ww.  j  av a2s.c  o m
        this.request = request;
        ByteArrayOutputStream copyBuffer = streams.get();
        copyBuffer.reset();
        IOUtils.copyBytes(request.getInputStream(), copyBuffer, 4096, false);
        final ByteArrayInputStream buffer = new ByteArrayInputStream(copyBuffer.toByteArray(), 0,
                copyBuffer.size());
        stream = new ServletInputStream() {
            @Override
            public int read() throws IOException {
                return buffer.read();
            }

            @Override
            public void mark(int readlimit) {
                buffer.mark(readlimit);
            }

            @Override
            public void reset() throws IOException {
                buffer.reset();
            }

            @Override
            public boolean markSupported() {
                return buffer.markSupported();
            }
        };
        stream.mark(10 * 1024 * 1024);
    } catch (IOException e) {
        throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
    }
}

From source file:org.apache.falcon.retention.EvictedInstanceSerDe.java

License:Apache License

private static void logEvictedInstancePaths(final FileSystem fs, final Path outPath) throws IOException {
    ByteArrayOutputStream writer = new ByteArrayOutputStream();
    InputStream instance = fs.open(outPath);
    IOUtils.copyBytes(instance, writer, 4096, true);
    LOG.debug("Instance Paths copied to {}", outPath);
    LOG.debug("Written {}", writer);
}

From source file:org.apache.falcon.retention.EvictedInstanceSerDe.java

License:Apache License

/**
 * This method deserializes the evicted instances from a log file on hdfs.
 * @see org.apache.falcon.messaging.JMSMessageProducer
 * *Note:* This is executed with in the falcon server
 *
 * @param fileSystem file system handle/*w w  w .j a  va 2  s .c  o m*/
 * @param logFile    File path to serialize the instances to
 * @return list of instances, comma separated
 * @throws IOException
 */
public static String[] deserializeEvictedInstancePaths(final FileSystem fileSystem, final Path logFile)
        throws IOException {
    ByteArrayOutputStream writer = new ByteArrayOutputStream();
    InputStream instance = fileSystem.open(logFile);
    IOUtils.copyBytes(instance, writer, 4096, true);
    String[] instancePaths = writer.toString().split(INSTANCEPATH_PREFIX);

    if (instancePaths.length <= 1) {
        LOG.info("Returning 0 instance paths for feed ");
        return new String[0];
    } else {
        LOG.info("Returning instance paths for feed {}", instancePaths[1]);
        return instancePaths[1].split(INSTANCEPATH_SEPARATOR);
    }
}

From source file:org.apache.falcon.retention.EvictedInstanceSerDeTest.java

License:Apache License

private String readLogFile(Path logFile) throws IOException {
    ByteArrayOutputStream writer = new ByteArrayOutputStream();
    InputStream date = fs.open(logFile);
    IOUtils.copyBytes(date, writer, 4096, true);
    return writer.toString();
}

From source file:org.apache.falcon.retention.EvictionHelper.java

License:Apache License

public static String[] getInstancePaths(final FileSystem fs, final Path logFile) throws FalconException {
    ByteArrayOutputStream writer = new ByteArrayOutputStream();
    try {/*  w  w  w  .  j a  v  a  2s .  c  o  m*/
        InputStream date = fs.open(logFile);
        IOUtils.copyBytes(date, writer, 4096, true);
    } catch (IOException e) {
        throw new FalconException(e);
    }
    String logData = writer.toString();
    if (StringUtils.isEmpty(logData)) {
        throw new FalconException("csv file is empty");
    }

    String[] parts = logData.split(INSTANCEPATH_FORMAT);
    if (parts.length != 2) {
        throw new FalconException("Instance path in csv file not in required format: " + logData);
    }

    // part[0] is instancePaths=
    return parts[1].split(INSTANCEPATH_SEPARATOR);
}