Example usage for org.apache.hadoop.io IOUtils cleanup

List of usage examples for org.apache.hadoop.io IOUtils cleanup

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. IOUtils
  6. cleanup

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils cleanup.

Prototype

@Deprecated
public static void cleanup(Log log, java.io.Closeable... closeables)

Source Link

Document

Close the Closeable objects and ignore any Throwable or null pointers.

Usage

From source file:BlockReaderFactory.java

License:Apache License

/**
 * Fetch a pair of short-circuit block descriptors from a local DataNode.
 *
 * @return    Null if we could not communicate with the datanode,
 *            a new ShortCircuitReplicaInfo object otherwise.
 *            ShortCircuitReplicaInfo objects may contain either an InvalidToken
 *            exception, or a ShortCircuitReplica object ready to use.
 *//*from w  w w .  ja  v a2  s . c o  m*/
@Override
public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
    if (createShortCircuitReplicaInfoCallback != null) {
        ShortCircuitReplicaInfo info = createShortCircuitReplicaInfoCallback.createShortCircuitReplicaInfo();
        if (info != null)
            return info;
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace(this + ": trying to create ShortCircuitReplicaInfo.");
    }
    BlockReaderPeer curPeer;
    while (true) {
        curPeer = nextDomainPeer();
        if (curPeer == null)
            break;
        if (curPeer.fromCache)
            remainingCacheTries--;
        DomainPeer peer = (DomainPeer) curPeer.peer;
        Slot slot = null;
        ShortCircuitCache cache = clientContext.getShortCircuitCache();
        try {
            MutableBoolean usedPeer = new MutableBoolean(false);
            slot = cache.allocShmSlot(datanode, peer, usedPeer,
                    new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()), clientName);
            if (usedPeer.booleanValue()) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace(this + ": allocShmSlot used up our previous socket " + peer.getDomainSocket()
                            + ".  Allocating a new one...");
                }
                curPeer = nextDomainPeer();
                if (curPeer == null)
                    break;
                peer = (DomainPeer) curPeer.peer;
            }
            ShortCircuitReplicaInfo info = requestFileDescriptors(peer, slot);
            clientContext.getPeerCache().put(datanode, peer);
            return info;
        } catch (IOException e) {
            if (slot != null) {
                cache.freeSlot(slot);
            }
            if (curPeer.fromCache) {
                // Handle an I/O error we got when using a cached socket.
                // These are considered less serious, because the socket may be stale.
                if (LOG.isDebugEnabled()) {
                    LOG.debug(this + ": closing stale domain peer " + peer, e);
                }
                IOUtils.cleanup(LOG, peer);
            } else {
                // Handle an I/O error we got when using a newly created socket.
                // We temporarily disable the domain socket path for a few minutes in
                // this case, to prevent wasting more time on it.
                LOG.warn(this + ": I/O error requesting file descriptors.  " + "Disabling domain socket "
                        + peer.getDomainSocket(), e);
                IOUtils.cleanup(LOG, peer);
                clientContext.getDomainSocketFactory().disableDomainSocketPath(pathInfo.getPath());
                return null;
            }
        }
    }
    return null;
}

From source file:BlockReaderFactory.java

License:Apache License

/**
 * Get a RemoteBlockReader that communicates over a UNIX domain socket.
 *
 * @return The new BlockReader, or null if we failed to create the block
 * reader.//from   www . j a  va 2s . co m
 *
 * @throws InvalidToken    If the block token was invalid.
 * Potentially other security-related execptions.
 */
private BlockReader getRemoteBlockReaderFromDomain() throws IOException {
    if (pathInfo == null) {
        pathInfo = clientContext.getDomainSocketFactory().getPathInfo(inetSocketAddress, conf);
    }
    if (!pathInfo.getPathState().getUsableForDataTransfer()) {
        PerformanceAdvisory.LOG.debug(this + ": not trying to create a "
                + "remote block reader because the UNIX domain socket at " + pathInfo + " is not usable.");
        return null;
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace(this + ": trying to create a remote block reader from the " + "UNIX domain socket at "
                + pathInfo.getPath());
    }

    while (true) {
        BlockReaderPeer curPeer = nextDomainPeer();
        if (curPeer == null)
            break;
        if (curPeer.fromCache)
            remainingCacheTries--;
        DomainPeer peer = (DomainPeer) curPeer.peer;
        BlockReader blockReader = null;
        try {
            blockReader = getRemoteBlockReader(peer);
            return blockReader;
        } catch (IOException ioe) {
            IOUtils.cleanup(LOG, peer);
            if (isSecurityException(ioe)) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace(this + ": got security exception while constructing "
                            + "a remote block reader from the unix domain socket at " + pathInfo.getPath(),
                            ioe);
                }
                throw ioe;
            }
            if (curPeer.fromCache) {
                // Handle an I/O error we got when using a cached peer.  These are
                // considered less serious, because the underlying socket may be stale.
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Closed potentially stale domain peer " + peer, ioe);
                }
            } else {
                // Handle an I/O error we got when using a newly created domain peer.
                // We temporarily disable the domain socket path for a few minutes in
                // this case, to prevent wasting more time on it.
                LOG.warn("I/O error constructing remote block reader.  Disabling " + "domain socket "
                        + peer.getDomainSocket(), ioe);
                clientContext.getDomainSocketFactory().disableDomainSocketPath(pathInfo.getPath());
                return null;
            }
        } finally {
            if (blockReader == null) {
                IOUtils.cleanup(LOG, peer);
            }
        }
    }
    return null;
}

From source file:BlockReaderFactory.java

License:Apache License

/**
 * Get a RemoteBlockReader that communicates over a TCP socket.
 *
 * @return The new BlockReader.  We will not return null, but instead throw
 *         an exception if this fails.//from  w  w  w.  j a  va  2 s .c om
 *
 * @throws InvalidToken
 *             If the block token was invalid.
 *         InvalidEncryptionKeyException
 *             If the encryption key was invalid.
 *         Other IOException
 *             If there was another problem.
 */
private BlockReader getRemoteBlockReaderFromTcp() throws IOException {
    if (LOG.isTraceEnabled()) {
        LOG.trace(this + ": trying to create a remote block reader from a " + "TCP socket");
    }
    BlockReader blockReader = null;
    while (true) {
        BlockReaderPeer curPeer = null;
        Peer peer = null;
        try {
            curPeer = nextTcpPeer();
            if (curPeer == null)
                break;
            if (curPeer.fromCache)
                remainingCacheTries--;
            peer = curPeer.peer;
            blockReader = getRemoteBlockReader(peer);
            return blockReader;
        } catch (IOException ioe) {
            if (isSecurityException(ioe)) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace(this + ": got security exception while constructing "
                            + "a remote block reader from " + peer, ioe);
                }
                throw ioe;
            }
            if ((curPeer != null) && curPeer.fromCache) {
                // Handle an I/O error we got when using a cached peer.  These are
                // considered less serious, because the underlying socket may be
                // stale.
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Closed potentially stale remote peer " + peer, ioe);
                }
            } else {
                // Handle an I/O error we got when using a newly created peer.
                LOG.warn("I/O error constructing remote block reader.", ioe);
                throw ioe;
            }
        } finally {
            if (blockReader == null) {
                IOUtils.cleanup(LOG, peer);
            }
        }
    }
    return null;
}

From source file:ApplicationMaster.java

License:Apache License

/**
 * Dump out contents of $CWD and the environment to stdout for debugging
 *///from w w w. j a  va2  s .c  o  m
private void dumpOutDebugInfo() {

    LOG.info("Dump debug output");
    Map<String, String> envs = System.getenv();
    for (Map.Entry<String, String> env : envs.entrySet()) {
        LOG.info("System env: key=" + env.getKey() + ", val=" + env.getValue());
        System.out.println("System env: key=" + env.getKey() + ", val=" + env.getValue());
    }

    BufferedReader buf = null;
    try {
        String lines = Shell.WINDOWS ? Shell.execCommand("cmd", "/c", "dir") : Shell.execCommand("ls", "-al");
        buf = new BufferedReader(new StringReader(lines));
        String line = "";
        while ((line = buf.readLine()) != null) {
            LOG.info("System CWD content: " + line);
            System.out.println("System CWD content: " + line);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        IOUtils.cleanup(LOG, buf);
    }
}

From source file:cn.edu.buaa.act.petuumOnYarn.ApplicationMaster.java

License:Apache License

/**
 * Dump out contents of $CWD and the environment to stdout for debugging
 *///from w ww . ja va  2 s .c  o  m
private void dumpOutDebugInfo() {

    LOG.info("Dump debug output");
    Map<String, String> envs = System.getenv();
    for (Map.Entry<String, String> env : envs.entrySet()) {
        LOG.info("System env: key=" + env.getKey() + ", val=" + env.getValue());
        System.out.println("System env: key=" + env.getKey() + ", val=" + env.getValue());
    }

    BufferedReader buf = null;
    try {
        String lines = Shell.execCommand("ls", "-al");
        buf = new BufferedReader(new StringReader(lines));
        String line = "";
        while ((line = buf.readLine()) != null) {
            LOG.info("System CWD content: " + line);
            System.out.println("System CWD content: " + line);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        IOUtils.cleanup(LOG, buf);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.lib.DynamicInputChunk.java

License:Apache License

/**
 * Closes streams opened to the chunk-file.
 * @throws java.io.IOException On failure to close the write-stream.
 */// www  .j a  va  2 s  .  com
public void close() throws IOException {
    IOUtils.cleanup(LOG, reader);
    try {
        if (writer != null)
            writer.close();
    } catch (IOException exception) {
        LOG.error("Could not close writer: ", exception);
        throw exception;
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private static void touchFile(String path) throws Exception {
    FileSystem fs;//w  w w . j ava2s  . co m
    DataOutputStream outputStream = null;
    GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfiguration());
    Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
    OutputStream compressedOut = null;
    try {
        fs = cluster.getFileSystem();
        final Path qualifiedPath = new Path(path).makeQualified(fs);
        final long blockSize = fs.getDefaultBlockSize() * 2;
        outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize);
        compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor);
        Message msg = new Message("generating test data".getBytes());
        AuditUtil.attachHeaders(msg, currentTimestamp);
        byte[] encodeMsg = Base64.encodeBase64(msg.getData().array());
        compressedOut.write(encodeMsg);
        compressedOut.write("\n".getBytes());
        compressedOut.write(encodeMsg);
        compressedOut.write("\n".getBytes());
        // Genearate a msg with different timestamp.  Default window period is 60sec
        AuditUtil.attachHeaders(msg, nextMinuteTimeStamp);
        encodeMsg = Base64.encodeBase64(msg.getData().array());
        compressedOut.write(encodeMsg);
        compressedOut.write("\n".getBytes());
        compressedOut.flush();
        compressedOut.close();
        pathList.add(qualifiedPath);
        ++nFiles;

        FileStatus fileStatus = fs.getFileStatus(qualifiedPath);
        System.out.println(fileStatus.getBlockSize());
        System.out.println(fileStatus.getReplication());
    } finally {
        compressedOut.close();
        IOUtils.cleanup(null, outputStream);
        CodecPool.returnCompressor(gzipCompressor);
    }
}

From source file:com.inmobi.conduit.distcp.tools.TestDistCp.java

License:Apache License

private static void touchFile(String path) throws Exception {
    FileSystem fs;//w  ww .j  a  v a 2 s  . c o m
    DataOutputStream outputStream = null;
    GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfigurationForCluster());
    Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
    OutputStream compressedOut = null;
    try {
        fs = cluster.getFileSystem();
        final Path qualifiedPath = new Path(path).makeQualified(fs);
        final long blockSize = fs.getDefaultBlockSize() * 2;
        outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize);
        compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor);
        compressedOut.write(new byte[FILE_SIZE]);
        compressedOut.write("\n".getBytes());
        compressedOut.flush();
        //outputStream.write(new byte[FILE_SIZE]);
        pathList.add(qualifiedPath);
    } finally {
        compressedOut.close();
        IOUtils.cleanup(null, outputStream);
        CodecPool.returnCompressor(gzipCompressor);
    }
}

From source file:com.inmobi.conduit.distcp.tools.TestGlobbedCopyListing.java

License:Apache License

private static void mkdirs(String path) throws Exception {
    FileSystem fileSystem = null;
    try {/*from   w w w.  j  a v  a  2  s .co m*/
        fileSystem = cluster.getFileSystem();
        fileSystem.mkdirs(new Path(path));
        recordInExpectedValues(path);
    } finally {
        IOUtils.cleanup(null, fileSystem);
    }
}

From source file:com.mellanox.r4h.DFSClient.java

License:Apache License

@Override
// RemotePeerFactory
public Peer newConnectedPeer(InetSocketAddress addr, Token<BlockTokenIdentifier> blockToken,
        DatanodeID datanodeId) throws IOException {
    Peer peer = null;/*from w  w w . ja v a2  s.  c o m*/
    boolean success = false;
    Socket sock = null;
    try {
        sock = socketFactory.createSocket();
        NetUtils.connect(sock, addr, getRandomLocalInterfaceAddr(), dfsClientConf.getSocketTimeout());
        peer = TcpPeerServer.peerFromSocketAndKey(saslClient, sock, this, blockToken, datanodeId);
        peer.setReadTimeout(dfsClientConf.socketTimeout());
        success = true;
        return peer;
    } finally {
        if (!success) {
            IOUtils.cleanup(LOG, peer);
            IOUtils.closeSocket(sock);
        }
    }
}