Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException

Source Link

Document

Check if a path exists.

Usage

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

private boolean runElasticSearchMarkerQueries() {
    boolean success = true;
    FileSystem hdfs;
    FSDataInputStream in;/*www.  j  a  va 2s. c o m*/
    dstPath = new Path(elasticsearchMarkerFolder);
    LOG.info("DSTPATH: " + dstPath);
    try {
        hdfs = dstPath.getFileSystem(conf);
        if (hdfs.exists(dstPath)) {
            FileStatus[] fileListing = hdfs.listStatus(dstPath);
            for (FileStatus fs : fileListing) {
                if (!fs.isDir()) {
                    LOG.info("File marker path: " + fs.getPath());
                    in = hdfs.open(fs.getPath());
                    byte[] fileData = new byte[(int) fs.getLen()];
                    in.readFully(fileData);
                    in.close();
                    LOG.info("cleaning markerfile @: " + fs.getPath().toString());
                    cleanMarkerFile(fs.getPath().toString());
                    sendESQuery(elasticsearchUrl, new String(fileData));

                }
            }
        }
    } catch (Exception e) {
        success = false;
    }
    return success;
}

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

private boolean writeElasticSearchToMarkerFolder(StringBuilder httpQuery) {
    FileSystem hdfs;
    try {/*from  w w  w .ja v a 2s .  co m*/
        String markerFolder = conf.getElasticSearchDefaultMarkerFolder();
        dstPath = new Path(markerFolder);
        hdfs = dstPath.getFileSystem(conf);
        if (!hdfs.exists(dstPath)) {
            hdfs.mkdirs(dstPath);
        }

        dstPath = new Path(markerFolder + "/es-" + System.currentTimeMillis() + ".marker");
        System.out.println("creating file at: " + dstPath.toString());
        FSDataOutputStream writer_marker = hdfs.create(dstPath);
        writer_marker.writeBytes(httpQuery + "\n");
        writer_marker.close();
        dstPath = null;
        writer_marker = null;
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        return false;
    }
    return true;
}

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

private boolean runHiveMarkerQueries() {
    boolean queryStatus = true;
    FileSystem hdfs;
    FSDataInputStream in;/*  ww  w.  ja v  a2  s . c  om*/
    dstPath = new Path(hiveMarkerFolder);
    LOG.info("DSTPATH: " + dstPath);
    try {
        hdfs = dstPath.getFileSystem(conf);
        if (hdfs.exists(dstPath)) {
            FileStatus[] fileListing = hdfs.listStatus(dstPath);
            for (FileStatus fs : fileListing) {
                if (!fs.isDir()) {
                    LOG.info("File marker path: " + fs.getPath());
                    in = hdfs.open(fs.getPath());
                    byte[] fileData = new byte[(int) fs.getLen()];
                    in.readFully(fileData);
                    String[] splitTab = new String(fileData).split("\t");
                    if (splitTab.length == 2) {
                        dstPath = new Path(splitTab[0]);
                        FileSystem hiveFile = dstPath.getFileSystem(conf);
                        if (hiveFile.exists(dstPath)) {
                            LOG.info("marker file data: " + splitTab[1]);
                            if (runHiveQuery(splitTab[1])) {
                                LOG.info("Marker query is successful");
                                in.close();
                                cleanMarkerFile(fs.getPath().toString());
                            } else {
                                LOG.info("Error running marker query, marker point not deleted");
                                queryStatus = false;
                            }

                        } else {
                            LOG.info("marker points to invalid hive file location, deleting the marker");
                            in.close();
                            cleanMarkerFile(fs.getPath().toString());
                        }
                    }
                    //in.close();
                }
            }
        }
        hdfs.close();
    } catch (IOException e) {
        LOG.error("ERROR running runMarkerQueries:" + e.getMessage());
    }

    return queryStatus;
}

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

public boolean writeHiveMarker(String hqlQuery, String filePath, String hiveMarkerFolder,
        String hiveMarkerPath) {/*from   w  w  w  . java 2s.com*/
    LOG.debug("writing to hiveMarker: " + hiveMarkerFolder);

    FileSystem hdfs;
    dstPath = new Path(hiveMarkerFolder);
    try {
        hdfs = dstPath.getFileSystem(conf);

        if (!hdfs.exists(dstPath)) {
            hdfs.mkdirs(dstPath);
        }
        dstPath = new Path(hiveMarkerPath);
        FSDataOutputStream writer = hdfs.create(dstPath);
        writer.writeBytes(filePath + "\t" + hqlQuery + "\n");
        writer.close();
        dstPath = null;
        writer = null;

    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    /*
      dstPath = new Path(hiveMarkerPath);
      hdfs = dstPath.getFileSystem(conf);
            
      writer = hdfs.create(dstPath);      
      writer.writeUTF(hqlQuery);
      writer.close();
      writer = null;
     */
    return true;

}

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

public boolean mergeFiles(String folder, Path file, String hiveOutputLocation) {
    FileSystem hdfs;
    FSDataInputStream in;/*ww w  .j av a  2s  .  c  o  m*/
    FSDataOutputStream out;
    List<Path> fileCollection = new ArrayList<Path>();
    dstPath = new Path(folder);
    LOG.info("mergeFiles DSTPATH: " + dstPath);
    try {
        hdfs = dstPath.getFileSystem(conf);

        if (hdfs.exists(dstPath)) {
            FileStatus[] fileListing = hdfs.listStatus(dstPath);
            LOG.error("Creating file @: " + hiveOutputLocation);
            out = hdfs.create(new Path(hiveOutputLocation));

            in = hdfs.open(file);
            byte[] fileData = new byte[(int) hdfs.getFileStatus(file).getLen()];
            in.readFully(fileData);
            out.write(fileData);

            for (FileStatus fs : fileListing) {
                if (!fs.isDir()) {
                    LOG.info("mergeFiles File marker path: " + fs.getPath());
                    fileCollection.add(fs.getPath());
                    in = hdfs.open(fs.getPath());
                    fileData = new byte[(int) fs.getLen()];
                    in.readFully(fileData);
                    out.write(fileData);
                }
            }
            out.close();
        }

        hdfs.close();
        LOG.error("Written file: " + hiveOutputLocation);

        //lets start the purge process, delete all files except the merged file
        hdfs = dstPath.getFileSystem(conf);
        for (Path p : fileCollection) {
            if (hdfs.delete(p, false)) {
                LOG.error("Successfully deleted: " + p);
            } else {
                LOG.error("Error deleting file: " + p);
            }
        }

    } catch (IOException e) {
        LOG.error("ERROR running runMarkerQueries:" + e.getMessage());
    }
    LOG.error("mergeFiles Done merging files");
    return false;
}

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

public boolean checkIfPartitionExists(String filePath) {
    dstPath = new Path(filePath);
    FileSystem hdfs;
    try {//w w w .j a  v a2 s  .  co  m
        hdfs = dstPath.getFileSystem(conf);
        return hdfs.exists(dstPath);
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    return false;
}

From source file:com.cloudera.GetBlockLocations.java

License:Apache License

public static void main(String[] args) throws Exception {
    final Configuration conf = new Configuration();
    String url = getStringOrDie("get.block.locations.path");
    final FileSystem fs = FileSystem.get(new URI(url), conf);

    if (!fs.exists(new Path(url))) {
        System.out.println("no file at " + url);
        System.exit(1);/* w  w w  .  jav  a2s .c  om*/
    }
    BlockLocation locs[] = null;
    try {
        locs = fs.getFileBlockLocations(new Path(url), 0, Long.MAX_VALUE);
    } catch (IOException e) {
        System.out.println("Error calling getFileBlockLocations(" + url + ")\n");
        e.printStackTrace(System.err);
        System.exit(1);
    }

    String prefix = "";
    for (BlockLocation loc : locs) {
        System.out.println(prefix);
        System.out.println("{");
        System.out.println("  hosts =         " + Arrays.toString(loc.getHosts()));
        System.out.println("  cachedHosts =   " + Arrays.toString(loc.getCachedHosts()));
        System.out.println("  names    =      " + Arrays.toString(loc.getNames()));
        System.out.println("  topologyPaths = " + Arrays.toString(loc.getTopologyPaths()));
        System.out.println("  offset =        " + loc.getOffset());
        System.out.println("  length =        " + loc.getLength());
        System.out.println("  corrupt =       " + loc.isCorrupt());
        System.out.println("}");
        prefix = ",";
    }
}

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.handlers.CREATEHandler.java

License:Apache License

@Override
protected CREATEResponse doHandle(NFS4Handler server, Session session, CREATERequest request)
        throws NFS4Exception, IOException {
    if (session.getCurrentFileHandle() == null) {
        throw new NFS4Exception(NFS4ERR_NOFILEHANDLE);
    }/*from w  w w  .j av a 2 s  .  c  o m*/
    if (request.getType() != NFS4_DIR) {
        throw new UnsupportedOperationException(
                "Create files of  type " + request.getType() + " is not supported.");
    }
    if ("".equals(request.getName())) {
        throw new NFS4Exception(NFS4ERR_INVAL);
    }
    Path parent = server.getPath(session.getCurrentFileHandle());
    Path path = new Path(parent, request.getName());
    FileSystem fs = session.getFileSystem();
    if (!fs.exists(parent)) {
        throw new NFS4Exception(NFS4ERR_STALE, "Parent " + parent + " does not exist");
    }
    if (fs.exists(path)) {
        throw new NFS4Exception(NFS4ERR_EXIST, "Path " + path + " already exists.");
    }
    long parentModTimeBefore = fs.getFileStatus(parent).getModificationTime();
    if (!fs.mkdirs(path)) {
        throw new NFS4Exception(NFS4ERR_IO);
    }
    long parentModTimeAfter = fs.getFileStatus(parent).getModificationTime();
    FileStatus fileStatus = fs.getFileStatus(path);
    ImmutableMap<Integer, Attribute> requestAttrs = request.getAttrValues();
    // TODO Handlers should have annotations so that setAttrs can throw an
    // error if they require the stateID to be set. 
    Bitmap responseAttrs = Attribute.setAttrs(server, session, request.getAttrs(), requestAttrs, fs, fileStatus,
            null);
    session.setCurrentFileHandle(server.createFileHandle(path));
    CREATEResponse response = createResponse();
    response.setChangeInfo(ChangeInfo.newChangeInfo(true, parentModTimeBefore, parentModTimeAfter));
    response.setStatus(NFS4_OK);
    response.setAttrs(responseAttrs);
    return response;
}

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.handlers.REMOVEHandler.java

License:Apache License

@Override
protected REMOVEResponse doHandle(NFS4Handler server, Session session, REMOVERequest request)
        throws NFS4Exception, IOException {
    if (session.getCurrentFileHandle() == null) {
        throw new NFS4Exception(NFS4ERR_NOFILEHANDLE);
    }//from  w  w w .j  a  v a 2  s . c  o  m
    if ("".equals(request.getName())) {
        throw new NFS4Exception(NFS4ERR_INVAL);
    }
    Path parentPath = server.getPath(session.getCurrentFileHandle());
    Path path = new Path(parentPath, request.getName());
    FileSystem fs = session.getFileSystem();
    if (!fs.exists(path)) {
        throw new NFS4Exception(NFS4ERR_NOENT);
    }
    REMOVEResponse response = createResponse();
    ChangeInfo changeInfo = new ChangeInfo();
    FileStatus parentStatus = fs.getFileStatus(parentPath);
    ChangeID changeIDBefore = new ChangeID();
    changeIDBefore.setChangeID(parentStatus.getModificationTime());
    changeInfo.setChangeIDBefore(changeIDBefore);

    fs.delete(path, false);

    parentStatus = fs.getFileStatus(parentPath);
    ChangeID changeIDAfter = new ChangeID();
    changeIDAfter.setChangeID(parentStatus.getModificationTime());
    changeInfo.setChangeIDAfter(changeIDAfter);
    changeInfo.setAtomic(true);
    response.setChangeInfo(changeInfo);
    response.setStatus(NFS4_OK);
    return response;
}

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.NFS4Handler.java

License:Apache License

/**
 * Files which are in the process of being created need to exist from a NFS
 * perspective even if they do not exist from an HDFS perspective. This
 * method intercepts requests for files that are open and calls
 * FileSystem.exists for other files./*from w  ww.j  a v a  2 s  .c  o m*/
 *
 * @param fs
 * @param path
 * @return true if the file exists or is open for write
 * @throws IOException
 */
public synchronized boolean fileExists(FileSystem fs, Path path) throws IOException {
    FileHolder fileHolder = mPathMap.get(realPath(path));
    if (fileHolder != null && fileHolder.isOpenForWrite()) {
        return true;
    }
    return fs.exists(path);
}