Example usage for org.apache.hadoop.fs FileSystem isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem isDirectory.

Prototype

@Deprecated
public boolean isDirectory(Path f) throws IOException 

Source Link

Document

True iff the named path is a directory.

Usage

From source file:com.ibm.bi.dml.runtime.transform.RecodeAgent.java

License:Open Source License

/**
 * Method to load recode maps of all attributes, at once.
 * //from   ww  w  .  java  2  s.  c  om
 * @param job
 * @throws IOException
 */
@Override
public void loadTxMtd(JobConf job, FileSystem fs, Path txMtdDir, TfUtils agents) throws IOException {
    if (_rcdList == null)
        return;

    _finalMaps = new HashMap<Integer, HashMap<String, String>>();

    if (fs.isDirectory(txMtdDir)) {
        for (int i = 0; i < _rcdList.length; i++) {
            int colID = _rcdList[i];

            Path path = new Path(txMtdDir + "/Recode/" + agents.getName(colID) + RCD_MAP_FILE_SUFFIX);
            TfUtils.checkValidInputFile(fs, path, true);

            HashMap<String, String> map = new HashMap<String, String>();

            BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(path)));
            String line = null, word = null;
            String rcdIndex = null;

            // Example line to parse: "WN (1)67492",1,61975
            while ((line = br.readLine()) != null) {

                // last occurrence of quotation mark
                int idxQuote = line.lastIndexOf('"');
                word = UtilFunctions.unquote(line.substring(0, idxQuote + 1));

                int idx = idxQuote + 2;
                while (line.charAt(idx) != TXMTD_SEP.charAt(0))
                    idx++;
                rcdIndex = line.substring(idxQuote + 2, idx);

                map.put(word, rcdIndex);
            }
            br.close();
            _finalMaps.put(colID, map);
        }
    } else {
        fs.close();
        throw new RuntimeException("Path to recode maps must be a directory: " + txMtdDir);
    }
}

---

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

/**
 * Returns the size of a file or directory on hdfs in bytes.
 * /*from  w w w  . ja  v a 2  s .  c  om*/
 * @param path
 * @return
 * @throws IOException
 */
public static long getFilesizeOnHDFS(Path path) throws IOException {
    FileSystem fs = FileSystem.get(_rJob);
    long ret = 0; //in bytes
    if (fs.isDirectory(path))
        ret = fs.getContentSummary(path).getLength();
    else
        ret = fs.getFileStatus(path).getLen();
    //note: filestatus would return 0 on directories

    return ret;
}

---

From source file:com.intel.hibench.datagen.streaming.util.SourceFileReader.java

License:Apache License

static public BufferedReader getReader(Configuration dfsConf, String path, long offset) {
    BufferedReader reader = null;
    try {//w ww .  j  a v  a  2s.com
        Path pt = new Path(path);
        FileSystem fs = FileSystem.get(dfsConf);
        InputStreamReader isr;
        if (fs.isDirectory(pt)) {
            //give path is an directory
            isr = new InputStreamReader(openMultipleParts(fs, pt, offset));
        } else {
            //give path is an file
            FSDataInputStream inputStream = fs.open(pt);
            if (offset > 0) {
                inputStream.seek(offset);
            }
            isr = new InputStreamReader(inputStream);
        }

        reader = new BufferedReader(isr);
    } catch (IOException e) {
        System.err.println("Fail to get reader from path: " + path);
        e.printStackTrace();
    }
    return reader;
}

---

From source file:com.intel.hibench.streambench.FileDataGenNew.java

License:Apache License

public BufferedReader loadDataFromFile(String filepath, long offset) {
    try {//w  ww .ja  v a2  s .  c  o  m
        Path pt = new Path(filepath);
        FileSystem fs = FileSystem.get(fsConf);
        InputStreamReader isr;
        if (fs.isDirectory(pt)) { // multiple parts
            isr = new InputStreamReader(OpenMultiplePartsWithOffset(fs, pt, offset));
        } else { // single file
            FSDataInputStream fileHandler = fs.open(pt);
            if (offset > 0)
                fileHandler.seek(offset);
            isr = new InputStreamReader(fileHandler);
        }

        BufferedReader reader = new BufferedReader(isr);
        if (offset > 0)
            reader.readLine(); // skip first line in case of seek
        return reader;
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}

---

From source file:com.linkedin.cubert.pig.piggybank.storage.avro.AvroStorage.java

License:Apache License

/**
 * Get avro schema of input path. There are three cases:
 * 1. if path is a file, then return its avro schema;
 * 2. if path is a first-level directory (no sub-directories), then
 * return the avro schema of one underlying file;
 * 3. if path contains sub-directories, then recursively check
 * whether all of them share the same schema and return it
 * if so or throw an exception if not./*ww w.j  av  a 2s .co m*/
 *
 * @param path input path
 * @param fs file system
 * @return avro schema of data
 * @throws IOException if underlying sub-directories do not share the same schema; or if input path is empty or does not exist
 */
@SuppressWarnings("deprecation")
protected Schema getAvroSchema(Path path, FileSystem fs) throws IOException {
    if (!fs.exists(path) || !AvroStorageUtils.PATH_FILTER.accept(path))
        return null;

    /* if path is first level directory or is a file */
    if (!fs.isDirectory(path)) {
        return getSchema(path, fs);
    }

    FileStatus[] ss = fs.listStatus(path, AvroStorageUtils.PATH_FILTER);
    Schema schema = null;
    if (ss.length > 0) {
        if (AvroStorageUtils.noDir(ss))
            return getSchema(path, fs);

        /*otherwise, check whether schemas of underlying directories are the same */
        for (FileStatus s : ss) {
            Schema newSchema = getAvroSchema(s.getPath(), fs);
            if (schema == null) {
                schema = newSchema;
                if (!checkSchema) {
                    System.out.println("Do not check schema; use schema of " + s.getPath());
                    return schema;
                }
            } else if (newSchema != null && !schema.equals(newSchema)) {
                throw new IOException("Input path is " + path + ". Sub-direcotry " + s.getPath()
                        + " contains different schema " + newSchema + " than " + schema);
            }
        }
    }

    if (schema == null)
        System.err.println("Cannot get avro schema! Input path " + path + " might be empty.");

    return schema;
}

---

From source file:com.liveramp.hank.hadoop.DomainBuilderAbstractOutputFormat.java

License:Apache License

public static void moveContentsAndDelete(Path srcDir, Path dstDir, FileSystem fs, Logger logger)
        throws IOException {
    if (!fs.exists(srcDir)) {
        return;/*from  w w  w .ja  va2  s .  com*/
    }
    if (fs.exists(srcDir) && !fs.isDirectory(srcDir)) {
        throw new IllegalArgumentException(srcDir + " is not a directory");
    }
    if (fs.exists(dstDir) && !fs.isDirectory(dstDir)) {
        throw new IllegalArgumentException(dstDir + " is not a directory");
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Moving contents of: " + srcDir + " to: " + dstDir);
    }
    FileStatus[] files = fs.listStatus(srcDir);
    for (FileStatus file : files) {
        Path sourcePath = file.getPath();
        Path targetPath = new Path(dstDir, file.getPath().getName());
        if (logger.isDebugEnabled()) {
            logger.debug("Moving: " + sourcePath + " to: " + targetPath);
        }
        if (!fs.mkdirs(targetPath.getParent())) {
            throw new IOException("Failed at creating directory " + targetPath.getParent());
        }
        if (!fs.rename(sourcePath, targetPath)) {
            throw new IOException("Failed at renaming " + sourcePath + " to " + targetPath);
        }
    }
    fs.delete(srcDir);
}

---

From source file:com.moz.fiji.mapreduce.DistributedCacheJars.java

License:Apache License

/**
 * Lists all jars in the specified directory.
 *
 * @param conf Configuration to get FileSystem from
 * @param jarDirectory The directory of jars to get.
 * @return A list of qualified paths to the jars in jarDirectory.
 * @throws IOException if there's a problem.
 */// ww  w  . j  a  v a 2  s  . c o  m
public static Collection<Path> listJarFilesFromDirectory(Configuration conf, Path jarDirectory)
        throws IOException {
    LOG.debug("Listing jar files {}/*.jar", jarDirectory);
    final FileSystem fs = jarDirectory.getFileSystem(conf);
    if (!fs.isDirectory(jarDirectory)) {
        throw new IOException("Attempted to add jars from non-directory: " + jarDirectory);
    }
    final List<Path> jarFiles = Lists.newArrayList();
    for (FileStatus status : fs.listStatus(jarDirectory)) {
        if (!status.isDir() && status.getPath().getName().endsWith(".jar")) {
            jarFiles.add(fs.makeQualified(status.getPath()));
        }
    }
    return jarFiles;
}

---

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

private String copyInHDFS(Channel channel, String rfile, String lfile, SSHDataStore remoteServer)
        throws Exception {

    String error = null;/*from w ww. jav  a2 s.c  o  m*/
    FileSystem fs = NameNodeVar.getFS();

    Map<String, String> p = remoteServer.getProperties(rfile);
    if (p.get(SSHInterface.key_type).equals("file")) {

        String nameRdm = RandomString.getRandomName(20);
        String tmpFileStr = System.getProperty("java.io.tmpdir") + "/" + nameRdm;

        if (channel.isClosed()) {
            channel.connect();
        }
        logger.info("Copy " + rfile + " to " + tmpFileStr);
        ((ChannelSftp) channel).get(rfile, tmpFileStr);
        logger.info("Copy local " + tmpFileStr + " to HDFS " + lfile);
        fs.copyFromLocalFile(new Path(tmpFileStr), new Path(lfile));
        new File(tmpFileStr).delete();

    } else {

        if (!fs.exists(new Path(lfile))) {
            if (!fs.mkdirs(new Path(lfile))) {
                // create the directory
                error = lfile + ": Cannot create such directory";
            }
        } else if (!fs.isDirectory(new Path(lfile))) {
            //already exists as a file
            error = lfile + ": Not a directory";
        }

        if (error == null) {
            logger.info("Create the directory " + lfile);

            Map<String, Map<String, String>> files = remoteServer.getChildrenProperties(rfile);
            logger.debug(files);

            for (String path : files.keySet()) {
                Map<String, String> props = files.get(path);

                logger.debug(props.get("type") + " " + path);

                String fileName = path.replaceFirst(rfile, "");
                //String fileName = path.substring(path.lastIndexOf("/"));
                logger.debug("fileName " + fileName);

                error = copyInHDFS(channel, rfile + fileName, lfile + fileName, remoteServer);
                if (error != null) {
                    break;
                }
            }
        }

    }

    return error;
}

---

From source file:com.redsqirl.workflow.server.oozie.SqirlNutcrackerAction.java

License:Open Source License

/**
 * Create an element for Idiro Engine Action in the Oozie action file
 * @param oozieXmlDoc//  ww  w  .  j ava 2 s .c om
 * @param action
 * @param fileNames
 * @throws RemoteException
 */
@Override
public void createOozieElement(Document oozieXmlDoc, Element action, String[] fileNames)
        throws RemoteException {

    logger.debug("createOozieElement SqirlNutcrackerAction ");

    Element java = oozieXmlDoc.createElement("java");

    defaultParam(oozieXmlDoc, java);
    String path = WorkflowPrefManager.getProperty(WorkflowPrefManager.sys_nutcracker_path);

    logger.debug("createOozieElement path " + path);

    try {
        FileSystem fs = NameNodeVar.getFS();
        if (fs.isDirectory(new Path(path))) {

            FileStatus[] fileStatus = fs.listStatus(new Path(path));

            Element property = oozieXmlDoc.createElement("property");
            Element confName = oozieXmlDoc.createElement("name");
            confName.appendChild(oozieXmlDoc.createTextNode("oozie.launcher.fs.hdfs.impl.disable.cache"));
            Element confValue = oozieXmlDoc.createElement("value");
            confValue.appendChild(oozieXmlDoc.createTextNode("true"));
            property.appendChild(confName);
            property.appendChild(confValue);
            Element configuration = (Element) java.getElementsByTagName("configuration").item(0);
            configuration.appendChild(property);

            property = oozieXmlDoc.createElement("property");
            confName = oozieXmlDoc.createElement("name");
            confValue = oozieXmlDoc.createElement("value");
            confName.appendChild(oozieXmlDoc.createTextNode("oozie.launcher.oozie.libpath"));
            confValue.appendChild(oozieXmlDoc.createTextNode(path));

            property.appendChild(confName);
            property.appendChild(confValue);
            configuration.appendChild(property);

            Element mainClass = oozieXmlDoc.createElement("main-class");
            mainClass.appendChild(oozieXmlDoc.createTextNode("com.sqirlnutcracker.SqirlNutcrackerMain"));
            java.appendChild(mainClass);

            Element javaopts = oozieXmlDoc.createElement("java-opts");
            javaopts.appendChild(oozieXmlDoc.createTextNode("-Duser.name=" + System.getProperty("user.name")));
            java.appendChild(javaopts);

            Element arg1 = oozieXmlDoc.createElement("arg");
            arg1.appendChild(oozieXmlDoc.createTextNode("namenode=${" + OozieManager.prop_namenode
                    + "},jobtracker=${" + OozieManager.prop_jobtracker + "}"));
            java.appendChild(arg1);

            Element arg2 = oozieXmlDoc.createElement("arg");
            String[] filename = fileNames[0].split("/");
            arg2.appendChild(oozieXmlDoc.createTextNode(filename[filename.length - 1]));
            java.appendChild(arg2);

            Element file = oozieXmlDoc.createElement("file");
            file.appendChild(oozieXmlDoc.createTextNode(fileNames[0]));
            java.appendChild(file);

            action.appendChild(java);
        } else {
            logger.debug("createOozieElement isDirectory false ");
        }

    } catch (IOException e) {
        e.printStackTrace();
        logger.debug("createOozieElement error " + e);
    }

}

---

From source file:com.splicemachine.derby.impl.io.HdfsDirFile.java

License:Apache License

@Override
public boolean isDirectory() {
    try {//  www  . j  a  v  a2 s  .com
        FileSystem fs = getFileSystem();
        return fs.isDirectory(new Path(path));
    } catch (IOException e) {
        LOG.error(String.format("An exception occurred while checking if the path '%s' is a directory.", path),
                e);
        return false;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• »