Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException 

Source Link

Document

Check if a path exists.

Usage

From source file:azkaban.jobtype.javautils.HadoopUtils.java

License:Apache License

public static JobConf addAllSubPaths(JobConf conf, Path path) throws IOException {
    if (shouldPathBeIgnored(path)) {
        throw new IllegalArgumentException(String.format("Path[%s] should be ignored.", path));
    }//from  w w  w.java2  s .  co  m

    final FileSystem fs = path.getFileSystem(conf);

    if (fs.exists(path)) {
        for (FileStatus status : fs.listStatus(path)) {
            if (!shouldPathBeIgnored(status.getPath())) {
                if (status.isDir()) {
                    addAllSubPaths(conf, status.getPath());
                } else {
                    FileInputFormat.addInputPath(conf, status.getPath());
                }
            }
        }
    }
    return conf;
}

---

From source file:azkaban.jobtype.javautils.HadoopUtils.java

License:Apache License

public static void saveProps(FileSystem fs, Props props, String file) throws IOException {
    Path path = new Path(file);

    // create directory if it does not exist.
    Path parent = path.getParent();
    if (!fs.exists(parent))
        fs.mkdirs(parent);/*from  w  w  w  .j a  va2  s .  co m*/

    // write out properties
    OutputStream output = fs.create(path);
    try {
        props.storeFlattened(output);
    } finally {
        output.close();
    }
}

---

From source file:azkaban.jobtype.javautils.Whitelist.java

License:Open Source License

/**
 * Updates whitelist if there's any change. If it needs to update whitelist, it enforces writelock
 * to make sure//ww  w . j  ava  2  s.c  om
 * there's an exclusive access on shared variables.
 */
@VisibleForTesting
Set<String> retrieveWhitelist(FileSystem fs, Path path) {
    try {
        Preconditions.checkArgument(fs.exists(path), "File does not exist at " + path);
        Preconditions.checkArgument(fs.isFile(path), "Whitelist path is not a file. " + path);

        Set<String> result = Sets.newHashSet();
        try (BufferedReader br = new BufferedReader(
                new InputStreamReader(fs.open(path), StandardCharsets.UTF_8))) {
            String s = null;
            while (!StringUtils.isEmpty((s = br.readLine()))) {
                result.add(s);
            }
        }
        return result;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:azkaban.viewer.hdfs.HdfsBrowserServlet.java

License:Apache License

private void handleFsDisplay(String user, HttpServletRequest req, HttpServletResponse resp, Session session)
        throws IOException, ServletException, IllegalArgumentException, IllegalStateException {
    FileSystem fs = null;
    try {// w w w.  j  a  va2  s .  c o  m
        fs = getFileSystem(user);
    } catch (HadoopSecurityManagerException e) {
        errorPage(user, req, resp, session, "Cannot get FileSystem.");
        return;
    }

    Path path = getPath(req);
    if (logger.isDebugEnabled()) {
        logger.debug("path: '" + path.toString() + "'");
    }

    try {
        if (!fs.exists(path)) {
            errorPage(user, req, resp, session, path.toUri().getPath() + " does not exist.");
            fs.close();
            return;
        }
    } catch (IOException ioe) {
        logger.error("Got exception while checking for existence of path '" + path + "'", ioe);
        errorPage(user, req, resp, session,
                path.toUri().getPath() + " Encountered error while trying to detect if path '" + path
                        + "' exists. Reason: " + ioe.getMessage());
        fs.close();
        return;
    }

    if (fs.isFile(path)) {
        displayFilePage(fs, user, req, resp, session, path);
    } else if (fs.getFileStatus(path).isDir()) {
        displayDirPage(fs, user, req, resp, session, path);
    } else {
        errorPage(user, req, resp, session,
                "It exists, it is not a file, and it is not a directory, what " + "is it precious?");
    }
    fs.close();
}

---

From source file:azkaban.viewer.hdfs.HdfsBrowserServlet.java

License:Apache License

private void handleAjaxAction(String username, HttpServletRequest request, HttpServletResponse response,
        Session session) throws ServletException, IOException {
    Map<String, Object> ret = new HashMap<String, Object>();
    FileSystem fs = null;
    try {/*  w  ww  .  ja  va  2s.c o  m*/
        try {
            fs = getFileSystem(username);
        } catch (HadoopSecurityManagerException e) {
            errorAjax(response, ret, "Cannot get FileSystem.");
            return;
        }

        String ajaxName = getParam(request, "ajax");
        Path path = null;
        if (!hasParam(request, "path")) {
            errorAjax(response, ret, "Missing parameter 'path'.");
            return;
        }

        path = new Path(getParam(request, "path"));
        if (!fs.exists(path)) {
            errorAjax(response, ret, path.toUri().getPath() + " does not exist.");
            return;
        }

        if (ajaxName.equals("fetchschema")) {
            handleAjaxFetchSchema(fs, request, ret, session, path);
        } else if (ajaxName.equals("fetchfile")) {
            // Note: fetchFile writes directly to the output stream. Thus, we need
            // to make sure we do not write to the output stream once this call
            // returns.
            ret = null;
            handleAjaxFetchFile(fs, request, response, session, path);
        } else {
            ret.put("error", "Unknown AJAX action " + ajaxName);
        }

        if (ret != null) {
            this.writeJSON(response, ret);
        }
    } finally {
        fs.close();
    }
}

---

From source file:babel.prep.corpus.MultipleXMLLangFileOutputFormat.java

License:Apache License

public RecordWriter<Text, Page> getBaseRecordWriter(final FileSystem fs, JobConf job, String name,
        final Progressable progress) throws IOException {
    final Path dumpFile = new Path(FileOutputFormat.getOutputPath(job), name);

    // Get the old copy out of the way
    if (fs.exists(dumpFile))
        fs.delete(dumpFile, true);//from   ww  w.  j  a v a 2s  .  c  o  m

    final XMLObjectWriter xmlWriter;

    try {
        xmlWriter = new XMLObjectWriter(fs.create(dumpFile), false);
    } catch (Exception e) {
        throw new RuntimeException("Failed to instantiate XMLObjectWriter.");
    }

    return new RecordWriter<Text, Page>() {
        public synchronized void write(Text key, Page page) throws IOException {
            try {
                xmlWriter.write(page);
            } catch (XMLStreamException e) {
                throw new RuntimeException("Error writing page XML.");
            }
        }

        public synchronized void close(Reporter reporter) throws IOException {
            try {
                xmlWriter.close();
            } catch (XMLStreamException e) {
                throw new RuntimeException("Error closing XMLObjectWriter.");
            }
        }
    };
}

---

From source file:babel.prep.datedcorpus.DatedLangFilesOutputFormat.java

License:Apache License

public RecordWriter<Text, Text> getBaseRecordWriter(final FileSystem fs, JobConf job, String name,
        final Progressable progress) throws IOException {
    final Path dumpFile = new Path(FileOutputFormat.getOutputPath(job), name);

    // Get the old copy out of the way
    if (fs.exists(dumpFile)) {
        fs.delete(dumpFile, true);//w  w  w  .  j  a va2  s .  c  om
    } else {
        fs.mkdirs(dumpFile.getParent());
    }

    return new RecordWriter<Text, Text>() {
        public synchronized void write(Text key, Text versText) throws IOException {
            try {
                BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
                        new FileOutputStream(new File(dumpFile.toUri()), true), DEFAULT_CHARSET));

                writer.write(versText.toString());
                writer.close();
            } catch (Exception e) {
                throw new RuntimeException("Error writing page versions: " + e.toString());
            }
        }

        public synchronized void close(Reporter reporter) throws IOException {
        }
    };
}

---

From source file:backup.store.ExternalExtendedBlockSort.java

License:Apache License

public ExtendedBlockEnum<T> getBlockEnum(String blockPoolId) throws Exception {
    Path output = getOutputFilePath(blockPoolId);
    FileSystem fileSystem = output.getFileSystem(conf);
    if (!fileSystem.exists(output)) {
        return null;
    }/*w w w  .  ja va  2s .c  o  m*/
    return new BlockEnum(blockPoolId, new Reader(conf, Reader.file(output)));
}

---

From source file:backup.store.ExternalExtendedBlockSort.java

License:Apache License

private synchronized void sortIfNeeded() throws IOException {
    for (String blockPoolId : writers.keySet()) {
        Path output = getOutputFilePath(blockPoolId);
        Path input = getInputFilePath(blockPoolId);
        FileSystem fileSystem = output.getFileSystem(conf);
        if (!fileSystem.exists(output) && fileSystem.exists(input)) {
            LocalFileSystem local = FileSystem.getLocal(conf);
            SequenceFile.Sorter sorter = new Sorter(local, ComparableBlock.class, dataClass, conf);
            sorter.sort(input, output);/*www .jav a2s . com*/
        }
    }
}

---

From source file:batch.BatchScan2Html.java

License:Apache License

public static void writeAccumuloTableToHdfsAsHtml() throws IOException, URISyntaxException {
    Configuration configuration = new Configuration();
    //TODO add options for URI and output Path
    FileSystem hdfs = FileSystem.get(new URI("hdfs://n001:54310"), configuration);
    Path file = new Path("hdfs://n001:54310/s2013/batch/table.html");
    //TODO add option to override file default: true
    if (hdfs.exists(file)) {
        hdfs.delete(file, true);//from  w  w w  .  j  a v a 2  s.  co  m
    }
    startTime = System.currentTimeMillis();
    OutputStream os = hdfs.create(file, new Progressable() {
        public void progress() {
            // TODO add a better progress descriptor
            crudeRunTime = System.currentTimeMillis() - startTime;
            out.println("...bytes written: [ " + bytesWritten + " ]");
            out.println("...bytes / second: [ " + (bytesWritten / crudeRunTime) * 1000 + " ]");
        }
    });
    BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
    //  TODO add option for table id { example }
    writeHtmlTableHeader(br, "example", new ArrayList<String>(Arrays.asList("Row ID", "Column Family",
            "Column Qualifier", "Column Visibility", "Timestamp", "Value")));
    writeHtmlTableBody(br);
    out.println("Total bytes written: " + bytesWritten);
    out.println("Total crude time: " + crudeRunTime / 1000);
    br.close();
    hdfs.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »