Example usage for org.apache.hadoop.fs FileSystem open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem open.

Prototype

public FSDataInputStream open(PathHandle fd) throws IOException 

Source Link

Document

Open an FSDataInputStream matching the PathHandle instance.

Usage

From source file:com.dalabs.droop.util.password.FilePasswordLoader.java

License:Apache License

/**
 * Read bytes from given file./* w w w.j  a  v  a  2 s.c o m*/
 *
 * @param fs Associated FileSystem
 * @param path Path
 * @return
 * @throws IOException
 */
protected byte[] readBytes(FileSystem fs, Path path) throws IOException {
    InputStream is = fs.open(path);
    try {
        return IOUtils.toByteArray(is);
    } finally {
        IOUtils.closeQuietly(is);
    }
}

---

From source file:com.dasasian.chok.util.FileUtil.java

License:Apache License

/**
 * Simply unzips the content from the source zip to the target folder. The
 * first level folder of the zip content is removed.
 *
 * @param sourceZip    the path to the source zip file, hadoop's IO services are used to
 *                     open this path//  www  . j a  v a  2s  .  c  o  m
 * @param targetFolder The directory that the zip file will be unpacked into
 * @param fileSystem   the hadoop file system object to use to open
 *                     <code>sourceZip</code>
 * @param localSpool   If true, the zip file is copied to the local file system before
 *                     being unzipped. The name used is <code>targetFolder.zip</code>. If
 *                     false, the unzip is streamed.
 */
public static void unzip(final Path sourceZip, final File targetFolder, final FileSystem fileSystem,
        final boolean localSpool) {
    try {
        if (localSpool) {
            targetFolder.mkdirs();
            final File shardZipLocal = new File(targetFolder + ".zip");
            if (shardZipLocal.exists()) {
                // make sure we overwrite cleanly
                shardZipLocal.delete();
            }
            try {
                fileSystem.copyToLocalFile(sourceZip, new Path(shardZipLocal.getAbsolutePath()));
                FileUtil.unzip(shardZipLocal, targetFolder);
            } finally {
                shardZipLocal.delete();
            }
        } else {
            FSDataInputStream fis = fileSystem.open(sourceZip);
            try {
                ZipInputStream zis = new ZipInputStream(fis);
                unzip(zis, targetFolder);
            } finally {
                if (fis != null) {
                    try {
                        fis.close();
                    } catch (Exception ignore) {
                        // ignore
                    }
                }
            }
        }
    } catch (IOException e) {
        throw new RuntimeException("unable to expand upgrade files for " + sourceZip + " to " + targetFolder,
                e);
    }

}

---

From source file:com.dasasian.chok.util.FileUtil.java

License:Apache License

public static void unzipInDfs(FileSystem fileSystem, final Path source, final Path target) {
    try {/*w  w w  .j a v a2 s .co  m*/
        FSDataInputStream dfsInputStream = fileSystem.open(source);
        fileSystem.mkdirs(target);
        final ZipInputStream zipInputStream = new ZipInputStream(dfsInputStream);
        ZipEntry entry;

        while ((entry = zipInputStream.getNextEntry()) != null) {
            final String entryPath = entry.getName();
            final int indexOf = entryPath.indexOf("/");
            final String cleanUpPath = entryPath.substring(indexOf + 1, entryPath.length());
            Path path = target;
            if (!cleanUpPath.equals("")) {
                path = new Path(target, cleanUpPath);
            }
            LOG.info("Extracting: " + entry + " to " + path);
            if (entry.isDirectory()) {
                fileSystem.mkdirs(path);
            } else {
                int count;
                final byte data[] = new byte[4096];
                FSDataOutputStream fsDataOutputStream = fileSystem.create(path);
                while ((count = zipInputStream.read(data, 0, 4096)) != -1) {
                    fsDataOutputStream.write(data, 0, count);
                }
                fsDataOutputStream.flush();
                fsDataOutputStream.close();
            }
        }
        zipInputStream.close();
    } catch (final Exception e) {
        LOG.error("can not open zip file", e);
        throw new RuntimeException("unable to expand upgrade files", e);
    }

}

---

From source file:com.datasalt.pangool.utils.HadoopUtils.java

License:Apache License

/**
 * Reads the content of a file into a String. Return null if the file does not
 * exist.//from w  ww  . j a v  a  2 s.c o m
 */
public static String fileToString(FileSystem fs, Path path) throws IOException {
    if (!fs.exists(path)) {
        return null;
    }

    InputStream is = fs.open(path);
    InputStreamReader isr = new InputStreamReader(is);
    BufferedReader br = new BufferedReader(isr);
    char[] buff = new char[256];
    StringBuilder sb = new StringBuilder();
    int read;
    while ((read = br.read(buff)) != -1) {
        sb.append(buff, 0, read);
    }
    br.close();
    return sb.toString();
}

---

From source file:com.datascience.hadoop.CsvInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, ListWritable<Text>> getRecordReader(InputSplit inputSplit, JobConf conf,
        Reporter reporter) throws IOException {
    String charsetName = conf.get(CHARSET);
    Charset charset = charsetName != null ? Charset.forName(charsetName) : StandardCharsets.UTF_8;

    FileSplit split = (FileSplit) inputSplit;
    Path path = split.getPath();//www . j a v  a  2 s .  c o m
    FileSystem fs = path.getFileSystem(conf);
    InputStream is = fs.open(path);

    // If the input is compressed, load the compression codec.
    CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf);
    CompressionCodec codec = codecFactory.getCodec(path);
    if (codec != null) {
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        is = codec.createInputStream(is, decompressor);
    }
    return new CsvRecordReader(new InputStreamReader(is, charset), createFormat(conf), split.getLength(),
            conf.getBoolean(STRICT_MODE, true));
}

---

From source file:com.datatorrent.contrib.parser.AbstractCsvParser.java

License:Apache License

@Override
public void setup(OperatorContext context) {
    if (fieldmappingFile != null) {
        Configuration conf = new Configuration();
        try {//from   w  w w.ja  v a2s.co m
            FileSystem fs = FileSystem.get(conf);
            Path filepath = new Path(fieldmappingFile);
            if (fs.exists(filepath)) {
                BufferedReader bfr = new BufferedReader(new InputStreamReader(fs.open(filepath)));
                String str;

                while ((str = bfr.readLine()) != null) {
                    logger.debug("string is {}", str);
                    String[] temp = str.split(fieldmappingFileDelimiter);
                    Field field = new Field();
                    field.setName(temp[0]);
                    field.setType(temp[1]);
                    getFields().add(field);
                }
            } else {
                logger.debug(
                        "File containing fields and their data types does not exist.Please specify the fields and data type through properties of this operator.");
            }
        } catch (IOException ex) {
            DTThrowable.rethrow(ex);
        }

    }

    int countKeyValue = getFields().size();
    properties = new String[countKeyValue];
    processors = new CellProcessor[countKeyValue];
    initialise(properties, processors);
    CsvPreference preference = new CsvPreference.Builder('"', fieldDelimiter, lineDelimiter).build();
    csvReader = getReader(csvStringReader, preference);

}

---

From source file:com.datatorrent.lib.parser.XmlParser.java

License:Apache License

@Override
public void setup(com.datatorrent.api.Context.OperatorContext context) {
    try {//from  w  w w . j  a va  2  s.  co m
        JAXBContext ctx = JAXBContext.newInstance(getClazz());
        unmarshaller = ctx.createUnmarshaller();
        if (schemaXSDFile != null) {
            Path filePath = new Path(schemaXSDFile);
            Configuration configuration = new Configuration();
            FileSystem fs = FileSystem.newInstance(filePath.toUri(), configuration);
            FSDataInputStream inputStream = fs.open(filePath);

            SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
            Schema schema = factory.newSchema(new StreamSource(inputStream));
            unmarshaller.setSchema(schema);
            validator = schema.newValidator();
            fs.close();
        }
    } catch (SAXException e) {
        DTThrowable.wrapIfChecked(e);
    } catch (JAXBException e) {
        DTThrowable.wrapIfChecked(e);
    } catch (IOException e) {
        DTThrowable.wrapIfChecked(e);
    }
}

---

From source file:com.datatorrent.stram.debug.TupleRecorderTest.java

License:Apache License

@Test
public void testRecorder() throws IOException {
    FileSystem fs = new LocalFileSystem();
    try {//from www  .  j a  v  a  2s  .c o  m
        TupleRecorder recorder = new TupleRecorder(null, "application_test_id_1");
        recorder.getStorage().setBytesPerPartFile(4096);
        recorder.getStorage().setLocalMode(true);
        recorder.getStorage().setBasePath("file://" + testWorkDir.getAbsolutePath() + "/recordings");

        recorder.addInputPortInfo("ip1", "str1");
        recorder.addInputPortInfo("ip2", "str2");
        recorder.addInputPortInfo("ip3", "str3");
        recorder.addOutputPortInfo("op1", "str4");
        recorder.setup(null, null);

        recorder.beginWindow(1000);
        recorder.beginWindow(1000);
        recorder.beginWindow(1000);

        Tuple t1 = new Tuple();
        t1.key = "speed";
        t1.value = "5m/h";
        recorder.writeTuple(t1, "ip1");
        recorder.endWindow();
        Tuple t2 = new Tuple();
        t2.key = "speed";
        t2.value = "4m/h";
        recorder.writeTuple(t2, "ip3");
        recorder.endWindow();
        Tuple t3 = new Tuple();
        t3.key = "speed";
        t3.value = "6m/h";
        recorder.writeTuple(t3, "ip2");
        recorder.endWindow();

        recorder.beginWindow(1000);
        Tuple t4 = new Tuple();
        t4.key = "speed";
        t4.value = "2m/h";
        recorder.writeTuple(t4, "op1");
        recorder.endWindow();
        recorder.teardown();

        fs.initialize((new Path(recorder.getStorage().getBasePath()).toUri()), new Configuration());
        Path path;
        FSDataInputStream is;
        String line;
        BufferedReader br;

        path = new Path(recorder.getStorage().getBasePath(), FSPartFileCollection.INDEX_FILE);
        is = fs.open(path);
        br = new BufferedReader(new InputStreamReader(is));

        line = br.readLine();
        //    Assert.assertEquals("check index", "B:1000:T:0:part0.txt", line);
        Assert.assertTrue("check index", line.matches(
                "F:part0.txt:\\d+-\\d+:4:T:1000-1000:33:\\{\"3\":\"1\",\"1\":\"1\",\"0\":\"1\",\"2\":\"1\"\\}"));

        path = new Path(recorder.getStorage().getBasePath(), FSPartFileCollection.META_FILE);
        is = fs.open(path);
        br = new BufferedReader(new InputStreamReader(is));

        ObjectMapper mapper = new ObjectMapper();
        line = br.readLine();
        Assert.assertEquals("check version", "1.2", line);
        br.readLine(); // RecordInfo
        //RecordInfo ri = mapper.readValue(line, RecordInfo.class);
        line = br.readLine();
        PortInfo pi = mapper.readValue(line, PortInfo.class);
        Assert.assertEquals("port1", recorder.getPortInfoMap().get(pi.name).id, pi.id);
        Assert.assertEquals("port1", recorder.getPortInfoMap().get(pi.name).type, pi.type);
        line = br.readLine();
        pi = mapper.readValue(line, PortInfo.class);
        Assert.assertEquals("port2", recorder.getPortInfoMap().get(pi.name).id, pi.id);
        Assert.assertEquals("port2", recorder.getPortInfoMap().get(pi.name).type, pi.type);
        line = br.readLine();
        pi = mapper.readValue(line, PortInfo.class);
        Assert.assertEquals("port3", recorder.getPortInfoMap().get(pi.name).id, pi.id);
        Assert.assertEquals("port3", recorder.getPortInfoMap().get(pi.name).type, pi.type);
        line = br.readLine();
        pi = mapper.readValue(line, PortInfo.class);
        Assert.assertEquals("port4", recorder.getPortInfoMap().get(pi.name).id, pi.id);
        Assert.assertEquals("port4", recorder.getPortInfoMap().get(pi.name).type, pi.type);
        Assert.assertEquals("port size", 4, recorder.getPortInfoMap().size());
        //line = br.readLine();

        path = new Path(recorder.getStorage().getBasePath(), "part0.txt");
        is = fs.open(path);
        br = new BufferedReader(new InputStreamReader(is));

        line = br.readLine();
        Assert.assertTrue("check part0", line.startsWith("B:"));
        Assert.assertTrue("check part0", line.endsWith(":1000"));

        line = br.readLine();
        Assert.assertTrue("check part0 1", line.startsWith("T:"));
        Assert.assertTrue("check part0 1", line.endsWith(":0:30:{\"key\":\"speed\",\"value\":\"5m/h\"}"));

        line = br.readLine();
        Assert.assertTrue("check part0 2", line.startsWith("T:"));
        Assert.assertTrue("check part0 2", line.endsWith(":2:30:{\"key\":\"speed\",\"value\":\"4m/h\"}"));

        line = br.readLine();
        Assert.assertTrue("check part0 3", line.startsWith("T:"));
        Assert.assertTrue("check part0 3", line.endsWith(":1:30:{\"key\":\"speed\",\"value\":\"6m/h\"}"));

        line = br.readLine();
        Assert.assertTrue("check part0 4", line.startsWith("T:"));
        Assert.assertTrue("check part0 4", line.endsWith(":3:30:{\"key\":\"speed\",\"value\":\"2m/h\"}"));

        line = br.readLine();
        Assert.assertTrue("check part0 5", line.startsWith("E:"));
        Assert.assertTrue("check part0 5", line.endsWith(":1000"));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    } finally {
        fs.close();
    }
}

---

From source file:com.datatorrent.stram.util.FSUtil.java

License:Apache License

/**
 * Copied from FileUtil to transfer ownership
 *
 * @param srcFS/*from  www .  jav  a2 s.  c o  m*/
 * @param srcStatus
 * @param dstFS
 * @param dst
 * @param deleteSource
 * @param overwrite
 * @param conf
 * @return
 * @throws IOException
 */
public static boolean copy(FileSystem srcFS, FileStatus srcStatus, FileSystem dstFS, Path dst,
        boolean deleteSource, boolean overwrite, Configuration conf) throws IOException {
    Path src = srcStatus.getPath();
    //dst = checkDest(src.getName(), dstFS, dst, overwrite);
    if (srcStatus.isDirectory()) {
        //checkDependencies(srcFS, src, dstFS, dst);
        if (!mkdirs(dstFS, dst)) {
            return false;
        }

        FileStatus contents[] = srcFS.listStatus(src);
        for (int i = 0; i < contents.length; i++) {
            copy(srcFS, contents[i], dstFS, new Path(dst, contents[i].getPath().getName()), deleteSource,
                    overwrite, conf);
        }
    } else {
        InputStream in = null;
        OutputStream out = null;
        try {
            in = srcFS.open(src);
            out = dstFS.create(dst, overwrite);
            org.apache.hadoop.io.IOUtils.copyBytes(in, out, conf, true);
        } catch (IOException e) {
            org.apache.hadoop.io.IOUtils.closeStream(out);
            org.apache.hadoop.io.IOUtils.closeStream(in);
            throw e;
        }
    }

    // TODO: change group and limit write to group
    if (srcStatus.isDirectory()) {
        dstFS.setPermission(dst, new FsPermission((short) 0777));
    } else {
        dstFS.setPermission(dst, new FsPermission((short) 0777)/*"ugo+w"*/);
    }
    //dstFS.setOwner(dst, null, srcStatus.getGroup());

    /*
        try {
          // transfer owner
          // DOES NOT WORK only super user can change file owner
          dstFS.setOwner(dst, srcStatus.getOwner(), srcStatus.getGroup());
        } catch (IOException e) {
          LOG.warn("Failed to change owner on {} to {}", dst, srcStatus.getOwner(), e);
          throw e;
        }
    */
    if (deleteSource) {
        return srcFS.delete(src, true);
    } else {
        return true;
    }

}

---

From source file:com.davidgildeh.hadoop.utils.FileUtils.java

License:Apache License

/**
 * Loads a Properties object with key/value properties from file on HDFS
 * //www  .jav a  2  s  .c o  m
 * @param path          The path to the properties file
 * @return              The initialised properties loaded from file
 * @throws IOException 
 */
public static Properties loadPropertiesFile(String path) throws IOException {

    // Properties Object to load properties file
    Properties propFile = new Properties();

    Path fsPath = new Path(path);
    FileSystem fileSystem = getFileSystem(fsPath);
    checkFileExists(fileSystem, fsPath);
    FSDataInputStream file = fileSystem.open(fsPath);
    propFile.load(file);
    file.close();
    fileSystem.close();
    return propFile;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• »