Example usage for org.apache.hadoop.io IOUtils closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream) 

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:org.apache.pig.piggybank.storage.AllLoader.java

License:Apache License

@Override
public void prepareToRead(@SuppressWarnings("rawtypes") RecordReader reader, PigSplit split)
        throws IOException {

    AllReader allReader = (AllReader) reader;

    if (currentPath == null || !(currentPath.equals(allReader.path))) {
        currentPathPartitionKeyMap = (partitionColumns == null) ? null
                : pathPartitionerHelper.getPathPartitionKeyValues(allReader.path.toString());
        currentPath = allReader.path;//from   ww w  .j  a va 2 s .  com
    }

    childLoadFunc = allReader.prepareLoadFuncForReading(split);

    String projectProperty = getUDFContext().getProperty(PROJECTION_ID);

    if (projectProperty != null) {

        // load the required field list from the current UDF context
        ByteArrayInputStream input = new ByteArrayInputStream(
                Base64.decodeBase64(projectProperty.getBytes("UTF-8")));

        ObjectInputStream objInput = new ObjectInputStream(input);

        try {
            requiredFieldList = (RequiredFieldList) objInput.readObject();
        } catch (ClassNotFoundException e) {
            throw new FrontendException(e.toString(), e);
        } finally {
            IOUtils.closeStream(objInput);
        }

        if (childLoadFunc.getClass().isAssignableFrom(LoadPushDown.class)) {
            supportPushDownProjection = true;
            ((LoadPushDown) childLoadFunc).pushProjection(requiredFieldList);
        } else {
            if (requiredFieldList != null) {
                requiredFieldHashSet = new TreeSet<Integer>();
                for (RequiredField requiredField : requiredFieldList.getFields()) {
                    requiredFieldHashSet.add(requiredField.getIndex());
                }
            }
        }

    }

}

---

From source file:org.apache.pig.piggybank.storage.AllLoader.java

License:Apache License

@Override
public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException {
    // save the required field list to the UDFContext properties.

    Properties properties = getUDFContext();

    ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
    ObjectOutputStream objOut = null;
    try {/*from  w ww  .  ja v a 2  s.  com*/
        objOut = new ObjectOutputStream(byteArray);
        objOut.writeObject(requiredFieldList);
    } catch (IOException e) {
        throw new FrontendException(e.toString(), e);
    } finally {
        IOUtils.closeStream(objOut);
    }

    // write out the whole required fields list as a base64 string
    try {
        properties.setProperty(PROJECTION_ID,
                new String(Base64.encodeBase64(byteArray.toByteArray()), "UTF-8"));
    } catch (UnsupportedEncodingException e) {
        throw new FrontendException(e.toString(), e);
    }

    return new RequiredFieldResponse(true);
}

---

From source file:org.apache.pig.piggybank.test.storage.TestSequenceFileLoader.java

License:Apache License

@Override
public void setUp() throws Exception {
    pigServer = new PigServer(LOCAL);
    File tmpFile = File.createTempFile("test", ".txt");
    tmpFileName = tmpFile.getAbsolutePath();
    System.err.println("fileName: " + tmpFileName);
    Path path = new Path("file:///" + tmpFileName);
    JobConf conf = new JobConf();
    FileSystem fs = FileSystem.get(path.toUri(), conf);

    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {// w  ww .  j  av a 2  s. co  m
        writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
        for (int i = 0; i < DATA.length; i++) {
            key.set(i);
            value.set(DATA[i]);
            writer.append(key, value);
        }
    } finally {
        IOUtils.closeStream(writer);
    }
}

---

From source file:org.apache.pig.piggybank.test.storage.TestSequenceFileLoader.java

License:Apache License

@Test
public void testReadBytesWritable() throws IOException {
    File inputFile = File.createTempFile("test", ".txt");
    System.err.println("fileName: " + inputFile.getAbsolutePath());
    Path path = new Path("file:///" + inputFile.getAbsolutePath());
    JobConf conf = new JobConf();
    FileSystem fs = FileSystem.get(path.toUri(), conf);

    IntWritable key = new IntWritable();
    SequenceFile.Writer writer = null;
    try {// w  ww. j a v  a 2 s .c o  m
        writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), BytesWritable.class);
        int numRecords = 3;
        for (int i = 0; i < numRecords; i++) {
            key.set(i);
            String val = "" + Math.pow(10, (numRecords - i));
            writer.append(key, new BytesWritable(val.getBytes()));
        }
    } finally {
        IOUtils.closeStream(writer);
    }

    Data data = resetData(pigServer);
    data.set("expected", tuple(0L, new DataByteArray("1000.0")), tuple(1L, new DataByteArray("100.0")),
            tuple(2L, new DataByteArray("10.0")));

    pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(inputFile.getAbsolutePath())
            + "' USING org.apache.pig.piggybank.storage.SequenceFileLoader() AS (key:long, val);");
    pigServer.registerQuery("STORE A into 'actual' USING mock.Storage();");

    assertEquals(data.get("expected"), data.get("actual"));

}

---

From source file:org.apache.slider.common.tools.ConfigHelper.java

License:Apache License

/**
 * This will load and parse a configuration to an XML document
 * @param fs filesystem//from   www .j a  va2  s. c om
 * @param path path
 * @return an XML document
 * @throws IOException IO failure
 */
public Document parseConfiguration(FileSystem fs, Path path) throws IOException {

    byte[] data = loadBytes(fs, path);
    //this is here to track down a parse issue
    //related to configurations
    String s = new String(data, 0, data.length);
    log.debug("XML resource {} is \"{}\"", path, s);
    /* JDK7
        try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
          Document document = parseConfigXML(in);
          return document;
        } catch (ParserConfigurationException | SAXException e) {
          throw new IOException(e);
        }
    */
    ByteArrayInputStream in = null;
    try {
        in = new ByteArrayInputStream(data);
        Document document = parseConfigXML(in);
        return document;
    } catch (ParserConfigurationException e) {
        throw new IOException(e);
    } catch (SAXException e) {
        throw new IOException(e);
    } finally {
        IOUtils.closeStream(in);
    }
}

---

From source file:org.apache.slider.common.tools.ConfigHelper.java

License:Apache License

public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
    int len = (int) fs.getLength(path);
    byte[] data = new byte[len];
    /* JDK7/*from  w  ww. j  av a 2  s . c  o m*/
    try(FSDataInputStream in = fs.open(path)) {
      in.readFully(0, data);
    }
    */
    FSDataInputStream in = null;
    in = fs.open(path);
    try {
        in.readFully(0, data);
    } finally {
        IOUtils.closeStream(in);
    }
    return data;
}

---

From source file:org.apache.slider.common.tools.CoreFileSystem.java

License:Apache License

/**
 * Verify that a user has write access to a directory.
 * It does this by creating then deleting a temp file
 *
 * @param dirPath actual directory to look for
 * @throws FileNotFoundException file not found
 * @throws IOException  trouble with FS//from   w  w  w  . j  av a2 s  .  co  m
 * @throws BadClusterStateException if the directory is not writeable
 */
public void verifyDirectoryWriteAccess(Path dirPath) throws IOException, SliderException {
    verifyPathExists(dirPath);
    Path tempFile = new Path(dirPath, "tmp-file-for-checks");
    try {
        FSDataOutputStream out;
        out = fileSystem.create(tempFile, true);
        IOUtils.closeStream(out);
        fileSystem.delete(tempFile, false);
    } catch (IOException e) {
        log.warn("Failed to create file {}: {}", tempFile, e);
        throw new BadClusterStateException(e, "Unable to write to directory %s : %s", dirPath, e.toString());
    }
}

---

From source file:org.apache.slider.common.tools.SliderUtils.java

License:Apache License

public static InputStream getApplicationResourceInputStream(FileSystem fs, Path appPath, String entry)
        throws IOException {
    InputStream is = null;/*from   www  .  j  a  v a  2s  . co  m*/
    FSDataInputStream appStream = null;
    try {
        appStream = fs.open(appPath);
        ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
        ZipArchiveEntry zipEntry;
        boolean done = false;
        while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
            if (entry.equals(zipEntry.getName())) {
                int size = (int) zipEntry.getSize();
                if (size != -1) {
                    log.info("Reading {} of size {}", zipEntry.getName(), zipEntry.getSize());
                    byte[] content = new byte[size];
                    int offset = 0;
                    while (offset < size) {
                        offset += zis.read(content, offset, size - offset);
                    }
                    is = new ByteArrayInputStream(content);
                } else {
                    log.debug("Size unknown. Reading {}", zipEntry.getName());
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    while (true) {
                        int byteRead = zis.read();
                        if (byteRead == -1) {
                            break;
                        }
                        baos.write(byteRead);
                    }
                    is = new ByteArrayInputStream(baos.toByteArray());
                }
                done = true;
            }
        }
    } finally {
        IOUtils.closeStream(appStream);
    }

    return is;
}

---

From source file:org.apache.slider.common.tools.SliderUtils.java

License:Apache License

/**
 * Look for the windows executable and check it has the right headers.
 * <code>File.canRead()</code> doesn't work on windows, so the reading
 * is mandatory./*from  ww  w .j a  va  2  s.co  m*/
 *
 * @param program program name for errors
 * @param exe executable
 * @throws IOException IOE
 */
public static void verifyWindowsExe(String program, File exe) throws IOException {
    verifyIsFile(program, exe);

    verifyFileSize(program, exe, 0x100);

    // now read two bytes and verify the header.

    FileReader reader = null;
    try {
        int[] header = new int[2];
        reader = new FileReader(exe);
        header[0] = reader.read();
        header[1] = reader.read();
        if ((header[0] != 'M' || header[1] != 'Z')) {
            throw new FileNotFoundException(program + " at " + exe + " is not a windows executable file");
        }
    } finally {
        IOUtils.closeStream(reader);
    }
}

---

From source file:org.apache.slider.common.tools.SliderUtils.java

License:Apache License

/**
 * Write bytes to a file//from  w  w  w. j a  va  2s .c o  m
 * @param outfile output file
 * @param data data to write
 * @param createParent flag to indicate that the parent dir should
 * be created
 * @throws IOException on any IO problem
 */
public static void write(File outfile, byte[] data, boolean createParent) throws IOException {
    File parentDir = outfile.getParentFile();
    if (createParent) {
        parentDir.mkdirs();
    }
    SliderUtils.verifyIsDir(parentDir, log);
    FileOutputStream out = new FileOutputStream(outfile);
    try {
        out.write(data);
    } finally {
        IOUtils.closeStream(out);
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »