Example usage for org.apache.hadoop.fs HarFileSystem HarFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs HarFileSystem HarFileSystem.

Prototype

public HarFileSystem() 

Source Link

Document

public construction of harfilesystem

Usage

From source file:com.github.dongjinleekr.hadoop.examples.DistributedCacheExample.java

License:Apache License

public static void printCachePath(Configuration conf) throws IOException, URISyntaxException {
    FileSystem fs = FileSystem.get(conf);
    URI[] archives = DistributedCache.getCacheArchives(conf);

    for (URI archive : archives) {
        HarFileSystem hfs = new HarFileSystem();
        String cacheUri = String.format("har://hdfs-%s:%d%s", fs.getUri().getHost(), fs.getUri().getPort(),
                archive.toString());/*from ww w.  j av  a2 s. co  m*/
        System.out.println(cacheUri);

        hfs.initialize(new URI(cacheUri), conf);

        FileStatus root = hfs.listStatus(new Path("."))[0];
        FileStatus[] children = hfs.listStatus(root.getPath());

        for (FileStatus child : children) {
            System.out.println(child.getPath());
        }

        IOUtils.closeStream(hfs);
    }
}

---

From source file:org.cripac.isee.vpe.util.hdfs.HadoopHelper.java

License:Open Source License

/**
 * Retrieve a track from the HDFS./*from w  ww .jav a 2 s .  com*/
 *
 * @param storeDir The directory storing the tracklets.
 * @param id       The identifier of the track.
 * @return The track retrieved.
 */
public static Tracklet retrieveTracklet(@Nonnull String storeDir, @Nonnull Tracklet.Identifier id,
        @Nullable Logger logger) {
    if (logger == null) {
        logger = new ConsoleLogger(Level.INFO);
    }
    try {
        // Open the Hadoop Archive of the task the track is generated in.
        HarFileSystem harFileSystem = new HarFileSystem();
        harFileSystem.initialize(new URI(storeDir), new Configuration());

        // Read verbal informations of the track.
        Gson gson = new Gson();
        Tracklet tracklet = gson.fromJson(
                new InputStreamReader(
                        harFileSystem.open(new Path(storeDir + "/" + id.serialNumber + "/info.txt"))),
                Tracklet.class);

        // Read frames concurrently..
        ContiguousSet.create(Range.closedOpen(0, tracklet.locationSequence.length), DiscreteDomain.integers())
                .parallelStream().forEach(idx -> {
                    Tracklet.BoundingBox bbox = tracklet.locationSequence[idx];
                    FSDataInputStream imgInputStream = null;
                    try {
                        imgInputStream = harFileSystem
                                .open(new Path(storeDir + "/" + id.toString() + "/" + idx + ".jpg"));
                        byte[] rawBytes = IOUtils.toByteArray(imgInputStream);
                        imgInputStream.close();
                        opencv_core.Mat img = imdecode(new opencv_core.Mat(rawBytes), CV_8UC3);
                        bbox.patchData = new byte[img.rows() * img.cols() * img.channels()];
                        img.data().get(bbox.patchData);
                        img.release();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                });
        harFileSystem.close();
        return tracklet;
    } catch (Exception e) {
        try {
            logger.error("Error when retrieving tracklets" + " from \"" + storeDir + "/" + id.serialNumber
                    + "/info.txt\".", e);
        } catch (Exception e1) {
            e1.printStackTrace();
        }
        return new FakePedestrianTracker().track(null)[0];
    }
}

---