Example usage for org.apache.hadoop.fs HarFileSystem initialize

Introduction

In this page you can find the example usage for org.apache.hadoop.fs HarFileSystem initialize.

Prototype

@Override
public void initialize(URI name, Configuration conf) throws IOException 

Source Link

Document

Initialize a Har filesystem per har archive.

Usage

From source file:com.dianping.cat.hadoop.hdfs.HarConnectionPool.java

License:Open Source License

public HarFileSystem getHarfsConnection(String id, Date date, FileSystem fs) throws IOException {
    String serverUri = m_serverConfigManager.getHarfsServerUri(id);
    String baseUri = m_serverConfigManager.getHarfsBaseDir(id);
    String harUri = m_format.format(new Object[] { serverUri, baseUri, date });
    Pair<HarFileSystem, Long> har = m_hars.get(harUri);
    long current = System.currentTimeMillis();

    if (har == null) {
        synchronized (this) {
            if (har == null) {

                URI uri = URI.create(harUri);
                HarFileSystem harfs = new HarFileSystem(fs);

                try {
                    harfs.initialize(uri, harfs.getConf());
                    har = new Pair<HarFileSystem, Long>(harfs, current);

                    m_hars.put(harUri, har);
                } catch (IOException e) {
                    // ignore
                }//from  w w w .  j  a  va2s.  com
            }
        }
    }

    if (har != null) {
        har.setValue(current);
        return har.getKey();
    } else {
        return null;
    }
}

---

From source file:com.github.dongjinleekr.hadoop.examples.DistributedCacheExample.java

License:Apache License

public static void printCachePath(Configuration conf) throws IOException, URISyntaxException {
    FileSystem fs = FileSystem.get(conf);
    URI[] archives = DistributedCache.getCacheArchives(conf);

    for (URI archive : archives) {
        HarFileSystem hfs = new HarFileSystem();
        String cacheUri = String.format("har://hdfs-%s:%d%s", fs.getUri().getHost(), fs.getUri().getPort(),
                archive.toString());//from   w  ww. jav a2  s.c  o  m
        System.out.println(cacheUri);

        hfs.initialize(new URI(cacheUri), conf);

        FileStatus root = hfs.listStatus(new Path("."))[0];
        FileStatus[] children = hfs.listStatus(root.getPath());

        for (FileStatus child : children) {
            System.out.println(child.getPath());
        }

        IOUtils.closeStream(hfs);
    }
}

---

From source file:org.cripac.isee.vpe.util.hdfs.HadoopHelper.java

License:Open Source License

/**
 * Retrieve a track from the HDFS.//from w  w w . ja  v a 2s. co m
 *
 * @param storeDir The directory storing the tracklets.
 * @param id       The identifier of the track.
 * @return The track retrieved.
 */
public static Tracklet retrieveTracklet(@Nonnull String storeDir, @Nonnull Tracklet.Identifier id,
        @Nullable Logger logger) {
    if (logger == null) {
        logger = new ConsoleLogger(Level.INFO);
    }
    try {
        // Open the Hadoop Archive of the task the track is generated in.
        HarFileSystem harFileSystem = new HarFileSystem();
        harFileSystem.initialize(new URI(storeDir), new Configuration());

        // Read verbal informations of the track.
        Gson gson = new Gson();
        Tracklet tracklet = gson.fromJson(
                new InputStreamReader(
                        harFileSystem.open(new Path(storeDir + "/" + id.serialNumber + "/info.txt"))),
                Tracklet.class);

        // Read frames concurrently..
        ContiguousSet.create(Range.closedOpen(0, tracklet.locationSequence.length), DiscreteDomain.integers())
                .parallelStream().forEach(idx -> {
                    Tracklet.BoundingBox bbox = tracklet.locationSequence[idx];
                    FSDataInputStream imgInputStream = null;
                    try {
                        imgInputStream = harFileSystem
                                .open(new Path(storeDir + "/" + id.toString() + "/" + idx + ".jpg"));
                        byte[] rawBytes = IOUtils.toByteArray(imgInputStream);
                        imgInputStream.close();
                        opencv_core.Mat img = imdecode(new opencv_core.Mat(rawBytes), CV_8UC3);
                        bbox.patchData = new byte[img.rows() * img.cols() * img.channels()];
                        img.data().get(bbox.patchData);
                        img.release();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                });
        harFileSystem.close();
        return tracklet;
    } catch (Exception e) {
        try {
            logger.error("Error when retrieving tracklets" + " from \"" + storeDir + "/" + id.serialNumber
                    + "/info.txt\".", e);
        } catch (Exception e1) {
            e1.printStackTrace();
        }
        return new FakePedestrianTracker().track(null)[0];
    }
}

---