Example usage for org.apache.hadoop.fs Path toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri()

Source Link

Document

Convert this Path to a URI.

Usage

From source file:com.datasalt.pangool.utils.test.AbstractHadoopTestLibrary.java

License:Apache License

public static void readTuples(Path file, Configuration conf, TupleVisitor iterator)
        throws IOException, InterruptedException {
    TupleFile.Reader reader = new TupleFile.Reader(FileSystem.get(file.toUri(), conf), conf, file);
    Tuple tuple = new Tuple(reader.getSchema());
    while (reader.next(tuple)) {
        iterator.onTuple(tuple);/* w w w .java2 s . co  m*/
    }
    reader.close();
}

From source file:com.datatorrent.common.util.FSStorageAgent.java

License:Apache License

public FSStorageAgent(String path, Configuration conf) {
    this.path = path;
    try {/* w  w  w  .  j a  v a2 s .  c o m*/
        logger.debug("Initialize storage agent with {}.", path);
        Path lPath = new Path(path);
        URI pathUri = lPath.toUri();

        if (pathUri.getScheme() != null) {
            fileContext = FileContext.getFileContext(pathUri, conf == null ? new Configuration() : conf);
        } else {
            fileContext = FileContext.getFileContext(conf == null ? new Configuration() : conf);
        }
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

From source file:com.datatorrent.contrib.hdht.HDHTFileAccessFSImpl.java

License:Open Source License

@Override
public void init() {
    if (fs == null) {
        Path dataFilePath = new Path(basePath);
        try {//from  w  ww  . ja va2  s .c o  m
            fs = FileSystem.newInstance(dataFilePath.toUri(), new Configuration());
        } catch (IOException e) {
            DTThrowable.rethrow(e);
        }
    }
}

From source file:com.datatorrent.flume.source.HdfsTestSource.java

License:Open Source License

private List<String> findFiles() throws IOException {
    List<String> files = Lists.newArrayList();
    Path directoryPath = new Path(directory);
    FileSystem lfs = FileSystem.newInstance(directoryPath.toUri(), configuration);
    try {/*  ww w.  java 2 s. c om*/
        logger.debug("checking for new files in {}", directoryPath);
        RemoteIterator<LocatedFileStatus> statuses = lfs.listFiles(directoryPath, true);
        for (; statuses.hasNext();) {
            FileStatus status = statuses.next();
            Path path = status.getPath();
            String filePathStr = path.toString();
            if (!filePathStr.endsWith(".gz")) {
                continue;
            }
            logger.debug("new file {}", filePathStr);
            files.add(path.toString());
        }
    } catch (FileNotFoundException e) {
        logger.warn("Failed to list directory {}", directoryPath, e);
        throw new RuntimeException(e);
    } finally {
        lfs.close();
    }
    return files;
}

From source file:com.datatorrent.lib.bucket.BucketManagerTest.java

License:Open Source License

@AfterClass
public static void teardown() throws IOException {
    manager.shutdownService();/*from   w  ww . j a  v a 2 s  . co m*/
    Path root = new Path(applicationPath);
    FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
    fs.delete(root, true);
}

From source file:com.datatorrent.lib.bucket.CategoricalBucketManagerTest.java

License:Open Source License

@After
public void teardown() throws IOException {
    manager.shutdownService();/*  w w  w. j a va 2s .c  o m*/
    Path root = new Path(applicationPath);
    FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
    fs.delete(root, true);
}

From source file:com.datatorrent.lib.bucket.ExpirableHdfsBucketStore.java

License:Open Source License

@Override
public void deleteExpiredBuckets(long time) throws IOException {
    Iterator<Long> iterator = windowToBuckets.keySet().iterator();
    for (; iterator.hasNext();) {
        long window = iterator.next();
        long timestamp = windowToTimestamp.get(window);
        if (timestamp < time) {
            Collection<Integer> indices = windowToBuckets.get(window);
            synchronized (indices) {
                if (indices.size() > 0) {
                    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
                    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
                    try {
                        if (fs.exists(dataFilePath)) {
                            logger.debug("start delete {}", window);
                            fs.delete(dataFilePath, true);
                            logger.debug("end delete {}", window);
                        }/*  ww  w  .  j  a va  2s .  c om*/
                        for (int bucketIdx : indices) {
                            Map<Long, Long> offsetMap = bucketPositions[bucketIdx];
                            if (offsetMap != null) {
                                synchronized (offsetMap) {
                                    offsetMap.remove(window);
                                }
                            }
                        }
                    } finally {
                        fs.close();
                    }
                }
                windowToTimestamp.remove(window);
                iterator.remove();
            }
        }
    }
}

From source file:com.datatorrent.lib.bucket.HdfsBucketStore.java

License:Open Source License

/**
 * {@inheritDoc}/*w w  w  .j  av  a  2 s.com*/
 */
@Override
public void storeBucketData(long window, long timestamp, Map<Integer, Map<Object, T>> data) throws IOException {
    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
    FSDataOutputStream dataStream = fs.create(dataFilePath);

    Output output = new Output(dataStream);
    try {
        long offset = 0;
        for (int bucketIdx : data.keySet()) {
            Map<Object, T> bucketData = data.get(bucketIdx);

            if (eventKeyClass == null) {
                Map.Entry<Object, T> eventEntry = bucketData.entrySet().iterator().next();
                eventKeyClass = eventEntry.getKey().getClass();
                if (!writeEventKeysOnly) {
                    @SuppressWarnings("unchecked")
                    Class<T> lEventClass = (Class<T>) eventEntry.getValue().getClass();
                    eventClass = lEventClass;
                }
            }
            //Write the size of data and then data
            dataStream.writeInt(bucketData.size());
            for (Map.Entry<Object, T> entry : bucketData.entrySet()) {
                writeSerde.writeObject(output, entry.getKey());

                if (!writeEventKeysOnly) {
                    int posLength = output.position();
                    output.writeInt(0); //temporary place holder
                    writeSerde.writeObject(output, entry.getValue());
                    int posValue = output.position();
                    int valueLength = posValue - posLength - 4;
                    output.setPosition(posLength);
                    output.writeInt(valueLength);
                    output.setPosition(posValue);
                }
            }
            output.flush();
            if (bucketPositions[bucketIdx] == null) {
                bucketPositions[bucketIdx] = Maps.newHashMap();
            }
            windowToBuckets.put(window, bucketIdx);
            windowToTimestamp.put(window, timestamp);
            synchronized (bucketPositions[bucketIdx]) {
                bucketPositions[bucketIdx].put(window, offset);
            }
            offset = dataStream.getPos();
        }
    } finally {
        output.close();
        dataStream.close();
        fs.close();
    }
}

From source file:com.datatorrent.lib.bucket.HdfsBucketStore.java

License:Open Source License

/**
 * {@inheritDoc}//from   ww w . ja  v a2s  .  c  o m
 */
@Override
public void deleteBucket(int bucketIdx) throws IOException {
    Map<Long, Long> offsetMap = bucketPositions[bucketIdx];
    if (offsetMap != null) {
        for (Long window : offsetMap.keySet()) {
            Collection<Integer> indices = windowToBuckets.get(window);
            synchronized (indices) {
                boolean elementRemoved = indices.remove(bucketIdx);
                if (indices.isEmpty() && elementRemoved) {
                    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
                    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
                    try {
                        if (fs.exists(dataFilePath)) {
                            logger.debug("start delete {}", window);
                            fs.delete(dataFilePath, true);
                            logger.debug("end delete {}", window);
                        }
                        windowToBuckets.removeAll(window);
                        windowToTimestamp.remove(window);
                    } finally {
                        fs.close();
                    }
                }
            }
        }
    }
    bucketPositions[bucketIdx] = null;
}

From source file:com.datatorrent.lib.dedup.DeduperBloomFilterTest.java

License:Open Source License

@After
public void teardown() {
    Path root = new Path(applicationPath);
    try {/*from  w  w w  . java  2  s .c  om*/
        FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
        fs.delete(root, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}