Example usage for org.apache.hadoop.fs Path toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri()

Source Link

Document

Convert this Path to a URI.

Usage

From source file:com.nearinfinity.blur.utils.BlurUtil.java

License:Apache License

public static void removeIndexFiles(String uri) throws IOException {
    Path tablePath = new Path(uri);
    FileSystem fileSystem = FileSystem.get(tablePath.toUri(), new Configuration());
    fileSystem.delete(tablePath, true);//from  w ww.  ja v  a2 s. c  o m
}

From source file:com.netease.news.text.SequenceFilesFromDirectory.java

License:Apache License

private int runSequential(Configuration conf, Path input, Path output, Map<String, String> options)
        throws IOException, InterruptedException, NoSuchMethodException {
    // Running sequentially
    Charset charset = Charset.forName(getOption(CHARSET_OPTION[0]));
    String keyPrefix = getOption(KEY_PREFIX_OPTION[0]);
    FileSystem fs = FileSystem.get(input.toUri(), conf);
    ChunkedWriter writer = new ChunkedWriter(conf, Integer.parseInt(options.get(CHUNK_SIZE_OPTION[0])), output);

    try {// w  w  w .  j a va2  s . com
        SequenceFilesFromDirectoryFilter pathFilter;
        String fileFilterClassName = options.get(FILE_FILTER_CLASS_OPTION[0]);
        if (PrefixAdditionFilter.class.getName().equals(fileFilterClassName)) {
            pathFilter = new PrefixAdditionFilter(conf, keyPrefix, options, writer, charset, fs);
        } else {
            pathFilter = ClassUtils.instantiateAs(fileFilterClassName, SequenceFilesFromDirectoryFilter.class,
                    new Class[] { Configuration.class, String.class, Map.class, ChunkedWriter.class,
                            Charset.class, FileSystem.class },
                    new Object[] { conf, keyPrefix, options, writer, charset, fs });
        }
        fs.listStatus(input, pathFilter);
    } finally {
        Closeables.close(writer, false);
    }
    return 0;
}

From source file:com.netflix.aegisthus.tools.DirectoryWalker.java

License:Apache License

public DirectoryWalker add(Path path) throws IOException {
    this.base = path.toUri().toString();
    if (!base.endsWith("/")) {
        base = base + "/";
    }//from w  w w . j a v  a 2  s. co m
    this.fs = path.getFileSystem(conf);
    process(new PartitionInfo(fs.getFileStatus(path)));
    return this;
}

From source file:com.netflix.aegisthus.tools.StorageHelper.java

License:Apache License

public void copyToTemp(String file, String prefix, boolean snappy) throws IOException {
    String target = getBaseTaskAttemptTempLocation();
    Path targetPath = new Path(target, prefix);
    Path filePath = new Path(file);
    Path fullPath = new Path(targetPath, filePath.getName());

    String log = String.format("copying %s to %s", file, fullPath.toUri().toString());
    LOG.info(log);//from ww  w . jav  a 2  s. c  o m
    ctx.setStatus(log);
    Utils.copy(new Path(file), fullPath, snappy, ctx);
}

From source file:com.netflix.aegisthus.tools.StorageHelper.java

License:Apache License

public void copyToTemp(String file, boolean snappy) throws IOException {
    String target = getBaseTaskAttemptTempLocation();
    Path targetPath = new Path(target);
    Path filePath = new Path(file);
    Path fullPath = new Path(targetPath, filePath.getName());
    String log = String.format("copying %s to %s", file, fullPath.toUri().toString());
    LOG.info(log);/* w ww.  j  av  a 2  s.co m*/
    ctx.setStatus(log);
    Utils.copy(filePath, fullPath, snappy, ctx);
}

From source file:com.netflix.aegisthus.tools.StorageHelper.java

License:Apache License

public void logCommit(String file) throws IOException {
    Path log = commitPath(getTaskId());
    if (debug) {//from   w  w w  .  j  a  va2s  .c o m
        LOG.info(String.format("logging (%s) to commit log (%s)", file, log.toUri().toString()));
    }
    FileSystem fs = log.getFileSystem(config);
    DataOutputStream os = null;
    if (fs.exists(log)) {
        os = fs.append(log);
    } else {
        os = fs.create(log);
    }
    os.writeBytes(file);
    os.write('\n');
    os.close();
}

From source file:com.netflix.aegisthus.tools.StorageHelper.java

License:Apache License

public void moveTaskOutputToFinal() throws IOException {
    String tempLocation = getBaseTaskAttemptTempLocation();
    Path path = new Path(tempLocation);
    List<String> relativePaths = Lists
            .newArrayList(DirectoryWalker.with(config).threaded().omitHidden().add(path).relativePathStrings());
    Path finalPath = getFinalPath();
    for (String relative : relativePaths) {
        LOG.info(String.format("moving (%s) from (%s) to (%s)", relative, path.toUri().toString(),
                finalPath.toUri().toString()));
        Utils.copy(new Path(relative), path, finalPath, ctx);
        ctx.progress();/* w  ww.  jav a2 s . c  o m*/
    }
}

From source file:com.netflix.bdp.s3mper.alert.impl.AlertJanitor.java

License:Apache License

/**
 * Writes out logs to the given path as a separate JSON message per line.
 * //from  w  ww .  j av a 2 s.  co m
 * @param queue
 * @param path
 * @throws IOException 
 */
public void writeLogs(String queue, Path path) throws IOException {
    FileSystem fs = FileSystem.get(path.toUri(), conf);
    DataOutputStream fout = fs.create(path);

    do {
        List<Message> messages = pull(queue, batchCount);

        if (messages.isEmpty()) {
            break;
        }

        for (Message m : messages) {
            fout.write((m.getBody().replaceAll("[\n|\r]", " ") + "\n").getBytes("UTF8"));
        }

        delete(queue, messages);
    } while (true);

    fout.close();
    fs.close();
}

From source file:com.netflix.bdp.s3mper.alert.impl.CloudWatchAlertDispatcher.java

License:Apache License

private void sendSQSConsistencyMessage(List<Path> paths, boolean recovered) {
    S3ConsistencyMessage message = new S3ConsistencyMessage();

    buildMessage(message);//from  ww  w  .j  av  a2 s  .  c  om

    List<String> pathStrings = new ArrayList<String>();
    boolean truncated = false;

    for (Path p : paths) {
        pathStrings.add(p.toUri().toString());

        //Truncate if the message payload gets to be too large (i.e. to many missing files)
        if (pathStrings.size() >= pathReportLimit) {
            truncated = true;
            break;
        }
    }

    message.setPaths(pathStrings);
    message.setTruncated(truncated);

    int missingFiles = paths.size();

    if (recovered) {
        missingFiles = 0;
    }

    message.setMissingFiles(missingFiles);
    message.setRecovered(recovered);

    sendMessage(consistencyQueueUrl, message);

    if (!recovered) {
        sendMessage(notificationQueueUrl, message);
    }
}

From source file:com.netflix.bdp.s3mper.cli.MetastoreListCommand.java

License:Apache License

@Override
public void execute(Configuration conf, String[] args) throws Exception {
    try {//from w  ww.j a va 2 s. c om

        Path path = new Path(args[0]);

        conf.set("s3mper.metastore.deleteMarker.enabled", "true");

        FileSystemMetastore meta = Metastore.getFilesystemMetastore(conf);
        meta.initalize(path.toUri(), conf);

        List<FileInfo> files = meta.list(Collections.singletonList(path));

        for (FileInfo f : files) {
            System.out.println(f.getPath() + (f.isDeleted() ? " [Deleted]" : ""));
        }
    } catch (Exception e) {
        System.out.println("Usage: s3mper metastore list <path>\n");

        e.printStackTrace();
    }
}