Example usage for org.apache.hadoop.fs Path toUri

List of usage examples for org.apache.hadoop.fs Path toUri

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri()

Source Link

Document

Convert this Path to a URI.

Usage

From source file:azkaban.jobtype.connectors.teradata.TeradataToHdfsJobRunnerMain.java

License:Apache License

private void runCopyTdToHdfs() throws IOException {
    if (Boolean.valueOf(_jobProps.getProperty("force.output.overwrite", "false").trim())) {
        Path path = new Path(_jobProps.getProperty(TdchConstants.TARGET_HDFS_PATH_KEY));
        _logger.info("Deleting output directory " + path.toUri());
        JobConf conf = new JobConf();
        path.getFileSystem(conf).delete(path, true);
    }//w  ww .  ja v a2  s. co  m
    _logger.info(String.format("Executing %s with params: %s",
            TeradataToHdfsJobRunnerMain.class.getSimpleName(), _params));
    TeradataImportTool.main(_params.toTdchParams());
}

From source file:azkaban.jobtype.connectors.TeradataToHdfsJobRunnerMain.java

License:Apache License

private void runCopyTdToHdfs() throws IOException {
    if (Boolean.valueOf(_jobProps.getProperty("force.output.overwrite", "false").trim())) {
        Path path = new Path(_jobProps.getProperty(TdchConstants.TARGET_HDFS_PATH_KEY));
        _logger.info("Deleting output directory " + path.toUri());
        JobConf conf = new JobConf();
        path.getFileSystem(conf).delete(path, true);
    }//from   w  w  w. j a va  2 s . c o  m

    _logger.info(String.format("Executing %s with params: %s",
            TeradataToHdfsJobRunnerMain.class.getSimpleName(), _params));
    TeradataImportTool.main(_params.toTdchParams());
}

From source file:azkaban.storage.HdfsStorage.java

License:Apache License

private String getRelativePath(final Path targetPath) {
    return URI.create(this.rootUri.getPath()).relativize(targetPath.toUri()).getPath();
}

From source file:azkaban.viewer.hdfs.AvroFileViewer.java

License:Apache License

@Override
public Set<Capability> getCapabilities(FileSystem fs, Path path) throws AccessControlException {
    if (logger.isDebugEnabled()) {
        logger.debug("path:" + path.toUri().getPath());
    }// www  .j  a v  a 2s.c o m

    DataFileStream<Object> avroDataStream = null;
    try {
        avroDataStream = getAvroDataStream(fs, path);
        Schema schema = avroDataStream.getSchema();
        return (schema != null) ? EnumSet.of(Capability.READ, Capability.SCHEMA)
                : EnumSet.noneOf(Capability.class);
    } catch (AccessControlException e) {
        throw e;
    } catch (IOException e) {
        if (logger.isDebugEnabled()) {
            logger.debug(path.toUri().getPath() + " is not an avro file.");
            logger.debug("Error in getting avro schema: ", e);
        }
        return EnumSet.noneOf(Capability.class);
    } finally {
        try {
            if (avroDataStream != null) {
                avroDataStream.close();
            }
        } catch (IOException e) {
            logger.error(e);
        }
    }
}

From source file:azkaban.viewer.hdfs.AvroFileViewer.java

License:Apache License

@Override
public String getSchema(FileSystem fs, Path path) {
    if (logger.isDebugEnabled()) {
        logger.debug("path:" + path.toUri().getPath());
    }/*  w w w . j  ava  2 s  .c  om*/

    DataFileStream<Object> avroDataStream = null;
    try {
        avroDataStream = getAvroDataStream(fs, path);
        Schema schema = avroDataStream.getSchema();
        return schema.toString(true);
    } catch (IOException e) {
        if (logger.isDebugEnabled()) {
            logger.debug(path.toUri().getPath() + " is not an avro file.");
            logger.debug("Error in getting avro schema: ", e);
        }
        return null;
    } finally {
        try {
            if (avroDataStream != null) {
                avroDataStream.close();
            }
        } catch (IOException e) {
            logger.error(e);
        }
    }
}

From source file:azkaban.viewer.hdfs.AvroFileViewer.java

License:Apache License

private DataFileStream<Object> getAvroDataStream(FileSystem fs, Path path) throws IOException {
    if (logger.isDebugEnabled()) {
        logger.debug("path:" + path.toUri().getPath());
    }//from   w ww.j  av a2s .com

    GenericDatumReader<Object> avroReader = new GenericDatumReader<Object>();
    InputStream hdfsInputStream = null;
    try {
        hdfsInputStream = fs.open(path);
    } catch (IOException e) {
        if (hdfsInputStream != null) {
            hdfsInputStream.close();
        }
        throw e;
    }

    DataFileStream<Object> avroDataFileStream = null;
    try {
        avroDataFileStream = new DataFileStream<Object>(hdfsInputStream, avroReader);
    } catch (IOException e) {
        if (hdfsInputStream != null) {
            hdfsInputStream.close();
        }
        throw e;
    }

    return avroDataFileStream;
}

From source file:azkaban.viewer.hdfs.AvroFileViewer.java

License:Apache License

@Override
public void displayFile(FileSystem fs, Path path, OutputStream outputStream, int startLine, int endLine)
        throws IOException {

    if (logger.isDebugEnabled()) {
        logger.debug("display avro file:" + path.toUri().getPath());
    }//w w w  . j a  v  a 2  s. c om

    DataFileStream<Object> avroDatastream = null;
    JsonGenerator g = null;

    try {
        avroDatastream = getAvroDataStream(fs, path);
        Schema schema = avroDatastream.getSchema();
        DatumWriter<Object> avroWriter = new GenericDatumWriter<Object>(schema);

        g = new JsonFactory().createJsonGenerator(outputStream, JsonEncoding.UTF8);
        g.useDefaultPrettyPrinter();
        Encoder encoder = EncoderFactory.get().jsonEncoder(schema, g);

        long endTime = System.currentTimeMillis() + STOP_TIME;
        int lineno = 1; // line number starts from 1
        while (avroDatastream.hasNext() && lineno <= endLine && System.currentTimeMillis() <= endTime) {
            Object datum = avroDatastream.next();
            if (lineno >= startLine) {
                String record = "\n\n Record " + lineno + ":\n";
                outputStream.write(record.getBytes("UTF-8"));
                avroWriter.write(datum, encoder);
                encoder.flush();
            }
            lineno++;
        }
    } catch (IOException e) {
        outputStream.write(("Error in display avro file: " + e.getLocalizedMessage()).getBytes("UTF-8"));
        throw e;
    } finally {
        if (g != null) {
            g.close();
        }
        avroDatastream.close();
    }
}

From source file:azkaban.viewer.hdfs.HdfsAvroFileViewer.java

License:Apache License

@Override
public boolean canReadFile(FileSystem fs, Path path) {

    if (logger.isDebugEnabled())
        logger.debug("path:" + path.toUri().getPath());

    DataFileStream<Object> avroDataStream = null;
    try {/*  w ww  .j a v a  2  s  .  co m*/
        avroDataStream = getAvroDataStream(fs, path);
        Schema schema = avroDataStream.getSchema();
        return schema != null;
    } catch (IOException e) {
        if (logger.isDebugEnabled()) {
            logger.debug(path.toUri().getPath() + " is not an avro file.");
            logger.debug("Error in getting avro schema: " + e.getLocalizedMessage());
        }
        return false;
    } finally {
        try {
            if (avroDataStream != null) {
                avroDataStream.close();
            }
        } catch (IOException e) {
            logger.error(e);
        }
    }
}

From source file:azkaban.viewer.hdfs.HdfsAvroFileViewer.java

License:Apache License

private DataFileStream<Object> getAvroDataStream(FileSystem fs, Path path) throws IOException {
    if (logger.isDebugEnabled())
        logger.debug("path:" + path.toUri().getPath());

    GenericDatumReader<Object> avroReader = new GenericDatumReader<Object>();
    InputStream hdfsInputStream = null;
    try {//  w w  w  . j a  v  a 2  s. c om
        hdfsInputStream = fs.open(path);
    } catch (Exception e) {
        if (hdfsInputStream != null) {
            hdfsInputStream.close();
        }
    }

    DataFileStream<Object> avroDataFileStream = null;
    try {
        avroDataFileStream = new DataFileStream<Object>(hdfsInputStream, avroReader);
    } catch (IOException e) {
        if (hdfsInputStream != null) {
            hdfsInputStream.close();
        }
        throw e;
    }

    return avroDataFileStream;
}

From source file:azkaban.viewer.hdfs.HdfsAvroFileViewer.java

License:Apache License

@Override
public void displayFile(FileSystem fs, Path path, OutputStream outputStream, int startLine, int endLine)
        throws IOException {

    if (logger.isDebugEnabled())
        logger.debug("display avro file:" + path.toUri().getPath());

    DataFileStream<Object> avroDatastream = null;
    JsonGenerator g = null;/* w  ww.ja v  a2  s. c o m*/

    try {
        avroDatastream = getAvroDataStream(fs, path);
        Schema schema = avroDatastream.getSchema();
        DatumWriter<Object> avroWriter = new GenericDatumWriter<Object>(schema);

        g = new JsonFactory().createJsonGenerator(outputStream, JsonEncoding.UTF8);
        g.useDefaultPrettyPrinter();
        Encoder encoder = EncoderFactory.get().jsonEncoder(schema, g);

        long endTime = System.currentTimeMillis() + STOP_TIME;
        int lineno = 1; // line number starts from 1
        while (avroDatastream.hasNext() && lineno <= endLine && System.currentTimeMillis() <= endTime) {
            Object datum = avroDatastream.next();
            if (lineno >= startLine) {
                String record = "\n\n Record " + lineno + ":\n";
                outputStream.write(record.getBytes("UTF-8"));
                avroWriter.write(datum, encoder);
                encoder.flush();
            }
            lineno++;
        }
    } catch (IOException e) {
        outputStream.write(("Error in display avro file: " + e.getLocalizedMessage()).getBytes("UTF-8"));
        throw e;
    } finally {
        if (g != null) {
            g.close();
        }
        avroDatastream.close();
    }
}