Example usage for org.apache.hadoop.fs FSDataInputStream FSDataInputStream

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FSDataInputStream FSDataInputStream.

Prototype

public FSDataInputStream(InputStream in)

Source Link

Usage

From source file:alluxio.hadoop.AbstractFileSystem.java

License:Apache License

/**
 * Attempts to open the specified file for reading.
 *
 * @param path the file name to open/*  ww w  . j a v  a2s .c  o  m*/
 * @param bufferSize the size in bytes of the buffer to be used
 * @return an {@link FSDataInputStream} at the indicated path of a file
 * @throws IOException if the file cannot be opened (e.g., the path is a folder)
 */
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
    LOG.info("open({}, {})", path, bufferSize);
    if (mStatistics != null) {
        mStatistics.incrementReadOps(1);
    }

    AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
    return new FSDataInputStream(new HdfsFileInputStream(uri, getConf(), bufferSize, mStatistics));
}

From source file:boostingPL.MR.AdaBoostPLMapper.java

License:Open Source License

/** create instances header */
protected void setup(Context context) throws IOException, InterruptedException {
    String pathSrc = context.getConfiguration().get("BoostingPL.metadata");
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataInputStream dis = new FSDataInputStream(hdfs.open(new Path(pathSrc)));
    LineReader in = new LineReader(dis);
    insts = InstancesHelper.createInstancesFromMetadata(in);
    in.close();//from  w ww .  jav  a 2 s .  co  m
    dis.close();
}

From source file:boostingPL.MR.AdaBoostPLTestMapper.java

License:Open Source License

protected void setup(Context context) throws IOException, InterruptedException {
    // classifier file
    Path path = new Path(context.getConfiguration().get("BoostingPL.modelPath") + "/part-r-00000");
    String boostingName = context.getConfiguration().get("BoostingPL.boostingName");
    boostingPL = BoostingPLFactory.createBoostingPL(boostingName, context.getConfiguration(), path);

    // testing dataset metadata
    String pathSrc = context.getConfiguration().get("BoostingPL.metadata");
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataInputStream dis = new FSDataInputStream(hdfs.open(new Path(pathSrc)));
    LineReader in = new LineReader(dis);
    insts = InstancesHelper.createInstancesFromMetadata(in);
    in.close();//from   www .  j a  v  a  2s.c om
    dis.close();

    try {
        eval = new Evaluation(insts);
    } catch (Exception e) {
        LOG.error("[BoostingPL-Test]: Evaluation init error!");
        e.printStackTrace();
    }
    instanceCounter = context.getCounter("BoostingPL", "Number of instances");
}

From source file:boostingPL.MR.AdaBoostPLTestReducer.java

License:Open Source License

protected void setup(Context context) throws IOException, InterruptedException {
    // classifier file
    Path path = new Path(context.getConfiguration().get("BoostingPL.modelPath") + "/part-r-00000");
    String boostingName = context.getConfiguration().get("BoostingPL.boostingName");
    boostingPL = BoostingPLFactory.createBoostingPL(boostingName, context.getConfiguration(), path);

    // testing dataset metadata
    String pathSrc = context.getConfiguration().get("BoostingPL.metadata");
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataInputStream dis = new FSDataInputStream(hdfs.open(new Path(pathSrc)));
    LineReader in = new LineReader(dis);
    insts = InstancesHelper.createInstancesFromMetadata(in);
    in.close();//w  w  w.  ja  v a 2  s .c  o m
    dis.close();

    try {
        eval = new Evaluation(insts);
    } catch (Exception e) {
        LOG.error("[BoostingPL-Test]: Evaluation init error!");
        e.printStackTrace();
    }
}

From source file:cascading.tap.hadoop.HttpFileSystem.java

License:Open Source License

@Override
public FSDataInputStream open(Path path, int i) throws IOException {
    URL url = makeUrl(path);//from   w ww  . j a v  a2 s .co m

    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
    connection.setRequestMethod("GET");
    connection.connect();

    debugConnection(connection);

    return new FSDataInputStream(
            new FSDigestInputStream(connection.getInputStream(), getMD5SumFor(getConf(), path)));
}

From source file:cascading.tap.hadoop.S3HttpFileSystem.java

License:Open Source License

@Override
public FSDataInputStream open(Path path, int i) throws IOException {
    if (LOG.isDebugEnabled())
        LOG.debug("opening file: " + path);

    S3Object object = S3Util.getObject(s3Service, s3Bucket, path, S3Util.Request.OBJECT);
    FSDigestInputStream inputStream = new FSDigestInputStream(S3Util.getObjectInputStream(object),
            getMD5SumFor(getConf(), path));

    // ctor requires Seekable or PositionedReadable stream
    return new FSDataInputStream(inputStream);
}

From source file:co.nubetech.hiho.mapred.input.FileStreamRecordReader.java

License:Apache License

@Override
public FSDataInputStream createValue() {
    logger.debug("Creating value");
    FSDataInputStream stream = null;//from   w  w w.j  a va 2  s.  c o m
    Path file = split.getPath();
    logger.debug("Path is " + file);
    fileName = file.getName();
    try {
        FileSystem fs = file.getFileSystem(configuration);
        stream = new FSDataInputStream(fs.open(file));
    } catch (IOException e) {
        e.printStackTrace();
    }
    logger.debug("Opened stream");
    return stream;
}

From source file:co.nubetech.hiho.mapreduce.TestMySQLLoadMapper.java

License:Apache License

/**
 * @param string//from w  w  w.  j  a va2 s . c  om
 * @throws IOException
 * @throws SQLException
 * @throws InterruptedException
 */
private void runMapper(String tablename) throws IOException, SQLException, InterruptedException {
    Context context = mock(Context.class);
    MySQLLoadDataMapper mapper = new MySQLLoadDataMapper();
    FSDataInputStream val;
    val = new FSDataInputStream(new MyInputStream());
    Connection con = mock(Connection.class);
    com.mysql.jdbc.Statement stmt = mock(com.mysql.jdbc.Statement.class);
    mapper.setConnection(con);
    String query = "load data local infile 'abc.txt' into table tablename " + QUERY_SUFFIX
            + " (col1,col2,col3)";
    when(con.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)).thenReturn(stmt);
    Configuration conf = new Configuration();
    conf.set(HIHOConf.LOAD_QUERY_SUFFIX, QUERY_SUFFIX);
    conf.setBoolean(HIHOConf.LOAD_KEY_IS_TABLENAME, true);
    conf.setBoolean(HIHOConf.LOAD_HAS_HEADER, true);
    when(context.getConfiguration()).thenReturn(conf);
    when(stmt.executeUpdate(query)).thenReturn(10);
    Counter counter = mock(Counter.class);
    when(context.getCounter("MySQLLoadCounters", "ROWS_INSERTED_TABLE_tablename")).thenReturn(counter);
    when(context.getCounter("MySQLLoadCounters", "ROWS_INSERTED_TOTAL")).thenReturn(counter);
    mapper.map(new Text(tablename), val, context);
    verify(stmt).setLocalInfileInputStream(val);
    verify(stmt).executeUpdate(query);
    verify(counter, times(2)).increment(10);
}

From source file:com.aliyun.fs.oss.blk.OssFileSystem.java

License:Apache License

@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
    INode inode = checkFile(path);//from w  w  w.  j  a  v  a 2  s  .co m
    return new FSDataInputStream(new OssInputStream(getConf(), store, inode, statistics));
}

From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java

License:Apache License

@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
    FileStatus fs = getFileStatus(f); // will throw if the file doesn't exist
    if (fs.isDir()) {
        throw new FileNotFoundException("'" + f + "' is a directory");
    }// w w  w . j av  a  2 s .c  o m
    LOG.info("Opening '" + f + "' for reading");
    Path absolutePath = makeAbsolute(f);
    String key = pathToKey(absolutePath);
    return new FSDataInputStream(new BufferedFSInputStream(new NativeOssFsInputStream(key), bufferSize));
}