Example usage for org.apache.hadoop.fs FileSystem open

List of usage examples for org.apache.hadoop.fs FileSystem open

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem open.

Prototype

public FSDataInputStream open(PathHandle fd) throws IOException

Source Link

Document

Open an FSDataInputStream matching the PathHandle instance.

Usage

From source file:BooleanRetrievalCompressed.java

License:Apache License

private void initialize(String indexPath, String collectionPath, FileSystem fs) throws IOException {
    index = new MapFile.Reader(new Path(indexPath + "/part-r-00000"), fs.getConf());
    collection = fs.open(new Path(collectionPath));
    stack = new Stack<Set<Integer>>();
}

From source file:HdfsReader.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length < 1) {
        System.err.println("HdfsReader [FileSize i.e. 1g/10g/100g/200g]");
        return 1;
    }//www.ja  v  a2 s .  c  o m

    double fileSize;
    double fileSizeInMB;
    if (args[0].equals("1g")) {
        fileSize = 1073741824.0;
        fileSizeInMB = 1024.0;
    } else if (args[0].equals("10g")) {
        fileSize = 10737418240.0;
        fileSizeInMB = 10240.0;
    } else if (args[0].equals("100g")) {
        fileSize = 107374182400.0;
        fileSizeInMB = 102400.0;
    } else if (args[0].equals("200g")) {
        fileSize = 214748364800.0;
        fileSizeInMB = 204800.0;
    } else {
        throw new IllegalArgumentException("Invalid arg: " + args[0]);
    }

    String fileName = "read-" + args[0] + "-avg.txt";
    File avgFile = new File(fileName);
    PrintWriter avgPW = new PrintWriter(avgFile);
    fileName = "read-" + args[0] + "-min.txt";
    File minFile = new File(fileName);
    PrintWriter minPW = new PrintWriter(minFile);
    fileName = "read-" + args[0] + "-max.txt";
    File maxFile = new File(fileName);
    PrintWriter maxPW = new PrintWriter(maxFile);

    int numIters = 10;
    int bufferSize = 4096;
    long blockSize[] = new long[] { 67108864, 134217728, 268435456, 536870912, 1073741824 };
    short replication[] = new short[] { 1, 4 };
    String hdfsFile = "/hdfs_test/" + args[0] + "/1.in";
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    Path hdfsFilePath = new Path(hdfsFile);

    for (int i = 0; i < 5; i++) { // blockSize
        for (int j = 0; j < 2; j++) { // replication
            OutputStream os = fs.create(hdfsFilePath, true, bufferSize, replication[j], blockSize[i]);
            byte[] buf = new byte[bufferSize];
            for (int m = 0; m < bufferSize; m += 4) {
                buf[m] = (byte) m;
            }
            double numBufPerFile = fileSize / (double) bufferSize;

            for (double m = 0.0; m < numBufPerFile; m++) {
                os.write(buf);
            }
            os.close();
            long avg = 0, min = Long.MAX_VALUE, max = Long.MIN_VALUE;
            for (int k = 0; k < numIters; k++) {
                InputStream is = fs.open(hdfsFilePath);

                long startTime = System.currentTimeMillis();
                int bytesRead = is.read(buf);
                while (bytesRead != -1) {
                    bytesRead = is.read(buf);
                }
                is.close();
                long endTime = System.currentTimeMillis();
                long duration = (endTime - startTime);
                avg += duration;
                if (duration < min) {
                    min = duration;
                }
                if (duration > max) {
                    max = duration;
                }
            }
            // write result to output
            double avgBW = fileSizeInMB * 1000.0 * (double) numIters / (double) avg;
            avgPW.print(avgBW);
            avgPW.print("\t");
            double minBW = fileSizeInMB * 1000.0 / (double) max;
            minPW.print(minBW);
            minPW.print("\t");
            double maxBW = fileSizeInMB * 1000.0 / (double) min;
            maxPW.print(maxBW);
            maxPW.print("\t");
        }
        avgPW.println();
        minPW.println();
        maxPW.println();
    }
    avgPW.close();
    minPW.close();
    maxPW.close();
    return 0;
}

From source file:GetRetweetersAndCountPerUser.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 3) {
        System.err.println("Usage: GetRetweetersAndCountPerUser <in> <out> <num_reducers>");
        System.exit(2);//from  w  w w.j av a2s  .c o  m
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(RetweetersPerUser.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    System.out.println(otherArgs[0]);
    job.setMapperClass(TweetMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(IntWritable.class);
    job.setNumReduceTasks(Integer.parseInt(args[2]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    if (job.waitForCompletion(true)) {
        FileSystem hdfs = FileSystem.get(new URI(args[1]), conf);
        Path dir = new Path(args[1]);
        PathFilter filter = new PathFilter() {
            public boolean accept(Path file) {
                return file.getName().startsWith("part-r-");
            }
        };

        HashMap<Integer, Integer> counts_for_user = new HashMap<Integer, Integer>();
        FileStatus[] files = hdfs.listStatus(dir, filter);
        Arrays.sort(files);
        for (int i = 0; i != files.length; i++) {
            Path pt = files[i].getPath();
            BufferedReader br = new BufferedReader(new InputStreamReader(hdfs.open(pt)));
            String line = null;
            while ((line = br.readLine()) != null) {
                String[] columns = new String[2];
                columns = line.split("\t");
                int key = Integer.parseInt(columns[0]);
                if (counts_for_user.containsKey(key))
                    counts_for_user.put(key, counts_for_user.get(key) + 1);
                else
                    counts_for_user.put(key, 1);
            }
            br.close();
        }

        FSDataOutputStream fsDataOutputStream = hdfs.create(new Path(otherArgs[1] + "_count"));
        PrintWriter writer = new PrintWriter(fsDataOutputStream);
        for (Entry<Integer, Integer> e : counts_for_user.entrySet()) {
            writer.write(e.getKey() + "\t" + e.getValue() + "\n");
        }
        writer.close();
        fsDataOutputStream.close();
        hdfs.close();
        System.exit(0);
    }
    System.exit(1);
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndexInfo() {
    try {//w  w  w  .j a  va 2s . co  m
        String fileName = prefix + "testPersistentLineIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginLine = 11;
        info.endLine = 22;
        info.offset = 33;
        info.len = 44;
        info.idx = 55;

        info.persistentLineIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginLine = in.readInt();
        int endLine = in.readInt();
        long offset = in.readLong();
        long len = in.readLong();
        int idx = in.readInt();
        in.close();

        if (beginLine != 11) {
            fail("beginLine fail:" + beginLine);
        }
        if (endLine != 22) {
            fail("endLine fail:" + endLine);
        }
        if (offset != 33) {
            fail("offset fail:" + offset);
        }
        if (len != 44) {
            fail("len fail:" + len);
        }
        if (idx != 55) {
            fail("idx fail:" + idx);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndexInfo() {
    try {//from   www .  ja  va  2s.  c o m
        String fileName = prefix + "testPersistentKeyIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginKey = 111;
        info.endKey = 222;

        info.persistentKeyIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginKey = in.readInt();
        int endKey = in.readInt();
        in.close();

        if (beginKey != 111) {
            fail("beginKey fail:" + beginKey);
        }
        if (endKey != 222) {
            fail("beginKey fail:" + beginKey);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentLineIndexInfo() {
    try {//from  w w  w.j  av a  2s  .c  o  m
        String fileName = prefix + "testPersistentLineIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        IndexInfo info = new IndexInfo();
        info.unpersistentLineIndexInfo(in);
        in.close();

        if (info.beginLine != 11) {
            fail("beginLine fail:" + info.beginLine);
        }
        if (info.endLine != 22) {
            fail("endLine fail:" + info.endLine);
        }
        if (info.offset != 33) {
            fail("offset fail:" + info.offset);
        }
        if (info.len != 44) {
            fail("len fail:" + info.len);
        }
        if (info.idx != 55) {
            fail("idx fail:" + info.idx);
        }

        fs.close();
    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentKeyIndexInfo() {
    try {// w  ww  .  j  a va2  s .  c o m
        String fileName = prefix + "testPersistentKeyIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        IndexInfo info = new IndexInfo();
        info.unpersistentKeyIndexInfo(in);
        in.close();

        if (info.beginKey != 111) {
            fail("beginKey fail:" + info.beginKey);
        }
        if (info.endKey != 222) {
            fail("beginKey fail:" + info.beginKey);
        }

        fs.close();
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndex() {
    try {// w  w  w. j a  v  a 2s  . c  o  m
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginLine = i;
            info.endLine = i + 10;
            info.offset = i;
            info.len = i + 20;
            info.idx = i;

            index.addIndexInfo(info, ConstVar.LineMode);
        }

        String fileName = prefix + "testPersistentLineIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != i) {
                fail("beginLine fail:" + beginLine);
            }
            if (endLine != i + 10) {
                fail("endLine fail:" + endLine);
            }
            if (offset != i) {
                fail("offset fail:" + offset);
            }
            if (len != i + 20) {
                fail("len fail:" + len);
            }
            if (idx != i) {
                fail("idx fail:" + idx);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndex() {
    try {/*from   w  w w  .  j  av a  2  s .  c om*/
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginKey = i;
            info.endKey = i + 10;

            index.addIndexInfo(info, ConstVar.KeyMode);
        }

        String fileName = prefix + "testPersistentKeyIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginKey = in.readInt();
            int endKey = in.readInt();

            if (beginKey != i) {
                fail("beginKey fail:" + beginKey);
            }
            if (endKey != i + 10) {
                fail("endKey fail:" + endKey);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentField() throws IOException {
    {/*from ww w  .  ja v  a2  s .c  o m*/
        String file = prefix + "testPersistentField";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Field field = new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1);
        field.persistent(out);
        if (out.getPos() != 7) {
            fail("error out.pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);
        byte type = in.readByte();
        int len = in.readInt();
        short idx = in.readShort();

        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

    }
    {
    }
    {
    }
}