Example usage for org.apache.hadoop.fs FSDataOutputStream close

List of usage examples for org.apache.hadoop.fs FSDataOutputStream close

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FSDataOutputStream
  6. close

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FSDataOutputStream close.

Prototype

@Override
public void close() throws IOException

Source Link

Document

Close the underlying output stream.

Usage

From source file:GetRetweetersAndCountPerUser.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 3) {
        System.err.println("Usage: GetRetweetersAndCountPerUser <in> <out> <num_reducers>");
        System.exit(2);//w  ww  .  java2s  .  c  om
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(RetweetersPerUser.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    System.out.println(otherArgs[0]);
    job.setMapperClass(TweetMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(IntWritable.class);
    job.setNumReduceTasks(Integer.parseInt(args[2]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    if (job.waitForCompletion(true)) {
        FileSystem hdfs = FileSystem.get(new URI(args[1]), conf);
        Path dir = new Path(args[1]);
        PathFilter filter = new PathFilter() {
            public boolean accept(Path file) {
                return file.getName().startsWith("part-r-");
            }
        };

        HashMap<Integer, Integer> counts_for_user = new HashMap<Integer, Integer>();
        FileStatus[] files = hdfs.listStatus(dir, filter);
        Arrays.sort(files);
        for (int i = 0; i != files.length; i++) {
            Path pt = files[i].getPath();
            BufferedReader br = new BufferedReader(new InputStreamReader(hdfs.open(pt)));
            String line = null;
            while ((line = br.readLine()) != null) {
                String[] columns = new String[2];
                columns = line.split("\t");
                int key = Integer.parseInt(columns[0]);
                if (counts_for_user.containsKey(key))
                    counts_for_user.put(key, counts_for_user.get(key) + 1);
                else
                    counts_for_user.put(key, 1);
            }
            br.close();
        }

        FSDataOutputStream fsDataOutputStream = hdfs.create(new Path(otherArgs[1] + "_count"));
        PrintWriter writer = new PrintWriter(fsDataOutputStream);
        for (Entry<Integer, Integer> e : counts_for_user.entrySet()) {
            writer.write(e.getKey() + "\t" + e.getValue() + "\n");
        }
        writer.close();
        fsDataOutputStream.close();
        hdfs.close();
        System.exit(0);
    }
    System.exit(1);
}

From source file:FDFGenData.java

License:Open Source License

static void testgenrawfile(FileSystem fs, String filename, int recordnum) throws IOException {
    Random r = new Random();
    FSDataOutputStream fos = fs.create(new Path(filename));
    StringBuffer sb = new StringBuffer();
    for (int i = 0; i < recordnum; i++) {
        fos.writeByte(i / 1000);/*from   w  w  w .  ja va  2s . c  o m*/
        fos.writeShort(i / 1000);
        fos.writeInt(i / 1000);
        fos.writeLong(i / 1000);
        fos.writeFloat(i / 1000);
        fos.writeDouble(i / 1000);
        int strnum = r.nextInt(12) + 7;
        sb.setLength(0);
        for (int j = 0; j < strnum; j++) {
            sb.append((char) ('a' + j));
        }
        fos.writeUTF(sb.toString());
        if (i % 1000000 == 0) {
        }
    }
    fos.close();
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndexInfo() {
    try {/*from  ww  w. j a  va 2 s  .com*/
        String fileName = prefix + "testPersistentLineIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginLine = 11;
        info.endLine = 22;
        info.offset = 33;
        info.len = 44;
        info.idx = 55;

        info.persistentLineIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginLine = in.readInt();
        int endLine = in.readInt();
        long offset = in.readLong();
        long len = in.readLong();
        int idx = in.readInt();
        in.close();

        if (beginLine != 11) {
            fail("beginLine fail:" + beginLine);
        }
        if (endLine != 22) {
            fail("endLine fail:" + endLine);
        }
        if (offset != 33) {
            fail("offset fail:" + offset);
        }
        if (len != 44) {
            fail("len fail:" + len);
        }
        if (idx != 55) {
            fail("idx fail:" + idx);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndexInfo() {
    try {/*  ww  w.ja v a 2  s. c o m*/
        String fileName = prefix + "testPersistentKeyIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginKey = 111;
        info.endKey = 222;

        info.persistentKeyIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginKey = in.readInt();
        int endKey = in.readInt();
        in.close();

        if (beginKey != 111) {
            fail("beginKey fail:" + beginKey);
        }
        if (endKey != 222) {
            fail("beginKey fail:" + beginKey);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndex() {
    try {//w ww. j a  v  a 2 s  .  c  o m
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginLine = i;
            info.endLine = i + 10;
            info.offset = i;
            info.len = i + 20;
            info.idx = i;

            index.addIndexInfo(info, ConstVar.LineMode);
        }

        String fileName = prefix + "testPersistentLineIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != i) {
                fail("beginLine fail:" + beginLine);
            }
            if (endLine != i + 10) {
                fail("endLine fail:" + endLine);
            }
            if (offset != i) {
                fail("offset fail:" + offset);
            }
            if (len != i + 20) {
                fail("len fail:" + len);
            }
            if (idx != i) {
                fail("idx fail:" + idx);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndex() {
    try {//  w ww. j av a2s .c om
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginKey = i;
            info.endKey = i + 10;

            index.addIndexInfo(info, ConstVar.KeyMode);
        }

        String fileName = prefix + "testPersistentKeyIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginKey = in.readInt();
            int endKey = in.readInt();

            if (beginKey != i) {
                fail("beginKey fail:" + beginKey);
            }
            if (endKey != i + 10) {
                fail("endKey fail:" + endKey);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentField() throws IOException {
    {//from  ww w  .ja v a 2s. c  o m
        String file = prefix + "testPersistentField";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Field field = new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1);
        field.persistent(out);
        if (out.getPos() != 7) {
            fail("error out.pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);
        byte type = in.readByte();
        int len = in.readInt();
        short idx = in.readShort();

        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

    }
    {
    }
    {
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentFieldMap() {
    try {//from w w w .ja v a  2 s  .c o m
        FileSystem fs = FileSystem.get(new Configuration());

        String file1 = prefix + "testPersistentFieldMap1";
        Path path = new Path(file1);
        FSDataOutputStream out1 = fs.create(path);

        FieldMap fm1 = new FieldMap();
        fm1.persistent(out1);
        if (out1.getPos() != 2) {
            fail("persisitent null fieldmap fail, pos:" + out1.getPos());
        }
        out1.close();

        FSDataInputStream in1 = fs.open(path);
        short fieldNum = in1.readShort();
        if (fieldNum != 0) {
            fail("persistent null fieldmap fail, fieldNum:" + fieldNum);
        }

        String file2 = prefix + "testPersistentFieldMap2";
        path = new Path(file2);
        FSDataOutputStream out2 = fs.create(path);

        FieldMap fm2 = new FieldMap();
        fm2.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1));
        fm2.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 3));
        fm2.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 5));
        fm2.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 7));
        fm2.persistent(out2);
        if (out2.getPos() != (2 + 4 * 7)) {
            fail("persistent 4 field fail, pos:" + out2.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);
        fieldNum = in2.readShort();
        if (fieldNum != 4) {
            fail("persistent 4 field fail, fieldNum:" + fieldNum);
        }

        byte type = in2.readByte();
        int len = in2.readInt();
        short idx = in2.readShort();
        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Short) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Short) {
            fail("fail len:" + len);
        }
        if (idx != 3) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Int) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Int) {
            fail("fail len:" + len);
        }
        if (idx != 5) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Long) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Long) {
            fail("fail len:" + len);
        }
        if (idx != 7) {
            fail("fail idx:" + idx);
        }
    } catch (IOException e) {
        fail("testPersistentField fail1:" + e.getMessage());
    } catch (Exception e) {
        fail("testPersistentField fail2:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentHead() {

    try {// w  ww.j  a v a  2 s . c  o  m
        String fileName = prefix + "testPersistentHead";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Head head = new Head();
        head.persistent(out);
        if (out.getPos() != 17) {
            fail("persistent error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int magic = in.readInt();
        short ver = in.readShort();
        byte var = in.readByte();
        byte compress = in.readByte();
        byte compressStyle = in.readByte();
        short primaryIndex = in.readShort();
        byte encode = in.readByte();
        byte encodeStyle = in.readByte();
        short keyLen = in.readShort();
        short fieldNum = in.readShort();

        if (magic != head.magic()) {
            fail("error magic:" + magic);
        }
        if (ver != head.ver()) {
            fail("error ver:" + ver);
        }
        if (var != 0) {
            fail("error var:" + var);
        }
        if (compress != head.compress()) {
            fail("error compress:" + compress);
        }
        if (compressStyle != head.compressStyle()) {
            fail("error compressStyle:" + compressStyle);
        }
        if (primaryIndex != head.primaryIndex()) {
            fail("error primaryIndex:" + primaryIndex);
        }
        if (encode != head.encode()) {
            fail("error encode:" + encode);
        }
        if (encodeStyle != head.encodeStyle()) {
            fail("error encodeStyle:" + encodeStyle);
        }
        if (keyLen != 0) {
            fail("error keyLen:" + keyLen);
        }
        if (fieldNum != 0) {
            fail("error fieldNum:" + fieldNum);
        }

        fileName = prefix + "testPersistentHead2";
        path = new Path(fileName);
        FSDataOutputStream out2 = fs.create(path);

        Head head2 = new Head();
        String key = "hello konten";

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        head2.setFieldMap(fieldMap);
        head2.setKey(key);

        head2.persistent(out2);
        if (out2.getPos() != 13 + 2 + key.length() + fieldMap.len()) {
            fail("persistent error pos:" + out.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);

        magic = in2.readInt();
        ver = in2.readShort();
        var = in2.readByte();
        compress = in2.readByte();
        compressStyle = in2.readByte();
        primaryIndex = in2.readShort();
        encode = in2.readByte();
        encodeStyle = in2.readByte();

        keyLen = in2.readShort();
        if (keyLen == 0) {
            fail("error keyLen:" + keyLen);
        }
        byte[] buf = new byte[keyLen];
        in2.readFully(buf);
        String keykey = new String(buf);
        if (!key.equals(keykey)) {
            fail("error key:" + keykey);
        }

        FieldMap fieldMap22 = new FieldMap();
        fieldMap22.unpersistent(in2);
        if (fieldMap22.fieldNum() != 2) {
            fail("error fieldNum:" + fieldMap22.fieldNum());
        }
        Field field = fieldMap22.getField((short) 0);
        if (field.type() != ConstVar.FieldType_Byte) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Byte) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 0) {
            fail("fail index:" + field.index());
        }

        field = fieldMap22.getField((short) 1);
        if (field.type() != ConstVar.FieldType_Short) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Short) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 1) {
            fail("fail index:" + field.index());
        }

    } catch (Exception e) {
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecord() {
    try {//from   w w  w.j  a v a 2 s .  co  m
        String fileName = prefix + "testChunkToRecord";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        byte[] fb = new byte[ConstVar.Sizeof_Float];
        float f = (float) 5.5;
        Util.float2bytes(fb, f);
        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, fb,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());
        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        value = new byte[4];
        in.readFully(value);
        float fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;
        if (bitSet.length() != (fieldNum / 8 + 1) * 8) {
            fail("bitSet.len:" + bitSet.length());
        }

        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("bitSet should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }

        {
        }
        lv = Util.bytes2long(value, 0, len);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        fail("should not exception:" + e.getMessage());
    }
}