Example usage for org.apache.hadoop.fs FSDataInputStream seek

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FSDataInputStream seek.

Prototype

@Override
public void seek(long desired) throws IOException 

Source Link

Document

Seek to the given offset.

Usage

From source file:FormatStorage.Segment.java

License:Open Source License

public void unpersistentUnitIndex(FSDataInputStream in) throws Exception {
    if (unitIndex == null) {
        unitIndex = new UnitIndex();
    }// ww  w  .  ja v  a  2  s .c o m

    long unitIndexMetaOffset = -1;

    unitIndexMetaOffset = offset + len - ConstVar.IndexMetaOffset;

    in.seek(unitIndexMetaOffset);

    unpersistentIndexMeta(in);

    if (keyIndexOffset != -1) {
        in.seek(keyIndexOffset);
        unpersistentKeyUnitIndex(in);
    }

    if (lineIndexOffset < 0) {
        throw new SEException.InnerException("line index offset error when load unitIndex:" + lineIndexOffset);
    }
    in.seek(lineIndexOffset);
    unpersistentLineUnitIndex(in);

}

---

From source file:FormatStorage.Unit.java

License:Open Source License

byte[] loadUnitBuffer(FSDataInputStream in) throws IOException {
    if (unitBytes == null || unitBytes.length < len) {
        unitBytes = new byte[(int) len];
    }/*from  w  ww.  ja va2  s  . c o m*/
    in.seek(offset);
    in.read(unitBytes, 0, (int) len);

    return unitBytes;
}

---

From source file:FormatStorage.Unit.java

License:Open Source License

void loadDataMeta(FSDataInputStream in) throws IOException {
    if (metaOffset == -1) {
        long pos = offset + len - ConstVar.DataChunkMetaOffset;
        in.seek(pos);

        recordNum = in.readInt();// w  ww  .  j av a 2  s .c  o  m
        metaOffset = in.readLong();
        chunksLen = in.readInt();
    }

    if (offsetArray == null) {
        offsetArray = new long[recordNum];
    }

    if (segment.formatData.var() == ConstVar.VarFlag) {
        in.seek(metaOffset);
        for (int i = 0; i < recordNum; i++) {
            offsetArray[i] = in.readLong();
        }
    }
}

---

From source file:FormatStorage1.ISegmentIndex.java

License:Open Source License

@Override
public void unpersistent(DataInput in) throws IOException {
    if (fileInfo.formatfiletype() == ConstVar.OldFormatFile) {
        FSDataInputStream fsdin = (FSDataInputStream) in;
        int recordNum = fsdin.readInt();
        this.segnum = fsdin.readInt();
        long keyIndexOffset = fsdin.readLong();
        long lineIndexOffset = fsdin.readLong();

        if (keyIndexOffset != -1) {
            fsdin.seek(keyIndexOffset);
            for (int i = 0; i < segnum; i++) {
                int beginkey = in.readInt();
                int endkey = in.readInt();
                keyindexs.add(//from  w  w  w  .j av  a  2 s.c o  m
                        new IKeyIndex(new IRecord.IFValue(beginkey, -1), new IRecord.IFValue(endkey, -1), 0));
            }
        }

        if (lineIndexOffset == -1 && recordNum > 0) {
            throw new IOException("line index offset error when load seg index:" + lineIndexOffset);
        }

        if (recordNum > 0) {
            fsdin.seek(lineIndexOffset);

            for (int i = 0; i < segnum; i++) {
                ILineIndex lineIndex = new ILineIndex();
                lineIndex.setbeginline(in.readInt());
                lineIndex.setendline(in.readInt() - 1);
                segoffsets.add(in.readLong());
                seglens.add((int) in.readLong());
                @SuppressWarnings("unused")
                int idx = in.readInt();
                lineindexs.add(lineIndex);
                if (fileInfo.havekeyindex())
                    keyindexs.get(i).setRecnum(lineIndex.endline() - lineIndex.beginline() + 1);
            }
        }

    } else {
        this.segnum = in.readInt();
        for (int i = 0; i < segnum; i++) {
            segoffsets.add(in.readLong());
        }
        for (int i = 0; i < segnum; i++) {
            seglens.add(in.readInt());
        }
        if (fileInfo.havekeyindex()) {
            for (int i = 0; i < segnum; i++) {
                IKeyIndex keyIndex = new IKeyIndex(keyfieldType);
                keyIndex.unpersistent(in);
                keyindexs.add(keyIndex);
            }
        }
        if (fileInfo.havelineindex()) {
            for (int i = 0; i < segnum; i++) {
                ILineIndex lineIndex = new ILineIndex();
                lineIndex.unpersistent(in);
                lineindexs.add(lineIndex);
            }
        }
    }
}

---

From source file:FormatStorage1.IUnitIndex.java

License:Open Source License

@Override
public void unpersistent(DataInput in) throws IOException {
    if (fileInfo.formatfiletype() == ConstVar.OldFormatFile) {
        FSDataInputStream fsdin = (FSDataInputStream) in;
        int recordNum = fsdin.readInt();
        this.unitnum = fsdin.readInt();
        long keyIndexOffset = fsdin.readLong();
        long lineIndexOffset = fsdin.readLong();

        if (keyIndexOffset != -1) {
            fsdin.seek(keyIndexOffset);
            for (int i = 0; i < unitnum; i++) {
                int beginkey = in.readInt();
                int endkey = in.readInt();
                keyindexs.add(/*from   w w w. j  a v  a2  s .  c o m*/
                        new IKeyIndex(new IRecord.IFValue(beginkey, -1), new IRecord.IFValue(endkey, -1), 0));
            }
        }

        if (lineIndexOffset == -1 && recordNum > 0) {
            throw new IOException("line index offset error when load seg index:" + lineIndexOffset);
        }

        if (recordNum > 0) {
            fsdin.seek(lineIndexOffset);

            for (int i = 0; i < unitnum; i++) {
                ILineIndex lineIndex = new ILineIndex();
                lineIndex.setbeginline(in.readInt());
                lineIndex.setendline(in.readInt() - 1);
                unitoffsets.add(in.readLong());
                unitlens.add((int) in.readLong());
                @SuppressWarnings("unused")
                int idx = in.readInt();
                lineindexs.add(lineIndex);
                if (fileInfo.havekeyindex())
                    keyindexs.get(i).setRecnum(lineIndex.endline() - lineIndex.beginline() + 1);
            }
        }

    } else {

        this.unitnum = in.readInt();
        for (int i = 0; i < unitnum; i++) {
            unitoffsets.add(in.readLong());
        }
        if (fileInfo.havekeyindex()) {
            for (int i = 0; i < unitnum; i++) {
                IKeyIndex keyIndex = new IKeyIndex(keyfieldType);
                keyIndex.unpersistent(in);
                keyindexs.add(keyIndex);
            }
        }
        if (fileInfo.havelineindex()) {
            for (int i = 0; i < unitnum; i++) {
                ILineIndex lineIndex = new ILineIndex();
                lineIndex.unpersistent(in);
                lineindexs.add(lineIndex);
            }
        }
    }
}

---

From source file:gov.jgi.meta.hadoop.input.FastaBlockRecordReader.java

License:Open Source License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {

    LOG.info("initializing FastaBlockRecordReader");

    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    start = split.getStart();// w ww  .j  a  v  a2  s.co  m
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
        in = new FastaBlockLineReader(codec.createInputStream(fileIn), job);
        end = Long.MAX_VALUE;
    } else {
        if (start != 0) {
            skipFirstLine = false; // don't do this!
            //--start;                      or this
            fileIn.seek(start);
        }
        in = new FastaBlockLineReader(fileIn, job);
    }
    this.pos = start;
}

---

From source file:gov.jgi.meta.hadoop.input.FastaRecordReader.java

License:Open Source License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    start = split.getStart();//from  w  w  w.j a  va  2  s.c  o  m
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
        in = new FastaLineReader(codec.createInputStream(fileIn), job);
        end = Long.MAX_VALUE;
    } else {
        if (start != 0) {
            skipFirstLine = false; // don't do this!
            //--start;                      or this
            fileIn.seek(start);
        }
        in = new FastaLineReader(fileIn, job);
    }
    if (skipFirstLine) { // skip first line and re-establish "start".
        start += in.readLine(new Text(), new Text(), 0, (int) Math.min((long) Integer.MAX_VALUE, end - start));
    }
    this.pos = start;
}

---

From source file:gov.jgi.meta.hadoop.input.FastqBlockRecordReader.java

License:Open Source License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    start = split.getStart();/*from   www.  j  av a 2 s.  c  om*/
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
        in = new FastqBlockLineReader(codec.createInputStream(fileIn), job);
        end = Long.MAX_VALUE;
    } else {
        if (start != 0) {
            skipFirstLine = false; // don't do this!
            //--start;                      or this
            fileIn.seek(start);
        }
        in = new FastqBlockLineReader(fileIn, job);
    }
    this.pos = start;
}

---

From source file:gov.jgi.meta.hadoop.input.FastqRecordReader.java

License:Open Source License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    start = split.getStart();/*from  www .j  a va 2 s .com*/
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
        in = new FastqLineReader(codec.createInputStream(fileIn), job);
        end = Long.MAX_VALUE;
    } else {
        if (start != 0) {
            skipFirstLine = false; // don't do this!
            //--start;                      or this
            fileIn.seek(start);
        }
        in = new FastqLineReader(fileIn, job);
    }
    if (skipFirstLine) { // skip first line and re-establish "start".
        start += in.readLine(new Text(), new Text(), 0, (int) Math.min((long) Integer.MAX_VALUE, end - start));
    }
    this.pos = start;
}

---

From source file:gr.ntua.h2rdf.inputFormat.MyLineRecordReader.java

License:Open Source License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    MyFileSplit split = (MyFileSplit) (MyInputSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    start = split.getStart();//from  ww w  .  j a va2s.co  m
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
        in = new LineReader(codec.createInputStream(fileIn), job);
        end = Long.MAX_VALUE;
    } else {
        if (start != 0) {
            skipFirstLine = true;
            --start;
            fileIn.seek(start);
        }
        in = new LineReader(fileIn, job);
    }
    if (skipFirstLine) { // skip first line and re-establish "start".
        start += in.readLine(new Text(), 0, (int) Math.min((long) Integer.MAX_VALUE, end - start));
    }
    this.pos = start;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• »