List of usage examples for java.nio MappedByteBuffer rewind
@Override public final MappedByteBuffer rewind()
From source file:edu.harvard.iq.dataverse.ingest.IngestableDataChecker.java
/** * test this byte buffer against SAS Transport(XPT) spec * *//* www.j a v a 2 s .c om*/ public String testXPTformat(MappedByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; if (DEBUG) { out.println("applying the sas-transport test\n"); } // size test if (buff.capacity() < 91) { if (DEBUG) { out.println("this file is NOT sas-exort type\n"); } return result; } byte[] hdr1 = new byte[80]; byte[] hdr2 = new byte[11]; buff.get(hdr1, 0, 80); buff.get(hdr2, 0, 11); String hdr1st80 = new String(hdr1); String hdrnxt11 = new String(hdr2); if (DEBUG) { out.println("1st-80 bytes=" + hdr1st80); out.println("next-11 bytes=" + hdrnxt11); } if ((hdr1st80.equals(IngestableDataChecker.SAS_XPT_HEADER_80)) && (hdrnxt11.equals(IngestableDataChecker.SAS_XPT_HEADER_11))) { if (DEBUG) { out.println("this file is sas-export type\n"); } result = "application/x-sas-xport"; } else { if (DEBUG) { out.println("this file is NOT sas-exort type\n"); } } return result; }
From source file:edu.harvard.iq.dataverse.ingest.IngestableDataChecker.java
/** * test this byte buffer against STATA DTA spec * *//* w w w . jav a 2 s . c o m*/ public String testDTAformat(MappedByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; if (DEBUG) { dbgLog.info("applying the dta test\n"); } // ----------------------------------------- // Avoid java.nio.BufferUnderflowException // ----------------------------------------- if (buff.capacity() < 4) { return result; } // We first check if it's a "classic", old DTA format // (up to version 115): byte[] hdr4 = new byte[4]; buff.get(hdr4, 0, 4); if (DEBUG) { for (int i = 0; i < hdr4.length; ++i) { dbgLog.info(String.format("%d\t%02X\n", i, hdr4[i])); } } if (hdr4[2] != 1) { if (DEBUG) { dbgLog.info("3rd byte is not 1: given file is not stata-dta type"); } //return result; } else if ((hdr4[1] != 1) && (hdr4[1] != 2)) { if (DEBUG) { dbgLog.info("2nd byte is neither 0 nor 1: this file is not stata-dta type"); } //return result; } else if (!IngestableDataChecker.stataReleaseNumber.containsKey(hdr4[0])) { if (DEBUG) { dbgLog.info("1st byte (" + hdr4[0] + ") is not within the ingestable range [rel. 3-10]: this file is NOT stata-dta type"); } //return result; } else { if (DEBUG) { dbgLog.info("this file is stata-dta type: " + IngestableDataChecker.stataReleaseNumber.get(hdr4[0]) + "(No in HEX=" + hdr4[0] + ")"); } result = "application/x-stata"; } if ((result == null) && (buff.capacity() >= STATA_13_HEADER.length())) { // Let's see if it's a "new" STATA (v.13+) format: buff.rewind(); byte[] headerBuffer = null; String headerString = null; try { headerBuffer = new byte[STATA_13_HEADER.length()]; buff.get(headerBuffer, 0, STATA_13_HEADER.length()); headerString = new String(headerBuffer, "US-ASCII"); } catch (Exception ex) { // probably a buffer underflow exception; // we don't have to do anything... null will // be returned, below. } if (STATA_13_HEADER.equals(headerString)) { result = "application/x-stata-13"; } } return result; }
From source file:edu.harvard.iq.dataverse.ingest.IngestableDataChecker.java
/** * test this byte buffer against R data file * *//*from ww w.j a v a 2s . co m*/ public String testRDAformat(MappedByteBuffer buff) { String result = null; buff.rewind(); if (buff.capacity() < 4) { return null; } boolean DEBUG = false; if (DEBUG) { out.println("applying the RData test\n"); out.println("buffer capacity=" + buff.capacity()); } if (DEBUG) { byte[] rawhdr = new byte[4]; buff.get(rawhdr, 0, 4); for (int j = 0; j < 4; j++) { out.printf("%02X ", rawhdr[j]); } out.println(); buff.rewind(); } // get the first 4 bytes as an int and check its value; // if it is 0x1F8B0800, then gunzip and its first 4 bytes int magicNumber = buff.getInt(); if (DEBUG) { out.println("magicNumber in decimal =" + magicNumber); out.println("in binary=" + Integer.toBinaryString(magicNumber)); out.println("in oct=" + Integer.toOctalString(magicNumber)); out.println("in hex=" + Integer.toHexString(magicNumber)); } try { if (magicNumber == 0x1F8B0800) { if (DEBUG) { out.println("magicNumber is GZIP"); } // gunzip the first 5 bytes and check their bye-pattern // get gzip buffer size int gzip_buffer_size = this.getGzipBufferSize(buff); byte[] hdr = new byte[gzip_buffer_size]; buff.get(hdr, 0, gzip_buffer_size); GZIPInputStream gzin = new GZIPInputStream(new ByteArrayInputStream(hdr)); StringBuilder sb = new StringBuilder(); for (int i = 0; i < RDA_HEADER_SIZE; i++) { sb.append(String.format("%02X", gzin.read())); } String fisrt5bytes = sb.toString(); result = this.checkUncompressedFirst5bytes(fisrt5bytes); // end of compressed case } else { // uncompressed case? if (DEBUG) { out.println("magicNumber is not GZIP:" + magicNumber); out.println("test as an uncompressed RData file"); } buff.rewind(); byte[] uchdr = new byte[5]; buff.get(uchdr, 0, 5); StringBuilder sb = new StringBuilder(); for (int i = 0; i < uchdr.length; i++) { sb.append(String.format("%02X", uchdr[i])); } String fisrt5bytes = sb.toString(); result = this.checkUncompressedFirst5bytes(fisrt5bytes); // end of uncompressed case } } catch (IOException ex) { ex.printStackTrace(); } return result; }
From source file:edu.harvard.iq.dvn.ingest.dsb.SubsettableFileChecker.java
public String detectSubsettableFormat(File fh) { boolean DEBUG = false; String readableFormatType = null; try {/*w ww . j a v a 2 s .co m*/ int buffer_size = this.getBufferSize(fh); // set-up a FileChannel instance for a given file object FileChannel srcChannel = new FileInputStream(fh).getChannel(); // create a read-only MappedByteBuffer MappedByteBuffer buff = srcChannel.map(FileChannel.MapMode.READ_ONLY, 0, buffer_size); //this.printHexDump(buff, "hex dump of the byte-buffer"); //for (String fmt : defaultFormatSet){ buff.rewind(); dbgLog.fine("before the for loop"); for (String fmt : this.getTestFormatSet()) { // get a test method Method mthd = testMethods.get(fmt); try { // invoke this method Object retobj = mthd.invoke(this, buff); String result = (String) retobj; if (result != null) { dbgLog.fine("result for (" + fmt + ")=" + result); if (DEBUG) { out.println("result for (" + fmt + ")=" + result); } if (readableFileTypes.contains(result)) { readableFormatType = result; } dbgLog.fine("readableFormatType=" + readableFormatType); return readableFormatType; } else { dbgLog.fine("null was returned for " + fmt + " test"); if (DEBUG) { out.println("null was returned for " + fmt + " test"); } } } catch (InvocationTargetException e) { Throwable cause = e.getCause(); err.format(cause.getMessage()); e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } } return readableFormatType; } catch (FileNotFoundException fe) { dbgLog.fine("exception detected: file was not foud"); fe.printStackTrace(); } catch (IOException ie) { dbgLog.fine("other io exception detected"); ie.printStackTrace(); } return readableFormatType; }
From source file:edu.harvard.iq.dataverse.ingest.IngestableDataChecker.java
/** * test this byte buffer against SPSS Portable (POR) spec * *///from ww w .j a v a 2s . co m public String testPORformat(MappedByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; if (DEBUG) { out.println("applying the spss-por test\n"); } // size test int bufferCapacity = buff.capacity(); dbgLog.fine("Subsettable Checker: buffer capacity: " + bufferCapacity); if (bufferCapacity < 491) { if (DEBUG) { out.println("this file is NOT spss-por type\n"); } return result; } //windows [0D0A]=> [1310] = [CR/LF] //unix [0A] => [10] //mac [0D] => [13] // 3char [0D0D0A]=> [131310] spss for windows rel 15 // expected results // unix case: [0A] : [80], [161], [242], [323], [404], [485] // windows case: [0D0A] : [81], [163], [245], [327], [409], [491] // : [0D0D0A] : [82], [165], [248], [331], [414], [495] buff.rewind(); byte[] nlch = new byte[36]; int pos1; int pos2; int pos3; int ucase = 0; int wcase = 0; int mcase = 0; int three = 0; int nolines = 6; int nocols = 80; for (int i = 0; i < nolines; ++i) { int baseBias = nocols * (i + 1); // 1-char case pos1 = baseBias + i; if (pos1 > bufferCapacity - 1) { dbgLog.fine("Subsettable Checker: request to go beyond buffer capacity (" + pos1 + ")"); return result; } buff.position(pos1); if (DEBUG) { out.println("\tposition(1)=" + buff.position()); } int j = 6 * i; nlch[j] = buff.get(); if (nlch[j] == 10) { ucase++; } else if (nlch[j] == 13) { mcase++; } // 2-char case pos2 = baseBias + 2 * i; if (pos2 > bufferCapacity - 2) { dbgLog.fine("Subsettable Checker: request to read 2 bytes beyond buffer capacity (" + pos2 + ")"); return result; } buff.position(pos2); if (DEBUG) { out.println("\tposition(2)=" + buff.position()); } nlch[j + 1] = buff.get(); nlch[j + 2] = buff.get(); // 3-char case pos3 = baseBias + 3 * i; if (pos3 > bufferCapacity - 3) { dbgLog.fine("Subsettable Checker: request to read 3 bytes beyond buffer capacity (" + pos3 + ")"); return result; } buff.position(pos3); if (DEBUG) { out.println("\tposition(3)=" + buff.position()); } nlch[j + 3] = buff.get(); nlch[j + 4] = buff.get(); nlch[j + 5] = buff.get(); if (DEBUG) { out.println(i + "-th iteration position =" + nlch[j] + "\t" + nlch[j + 1] + "\t" + nlch[j + 2]); out.println(i + "-th iteration position =" + nlch[j + 3] + "\t" + nlch[j + 4] + "\t" + nlch[j + 5]); } if ((nlch[j + 3] == 13) && (nlch[j + 4] == 13) && (nlch[j + 5] == 10)) { three++; } else if ((nlch[j + 1] == 13) && (nlch[j + 2] == 10)) { wcase++; } buff.rewind(); } if (three == nolines) { if (DEBUG) { out.println("0D0D0A case"); } windowsNewLine = false; } else if ((ucase == nolines) && (wcase < nolines)) { if (DEBUG) { out.println("0A case"); } windowsNewLine = false; } else if ((ucase < nolines) && (wcase == nolines)) { if (DEBUG) { out.println("0D0A case"); } } else if ((mcase == nolines) && (wcase < nolines)) { if (DEBUG) { out.println("0D case"); } windowsNewLine = false; } buff.rewind(); int PORmarkPosition = POR_MARK_POSITION_DEFAULT; if (windowsNewLine) { PORmarkPosition = PORmarkPosition + 5; } else if (three == nolines) { PORmarkPosition = PORmarkPosition + 10; } byte[] pormark = new byte[8]; buff.position(PORmarkPosition); buff.get(pormark, 0, 8); String pormarks = new String(pormark); if (DEBUG) { out.println("pormark =>" + pormarks + "<-"); } if (pormarks.equals(POR_MARK)) { if (DEBUG) { out.println("this file is spss-por type"); } result = "application/x-spss-por"; } else { if (DEBUG) { out.println("this file is NOT spss-por type"); } } return result; }
From source file:edu.harvard.iq.dataverse.ingest.IngestableDataChecker.java
public String detectTabularDataFormat(File fh) { boolean DEBUG = false; String readableFormatType = null; try {//from w ww . j a v a 2 s. c o m int buffer_size = this.getBufferSize(fh); dbgLog.fine("buffer_size: " + buffer_size); // set-up a FileChannel instance for a given file object FileChannel srcChannel = new FileInputStream(fh).getChannel(); // create a read-only MappedByteBuffer MappedByteBuffer buff = srcChannel.map(FileChannel.MapMode.READ_ONLY, 0, buffer_size); //this.printHexDump(buff, "hex dump of the byte-buffer"); //for (String fmt : defaultFormatSet){ buff.rewind(); dbgLog.fine("before the for loop"); for (String fmt : this.getTestFormatSet()) { // get a test method Method mthd = testMethods.get(fmt); //dbgLog.info("mthd: " + mthd.getName()); try { // invoke this method Object retobj = mthd.invoke(this, buff); String result = (String) retobj; if (result != null) { dbgLog.fine("result for (" + fmt + ")=" + result); if (DEBUG) { out.println("result for (" + fmt + ")=" + result); } if (readableFileTypes.contains(result)) { readableFormatType = result; } dbgLog.fine("readableFormatType=" + readableFormatType); return readableFormatType; } else { dbgLog.fine("null was returned for " + fmt + " test"); if (DEBUG) { out.println("null was returned for " + fmt + " test"); } } } catch (InvocationTargetException e) { Throwable cause = e.getCause(); // added null check because of "homemade.zip" from https://redmine.hmdc.harvard.edu/issues/3273 if (cause.getMessage() != null) { err.format(cause.getMessage()); e.printStackTrace(); } else { dbgLog.info("cause.getMessage() was null for " + e); e.printStackTrace(); } } catch (IllegalAccessException e) { e.printStackTrace(); } catch (BufferUnderflowException e) { dbgLog.info("BufferUnderflowException " + e); e.printStackTrace(); } } return readableFormatType; } catch (FileNotFoundException fe) { dbgLog.fine("exception detected: file was not foud"); fe.printStackTrace(); } catch (IOException ie) { dbgLog.fine("other io exception detected"); ie.printStackTrace(); } return readableFormatType; }
From source file:edu.harvard.iq.dataverse.ingest.IngestableDataChecker.java
private int getGzipBufferSize(MappedByteBuffer buff) { int GZIP_BUFFER_SIZE = 120; /*/* w ww .j av a 2 s. c o m*/ note: gzip buffer size <= 118 causes "java.io.EOFException: Unexpected end of ZLIB input stream" with a byte buffer of 500 bytes */ // adjust gzip buffer size if necessary // file.size might be less than the default gzip buffer size if (buff.capacity() < GZIP_BUFFER_SIZE) { GZIP_BUFFER_SIZE = buff.capacity(); } buff.rewind(); return GZIP_BUFFER_SIZE; }
From source file:edu.harvard.iq.dataverse.ingest.IngestableDataChecker.java
/** * dump the data buffer in HEX//from w ww. j av a 2 s .co m * */ public void printHexDump(MappedByteBuffer buff, String hdr) { int counter = 0; if (hdr != null) { out.println(hdr); } for (int i = 0; i < buff.capacity(); i++) { counter = i + 1; out.print(String.format("%02X ", buff.get())); if (counter % 16 == 0) { out.println(); } else { if (counter % 8 == 0) { out.print(" "); } } } out.println(); buff.rewind(); }
From source file:pyromaniac.IO.MMFastaImporter.java
/** * _init qual.// w w w . j a v a 2s .co m * * @throws Exception the exception */ private void _initQual() throws Exception { FileInputStream tempStream = new FileInputStream(new File(this.qualFile)); FileChannel fcQual = tempStream.getChannel(); this.qualSizeLong = fcQual.size(); //qual starts LL contains pairs, marking file #no (in qualBuffers) and position #no (in the buffer). this.qualStartsLL = new ArrayList<Pair<Integer, Long>>(); for (long startPosition = 0L; startPosition < this.qualSizeLong; startPosition += HALF_GIGA) { MappedByteBuffer qualBuffer = fcQual.map(FileChannel.MapMode.READ_ONLY, startPosition, Math.min(this.qualSizeLong - startPosition, HALF_GIGA)); //map half a gig to this channel. this.qualBuffers.add(qualBuffer); int qbf_pos = qualBuffers.size() - 1; int maxBuffer = 2048; int bufferSize = (qualBuffer.capacity() > maxBuffer) ? maxBuffer : qualBuffer.capacity(); qualBuffer.limit(bufferSize); qualBuffer.position(0); while (qualBuffer.position() != qualBuffer.capacity()) { int prevPos = qualBuffer.position(); CharBuffer result = decoder.decode(qualBuffer); qualBuffer.position(prevPos); for (int i = 0; i < result.capacity(); i++) { char curr = result.charAt(i); int posInFile = prevPos + i; if (curr == BEGINNING_FASTA_HEADER) { qualStartsLL.add(new Pair<Integer, Long>(qbf_pos, new Long(posInFile))); } } int newPos = qualBuffer.limit(); if (qualBuffer.limit() + bufferSize > qualBuffer.capacity()) qualBuffer.limit(qualBuffer.capacity()); else qualBuffer.limit(qualBuffer.limit() + bufferSize); qualBuffer.position(newPos); } qualBuffer.rewind(); } }
From source file:pyromaniac.IO.MMFastaImporter.java
/** * _init seq./* w w w . jav a 2 s .c o m*/ * * @throws Exception the exception */ private void _initSeq() throws Exception { FileInputStream tempStream = new FileInputStream(new File(this.seqFile)); FileChannel fcSeq = tempStream.getChannel(); this.seqSizeLong = fcSeq.size(); this.seqStartsLL = new ArrayList<Pair<Integer, Long>>(); for (long startPosition = 0L; startPosition < this.seqSizeLong; startPosition += HALF_GIGA) { MappedByteBuffer seqBuffer = fcSeq.map(FileChannel.MapMode.READ_ONLY, startPosition, Math.min(this.seqSizeLong - startPosition, HALF_GIGA)); this.seqBuffers.add(seqBuffer); int sbf_pos = seqBuffers.size() - 1; int maxBuffer = 2048; int bufferSize = (seqBuffer.capacity() > maxBuffer) ? maxBuffer : seqBuffer.capacity(); seqBuffer.limit(bufferSize); seqBuffer.position(0); while (seqBuffer.position() != seqBuffer.capacity()) { int prevPos = seqBuffer.position(); CharBuffer result = decoder.decode(seqBuffer); seqBuffer.position(prevPos); for (int i = 0; i < result.capacity(); i++) { char curr = result.charAt(i); int posInFile = prevPos + i; if (curr == BEGINNING_FASTA_HEADER) { seqStartsLL.add(new Pair<Integer, Long>(sbf_pos, new Long(posInFile))); } } int newPos = seqBuffer.limit(); if (seqBuffer.limit() + bufferSize > seqBuffer.capacity()) seqBuffer.limit(seqBuffer.capacity()); else seqBuffer.limit(seqBuffer.limit() + bufferSize); seqBuffer.position(newPos); } seqBuffer.rewind(); } }