List of usage examples for java.nio ByteBuffer getInt
public abstract int getInt();
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.sav.SAVFileReader.java
void decodeRecordType999(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordType999(): start *****"); try {//from w ww . j a v a2s .c o m if (stream == null) { throw new IllegalArgumentException("RT999: stream == null!"); } // first check the 4-byte header value //if (stream.markSupported()){ stream.mark(1000); //} // 999.0 check the first 4 bytes byte[] headerCodeRt999 = new byte[LENGTH_RECORD_TYPE999_CODE]; //dbgLog.fine("RT999: stream position="+stream.pos); int nbytes_rt999 = stream.read(headerCodeRt999, 0, LENGTH_RECORD_TYPE999_CODE); // to-do check against nbytes //printHexDump(headerCodeRt999, "RT999 header test"); ByteBuffer bb_header_code_rt999 = ByteBuffer.wrap(headerCodeRt999, 0, LENGTH_RECORD_TYPE999_CODE); if (isLittleEndian) { bb_header_code_rt999.order(ByteOrder.LITTLE_ENDIAN); } int intRT999test = bb_header_code_rt999.getInt(); dbgLog.fine("header test value: RT999=" + intRT999test); if (intRT999test != 999) { //if (stream.markSupported()){ dbgLog.fine("intRT999test failed=" + intRT999test); stream.reset(); throw new IOException("RT999:Header value(999) was not correctly detected:" + intRT999test); //} } // 999.1 check 4-byte integer Filler block byte[] length_filler = new byte[LENGTH_RT999_FILLER]; int nbytes_rt999_1 = stream.read(length_filler, 0, LENGTH_RT999_FILLER); // to-do check against nbytes //printHexDump(length_how_many_line_bytes, "RT999 how_many_line_bytes"); ByteBuffer bb_filler = ByteBuffer.wrap(length_filler, 0, LENGTH_RT999_FILLER); if (isLittleEndian) { bb_filler.order(ByteOrder.LITTLE_ENDIAN); } int rt999filler = bb_filler.getInt(); dbgLog.fine("rt999filler=" + rt999filler); if (rt999filler == 0) { dbgLog.fine("the end of the dictionary section"); } else { throw new IOException("RT999: failed to detect the end mark(0): value=" + rt999filler); } // missing value processing concerning HIGHEST/LOWEST values Set<Map.Entry<String, InvalidData>> msvlc = invalidDataTable.entrySet(); for (Iterator<Map.Entry<String, InvalidData>> itc = msvlc.iterator(); itc.hasNext();) { Map.Entry<String, InvalidData> et = itc.next(); String variable = et.getKey(); dbgLog.fine("variable=" + variable); InvalidData invalidDataInfo = et.getValue(); if (invalidDataInfo.getInvalidRange() != null && !invalidDataInfo.getInvalidRange().isEmpty()) { if (invalidDataInfo.getInvalidRange().get(0).equals(OBSTypeHexValue.get("LOWEST"))) { dbgLog.fine("1st value is LOWEST"); invalidDataInfo.getInvalidRange().set(0, "LOWEST"); } else if (invalidDataInfo.getInvalidRange().get(1).equals(OBSTypeHexValue.get("HIGHEST"))) { dbgLog.fine("2nd value is HIGHEST"); invalidDataInfo.getInvalidRange().set(1, "HIGHEST"); } } } dbgLog.fine("invalidDataTable:\n" + invalidDataTable); smd.setInvalidDataTable(invalidDataTable); } catch (IOException ex) { //ex.printStackTrace(); //exit(1); throw ex; } dbgLog.fine("***** decodeRecordType999(): end *****"); }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeRecordType3and4(BufferedInputStream stream) throws IOException { dbgLog.fine("decodeRecordType3and4(): start"); Map<String, Map<String, String>> valueLabelTable = new LinkedHashMap<String, Map<String, String>>(); int safteyCounter = 0; while (true) { try {//from w w w. j av a 2s .c o m if (stream == null) { throw new IllegalArgumentException("stream == null!"); } // this secton may not exit so first check the 4-byte header value //if (stream.markSupported()){ stream.mark(1000); //} // 3.0 check the first 4 bytes byte[] headerCode = new byte[LENGTH_RECORD_TYPE3_CODE]; int nbytes_rt3 = stream.read(headerCode, 0, LENGTH_RECORD_TYPE3_CODE); // to-do check against nbytes //printHexDump(headerCode, "RT3 header test"); ByteBuffer bb_header_code = ByteBuffer.wrap(headerCode, 0, LENGTH_RECORD_TYPE3_CODE); if (isLittleEndian) { bb_header_code.order(ByteOrder.LITTLE_ENDIAN); } int intRT3test = bb_header_code.getInt(); dbgLog.fine("header test value: RT3=" + intRT3test); if (intRT3test != 3) { //if (stream.markSupported()){ dbgLog.fine("iteration=" + safteyCounter); // We have encountered a record that's not type 3. This means we've // processed all the type 3/4 record pairs. So we want to rewind // the stream and return -- so that the appropriate record type // reader can be called on it. // But before we return, we need to save all the value labels // we have found: //smd.setValueLabelTable(valueLabelTable); assignValueLabels(valueLabelTable); stream.reset(); return; //} } // 3.1 how many value-label pairs follow byte[] number_of_labels = new byte[LENGTH_RT3_HOW_MANY_LABELS]; int nbytes_3_1 = stream.read(number_of_labels); if (nbytes_3_1 == 0) { throw new IOException("RT 3: reading recordType3.1: no byte was read"); } ByteBuffer bb_number_of_labels = ByteBuffer.wrap(number_of_labels, 0, LENGTH_RT3_HOW_MANY_LABELS); if (isLittleEndian) { bb_number_of_labels.order(ByteOrder.LITTLE_ENDIAN); } int numberOfValueLabels = bb_number_of_labels.getInt(); dbgLog.fine("number of value-label pairs=" + numberOfValueLabels); ByteBuffer[] tempBB = new ByteBuffer[numberOfValueLabels]; String valueLabel[] = new String[numberOfValueLabels]; for (int i = 0; i < numberOfValueLabels; i++) { // read 8-byte as value byte[] value = new byte[LENGTH_RT3_VALUE]; int nbytes_3_value = stream.read(value); if (nbytes_3_value == 0) { throw new IOException("RT 3: reading recordType3 value: no byte was read"); } // note these 8 bytes are interpreted later // currently no information about which variable's (=> type unknown) ByteBuffer bb_value = ByteBuffer.wrap(value, 0, LENGTH_RT3_VALUE); if (isLittleEndian) { bb_value.order(ByteOrder.LITTLE_ENDIAN); } tempBB[i] = bb_value; dbgLog.fine("bb_value=" + Hex.encodeHex(bb_value.array())); /* double valueD = bb_value.getDouble(); dbgLog.fine("value="+valueD); */ // read 1st byte as unsigned integer = label_length // read label_length byte as label byte[] labelLengthByte = new byte[LENGTH_RT3_LABEL_LENGTH]; int nbytes_3_label_length = stream.read(labelLengthByte); // add check-routine here dbgLog.fine("labelLengthByte" + Hex.encodeHex(labelLengthByte)); dbgLog.fine("label length = " + labelLengthByte[0]); // the net-length of a value label is saved as // unsigned byte; however, the length is less than 127 // byte should be ok int rawLabelLength = labelLengthByte[0] & 0xFF; dbgLog.fine("rawLabelLength=" + rawLabelLength); // -1 =>1-byte already read int labelLength = getSAVobsAdjustedBlockLength(rawLabelLength + 1) - 1; byte[] valueLabelBytes = new byte[labelLength]; int nbytes_3_value_label = stream.read(valueLabelBytes); // ByteBuffer bb_label = ByteBuffer.wrap(valueLabel,0,labelLength); valueLabel[i] = StringUtils.stripEnd( new String(Arrays.copyOfRange(valueLabelBytes, 0, rawLabelLength), defaultCharSet), " "); dbgLog.fine(i + "-th valueLabel=" + valueLabel[i] + "<-"); } // iter rt3 dbgLog.fine("end of RT3 block"); dbgLog.fine("start of RT4 block"); // 4.0 check the first 4 bytes byte[] headerCode4 = new byte[LENGTH_RECORD_TYPE4_CODE]; int nbytes_rt4 = stream.read(headerCode4, 0, LENGTH_RECORD_TYPE4_CODE); if (nbytes_rt4 == 0) { throw new IOException("RT4: reading recordType4 value: no byte was read"); } //printHexDump(headerCode4, "RT4 header test"); ByteBuffer bb_header_code_4 = ByteBuffer.wrap(headerCode4, 0, LENGTH_RECORD_TYPE4_CODE); if (isLittleEndian) { bb_header_code_4.order(ByteOrder.LITTLE_ENDIAN); } int intRT4test = bb_header_code_4.getInt(); dbgLog.fine("header test value: RT4=" + intRT4test); if (intRT4test != 4) { throw new IOException("RT 4: reading recordType4 header: no byte was read"); } // 4.1 read the how-many-variables bytes byte[] howManyVariablesfollow = new byte[LENGTH_RT4_HOW_MANY_VARIABLES]; int nbytes_rt4_1 = stream.read(howManyVariablesfollow, 0, LENGTH_RT4_HOW_MANY_VARIABLES); ByteBuffer bb_howManyVariablesfollow = ByteBuffer.wrap(howManyVariablesfollow, 0, LENGTH_RT4_HOW_MANY_VARIABLES); if (isLittleEndian) { bb_howManyVariablesfollow.order(ByteOrder.LITTLE_ENDIAN); } int howManyVariablesRT4 = bb_howManyVariablesfollow.getInt(); dbgLog.fine("how many variables follow: RT4=" + howManyVariablesRT4); int length_indicies = LENGTH_RT4_VARIABLE_INDEX * howManyVariablesRT4; byte[] variableIdicesBytes = new byte[length_indicies]; int nbytes_rt4_2 = stream.read(variableIdicesBytes, 0, length_indicies); // !!!!! Caution: variableIndex in RT4 starts from 1 NOT ** 0 ** int[] variableIndex = new int[howManyVariablesRT4]; int offset = 0; for (int i = 0; i < howManyVariablesRT4; i++) { ByteBuffer bb_variable_index = ByteBuffer.wrap(variableIdicesBytes, offset, LENGTH_RT4_VARIABLE_INDEX); offset += LENGTH_RT4_VARIABLE_INDEX; if (isLittleEndian) { bb_variable_index.order(ByteOrder.LITTLE_ENDIAN); } variableIndex[i] = bb_variable_index.getInt(); dbgLog.fine(i + "-th variable index number=" + variableIndex[i]); } dbgLog.fine("variable index set=" + ArrayUtils.toString(variableIndex)); dbgLog.fine("subtract 1 from variableIndex for getting a variable info"); boolean isNumeric = OBSwiseTypelList.get(variableIndex[0] - 1) == 0 ? true : false; Map<String, String> valueLabelPair = new LinkedHashMap<String, String>(); if (isNumeric) { // numeric variable dbgLog.fine("processing of a numeric value-label table"); for (int j = 0; j < numberOfValueLabels; j++) { valueLabelPair.put(doubleNumberFormatter.format(tempBB[j].getDouble()), valueLabel[j]); } } else { // String variable dbgLog.fine("processing of a string value-label table"); for (int j = 0; j < numberOfValueLabels; j++) { valueLabelPair.put( StringUtils.stripEnd(new String((tempBB[j].array()), defaultCharSet), " "), valueLabel[j]); } } dbgLog.fine("valueLabePair=" + valueLabelPair); dbgLog.fine("key variable's (raw) index =" + variableIndex[0]); valueLabelTable.put(OBSIndexToVariableName.get(variableIndex[0] - 1), valueLabelPair); dbgLog.fine("valueLabelTable=" + valueLabelTable); // create a mapping table that finds the key variable for this mapping table String keyVariableName = OBSIndexToVariableName.get(variableIndex[0] - 1); for (int vn : variableIndex) { valueVariableMappingTable.put(OBSIndexToVariableName.get(vn - 1), keyVariableName); } dbgLog.fine("valueVariableMappingTable:\n" + valueVariableMappingTable); } catch (IOException ex) { //ex.printStackTrace(); throw ex; } safteyCounter++; if (safteyCounter >= 1000000) { break; } } //while ///smd.setValueLabelTable(valueLabelTable); assignValueLabels(valueLabelTable); dbgLog.fine("***** decodeRecordType3and4(): end *****"); }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.sav.SAVFileReader.java
void decodeRecordType1(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordType1(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }/*from w w w. j a v a 2s .c o m*/ // how to read each recordType // 1. set-up the following objects before reading bytes // a. the working byte array // b. the storage object // the length of this field: 172bytes = 60 + 4 + 12 + 4 + 8 + 84 // this field consists of 6 distinct blocks byte[] recordType1 = new byte[LENGTH_RECORDTYPE1]; // int caseWeightVariableOBSIndex = 0; try { int nbytes = stream.read(recordType1, 0, LENGTH_RECORDTYPE1); //printHexDump(recordType1, "recordType1"); if (nbytes == 0) { throw new IOException("reading recordType1: no byte was read"); } // 1.1 60 byte-String that tells the platform/version of SPSS that // wrote this file int offset_start = 0; int offset_end = LENGTH_SPSS_PRODUCT_INFO; // 60 bytes String productInfo = new String(Arrays.copyOfRange(recordType1, offset_start, offset_end), "US-ASCII"); dbgLog.fine("productInfo:\n" + productInfo + "\n"); // add the info to the fileInfo smd.getFileInformation().put("productInfo", productInfo); // try to parse out the SPSS version that created this data // file: String spssVersionNumberTag = null; String regexpVersionNumber = ".*Release ([0-9]*)"; Pattern patternJsession = Pattern.compile(regexpVersionNumber); Matcher matcher = patternJsession.matcher(productInfo); if (matcher.find()) { spssVersionNumberTag = matcher.group(1); dbgLog.fine("SPSS Version Number: " + spssVersionNumberTag); } if (spssVersionNumberTag != null && !spssVersionNumberTag.equals("")) { spssVersionNumber = Integer.valueOf(spssVersionNumberTag).intValue(); /* * Starting with SPSS version 16, the default encoding is * UTF-8. * But we are only going to use it if the user did not explicitly * specify the encoding on the addfiles page. Then we'd want * to stick with whatever they entered. */ if (spssVersionNumber > 15) { if (getDataLanguageEncoding() == null) { defaultCharSet = "UTF-8"; } } } smd.getFileInformation().put("charset", defaultCharSet); // 1.2) 4-byte file-layout-code (byte-order) offset_start = offset_end; offset_end += LENGTH_FILE_LAYOUT_CODE; // 4 byte ByteBuffer bb_fileLayout_code = ByteBuffer.wrap(recordType1, offset_start, LENGTH_FILE_LAYOUT_CODE); ByteBuffer byteOderTest = bb_fileLayout_code.duplicate(); // interprete the 4 byte as int int int2test = byteOderTest.getInt(); if (int2test == 2 || int2test == 3) { dbgLog.fine("integer == " + int2test + ": the byte-oder of the writer is the same " + "as the counterpart of Java: Big Endian"); } else { // Because Java's byte-order is always big endian, // this(!=2) means this sav file was written on a little-endian machine // non-string, multi-bytes blocks must be byte-reversed bb_fileLayout_code.order(ByteOrder.LITTLE_ENDIAN); int2test = bb_fileLayout_code.getInt(); if (int2test == 2 || int2test == 3) { dbgLog.fine("The sav file was saved on a little endian machine"); dbgLog.fine("Reveral of the bytes is necessary to decode " + "multi-byte, non-string blocks"); isLittleEndian = true; } else { throw new IOException("reading recordType1:unknown file layout code=" + int2test); } } dbgLog.fine("Endian of this platform:" + ByteOrder.nativeOrder().toString()); smd.getFileInformation().put("OSByteOrder", ByteOrder.nativeOrder().toString()); smd.getFileInformation().put("byteOrder", int2test); // 1.3 4-byte Number_Of_OBS_Units_Per_Case // (= how many RT2 records => how many varilables) offset_start = offset_end; offset_end += LENGTH_NUMBER_OF_OBS_UNITS_PER_CASE; // 4 byte ByteBuffer bb_OBS_units_per_case = ByteBuffer.wrap(recordType1, offset_start, LENGTH_NUMBER_OF_OBS_UNITS_PER_CASE); if (isLittleEndian) { bb_OBS_units_per_case.order(ByteOrder.LITTLE_ENDIAN); } OBSUnitsPerCase = bb_OBS_units_per_case.getInt(); dbgLog.fine("RT1: OBSUnitsPerCase=" + OBSUnitsPerCase); smd.getFileInformation().put("OBSUnitsPerCase", OBSUnitsPerCase); // 1.4 4-byte Compression_Switch offset_start = offset_end; offset_end += LENGTH_COMPRESSION_SWITCH; // 4 byte ByteBuffer bb_compression_switch = ByteBuffer.wrap(recordType1, offset_start, LENGTH_COMPRESSION_SWITCH); if (isLittleEndian) { bb_compression_switch.order(ByteOrder.LITTLE_ENDIAN); } int compression_switch = bb_compression_switch.getInt(); if (compression_switch == 0) { // data section is not compressed isDataSectionCompressed = false; dbgLog.fine("data section is not compressed"); } else { dbgLog.fine("data section is compressed:" + compression_switch); } smd.getFileInformation().put("compressedData", compression_switch); // 1.5 4-byte Case-Weight Variable Index // warning: this variable index starts from 1, not 0 offset_start = offset_end; offset_end += LENGTH_CASE_WEIGHT_VARIABLE_INDEX; // 4 byte ByteBuffer bb_Case_Weight_Variable_Index = ByteBuffer.wrap(recordType1, offset_start, LENGTH_CASE_WEIGHT_VARIABLE_INDEX); if (isLittleEndian) { bb_Case_Weight_Variable_Index.order(ByteOrder.LITTLE_ENDIAN); } caseWeightVariableOBSIndex = bb_Case_Weight_Variable_Index.getInt(); smd.getFileInformation().put("caseWeightVariableOBSIndex", caseWeightVariableOBSIndex); // 1.6 4-byte Number of Cases offset_start = offset_end; offset_end += LENGTH_NUMBER_OF_CASES; // 4 byte ByteBuffer bb_Number_Of_Cases = ByteBuffer.wrap(recordType1, offset_start, LENGTH_NUMBER_OF_CASES); if (isLittleEndian) { bb_Number_Of_Cases.order(ByteOrder.LITTLE_ENDIAN); } int numberOfCases = bb_Number_Of_Cases.getInt(); if (numberOfCases < 0) { // -1 if numberOfCases is unknown throw new RuntimeException("number of cases is not recorded in the header"); } else { dbgLog.fine("RT1: number of cases is recorded= " + numberOfCases); caseQnty = numberOfCases; smd.getFileInformation().put("caseQnty", numberOfCases); } // 1.7 8-byte compression-bias [not long but double] offset_start = offset_end; offset_end += LENGTH_COMPRESSION_BIAS; // 8 byte ByteBuffer bb_compression_bias = ByteBuffer .wrap(Arrays.copyOfRange(recordType1, offset_start, offset_end)); if (isLittleEndian) { bb_compression_bias.order(ByteOrder.LITTLE_ENDIAN); } Double compressionBias = bb_compression_bias.getDouble(); if (compressionBias == 100d) { // 100 is expected dbgLog.fine("compressionBias is 100 as expected"); smd.getFileInformation().put("compressionBias", 100); } else { dbgLog.fine("compression bias is not 100: " + compressionBias); smd.getFileInformation().put("compressionBias", compressionBias); } // 1.8 84-byte File Creation Information (date/time: dd MM yyhh:mm:ss + // 64-bytelabel) offset_start = offset_end; offset_end += LENGTH_FILE_CREATION_INFO; // 84 bytes String fileCreationInfo = getNullStrippedString( new String(Arrays.copyOfRange(recordType1, offset_start, offset_end), "US-ASCII")); dbgLog.fine("fileCreationInfo:\n" + fileCreationInfo + "\n"); String fileCreationDate = fileCreationInfo.substring(0, length_file_creation_date); int dateEnd = length_file_creation_date + length_file_creation_time; String fileCreationTime = fileCreationInfo.substring(length_file_creation_date, (dateEnd)); String fileCreationNote = fileCreationInfo.substring(dateEnd, length_file_creation_label); dbgLog.fine("fileDate=" + fileCreationDate); dbgLog.fine("fileTime=" + fileCreationTime); dbgLog.fine("fileNote" + fileCreationNote); smd.getFileInformation().put("fileDate", fileCreationDate); smd.getFileInformation().put("fileTime", fileCreationTime); smd.getFileInformation().put("fileNote", fileCreationNote); smd.getFileInformation().put("varFormat_schema", "SPSS"); // add the info to the fileInfo smd.getFileInformation().put("mimeType", MIME_TYPE[0]); smd.getFileInformation().put("fileFormat", MIME_TYPE[0]); smd.setValueLabelMappingTable(valueVariableMappingTable); } catch (IOException ex) { //ex.printStackTrace(); throw ex; } dbgLog.fine("***** decodeRecordType1(): end *****"); }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.sav.SAVFileReader.java
void decodeRecordType3and4(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordType3and4(): start *****"); Map<String, Map<String, String>> valueLabelTable = new LinkedHashMap<String, Map<String, String>>(); int safteyCounter = 0; while (true) { try {/*from w w w . j a v a 2 s. co m*/ if (stream == null) { throw new IllegalArgumentException("stream == null!"); } // this secton may not exit so first check the 4-byte header value //if (stream.markSupported()){ stream.mark(1000); //} // 3.0 check the first 4 bytes byte[] headerCode = new byte[LENGTH_RECORD_TYPE3_CODE]; int nbytes_rt3 = stream.read(headerCode, 0, LENGTH_RECORD_TYPE3_CODE); // to-do check against nbytes //printHexDump(headerCode, "RT3 header test"); ByteBuffer bb_header_code = ByteBuffer.wrap(headerCode, 0, LENGTH_RECORD_TYPE3_CODE); if (isLittleEndian) { bb_header_code.order(ByteOrder.LITTLE_ENDIAN); } int intRT3test = bb_header_code.getInt(); dbgLog.fine("header test value: RT3=" + intRT3test); if (intRT3test != 3) { //if (stream.markSupported()){ dbgLog.fine("iteration=" + safteyCounter); // We have encountered a record that's not type 3. This means we've // processed all the type 3/4 record pairs. So we want to rewind // the stream and return -- so that the appropriate record type // reader can be called on it. // But before we return, we need to save all the value labels // we have found: smd.setValueLabelTable(valueLabelTable); stream.reset(); return; //} } // 3.1 how many value-label pairs follow byte[] number_of_labels = new byte[LENGTH_RT3_HOW_MANY_LABELS]; int nbytes_3_1 = stream.read(number_of_labels); if (nbytes_3_1 == 0) { throw new IOException("RT 3: reading recordType3.1: no byte was read"); } ByteBuffer bb_number_of_labels = ByteBuffer.wrap(number_of_labels, 0, LENGTH_RT3_HOW_MANY_LABELS); if (isLittleEndian) { bb_number_of_labels.order(ByteOrder.LITTLE_ENDIAN); } int numberOfValueLabels = bb_number_of_labels.getInt(); dbgLog.fine("number of value-label pairs=" + numberOfValueLabels); ByteBuffer[] tempBB = new ByteBuffer[numberOfValueLabels]; String valueLabel[] = new String[numberOfValueLabels]; for (int i = 0; i < numberOfValueLabels; i++) { // read 8-byte as value byte[] value = new byte[LENGTH_RT3_VALUE]; int nbytes_3_value = stream.read(value); if (nbytes_3_value == 0) { throw new IOException("RT 3: reading recordType3 value: no byte was read"); } // note these 8 bytes are interpreted later // currently no information about which variable's (=> type unknown) ByteBuffer bb_value = ByteBuffer.wrap(value, 0, LENGTH_RT3_VALUE); if (isLittleEndian) { bb_value.order(ByteOrder.LITTLE_ENDIAN); } tempBB[i] = bb_value; dbgLog.fine("bb_value=" + Hex.encodeHex(bb_value.array())); /* double valueD = bb_value.getDouble(); dbgLog.fine("value="+valueD); */ // read 1st byte as unsigned integer = label_length // read label_length byte as label byte[] labelLengthByte = new byte[LENGTH_RT3_LABEL_LENGTH]; int nbytes_3_label_length = stream.read(labelLengthByte); // add check-routine here dbgLog.fine("labelLengthByte" + Hex.encodeHex(labelLengthByte)); dbgLog.fine("label length = " + labelLengthByte[0]); // the net-length of a value label is saved as // unsigned byte; however, the length is less than 127 // byte should be ok int rawLabelLength = labelLengthByte[0] & 0xFF; dbgLog.fine("rawLabelLength=" + rawLabelLength); // -1 =>1-byte already read int labelLength = getSAVobsAdjustedBlockLength(rawLabelLength + 1) - 1; byte[] valueLabelBytes = new byte[labelLength]; int nbytes_3_value_label = stream.read(valueLabelBytes); // ByteBuffer bb_label = ByteBuffer.wrap(valueLabel,0,labelLength); valueLabel[i] = StringUtils.stripEnd( new String(Arrays.copyOfRange(valueLabelBytes, 0, rawLabelLength), defaultCharSet), " "); dbgLog.fine(i + "-th valueLabel=" + valueLabel[i] + "<-"); } // iter rt3 dbgLog.fine("end of RT3 block"); dbgLog.fine("start of RT4 block"); // 4.0 check the first 4 bytes byte[] headerCode4 = new byte[LENGTH_RECORD_TYPE4_CODE]; int nbytes_rt4 = stream.read(headerCode4, 0, LENGTH_RECORD_TYPE4_CODE); if (nbytes_rt4 == 0) { throw new IOException("RT4: reading recordType4 value: no byte was read"); } //printHexDump(headerCode4, "RT4 header test"); ByteBuffer bb_header_code_4 = ByteBuffer.wrap(headerCode4, 0, LENGTH_RECORD_TYPE4_CODE); if (isLittleEndian) { bb_header_code_4.order(ByteOrder.LITTLE_ENDIAN); } int intRT4test = bb_header_code_4.getInt(); dbgLog.fine("header test value: RT4=" + intRT4test); if (intRT4test != 4) { throw new IOException("RT 4: reading recordType4 header: no byte was read"); } // 4.1 read the how-many-variables bytes byte[] howManyVariablesfollow = new byte[LENGTH_RT4_HOW_MANY_VARIABLES]; int nbytes_rt4_1 = stream.read(howManyVariablesfollow, 0, LENGTH_RT4_HOW_MANY_VARIABLES); ByteBuffer bb_howManyVariablesfollow = ByteBuffer.wrap(howManyVariablesfollow, 0, LENGTH_RT4_HOW_MANY_VARIABLES); if (isLittleEndian) { bb_howManyVariablesfollow.order(ByteOrder.LITTLE_ENDIAN); } int howManyVariablesRT4 = bb_howManyVariablesfollow.getInt(); dbgLog.fine("how many variables follow: RT4=" + howManyVariablesRT4); int length_indicies = LENGTH_RT4_VARIABLE_INDEX * howManyVariablesRT4; byte[] variableIdicesBytes = new byte[length_indicies]; int nbytes_rt4_2 = stream.read(variableIdicesBytes, 0, length_indicies); // !!!!! Caution: variableIndex in RT4 starts from 1 NOT ** 0 ** int[] variableIndex = new int[howManyVariablesRT4]; int offset = 0; for (int i = 0; i < howManyVariablesRT4; i++) { ByteBuffer bb_variable_index = ByteBuffer.wrap(variableIdicesBytes, offset, LENGTH_RT4_VARIABLE_INDEX); offset += LENGTH_RT4_VARIABLE_INDEX; if (isLittleEndian) { bb_variable_index.order(ByteOrder.LITTLE_ENDIAN); } variableIndex[i] = bb_variable_index.getInt(); dbgLog.fine(i + "-th variable index number=" + variableIndex[i]); } dbgLog.fine("variable index set=" + ArrayUtils.toString(variableIndex)); dbgLog.fine("subtract 1 from variableIndex for getting a variable info"); boolean isNumeric = OBSwiseTypelList.get(variableIndex[0] - 1) == 0 ? true : false; Map<String, String> valueLabelPair = new LinkedHashMap<String, String>(); if (isNumeric) { // numeric variable dbgLog.fine("processing of a numeric value-label table"); for (int j = 0; j < numberOfValueLabels; j++) { valueLabelPair.put(doubleNumberFormatter.format(tempBB[j].getDouble()), valueLabel[j]); } } else { // String variable dbgLog.fine("processing of a string value-label table"); for (int j = 0; j < numberOfValueLabels; j++) { valueLabelPair.put( StringUtils.stripEnd(new String((tempBB[j].array()), defaultCharSet), " "), valueLabel[j]); } } dbgLog.fine("valueLabePair=" + valueLabelPair); dbgLog.fine("key variable's (raw) index =" + variableIndex[0]); valueLabelTable.put(OBSIndexToVariableName.get(variableIndex[0] - 1), valueLabelPair); dbgLog.fine("valueLabelTable=" + valueLabelTable); // create a mapping table that finds the key variable for this mapping table String keyVariableName = OBSIndexToVariableName.get(variableIndex[0] - 1); for (int vn : variableIndex) { valueVariableMappingTable.put(OBSIndexToVariableName.get(vn - 1), keyVariableName); } dbgLog.fine("valueVariableMappingTable:\n" + valueVariableMappingTable); } catch (IOException ex) { //ex.printStackTrace(); throw ex; } safteyCounter++; if (safteyCounter >= 1000000) { break; } } //while smd.setValueLabelTable(valueLabelTable); dbgLog.fine("***** decodeRecordType3and4(): end *****"); }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeRecordType7(BufferedInputStream stream) throws IOException { dbgLog.fine("decodeRecordType7(): start"); int counter = 0; int[] headerSection = new int[2]; // the variables below may no longer needed; // but they may be useful for debugging/logging purposes. /// // RecordType 7 /// // Subtype 3 /// List<Integer> releaseMachineSpecificInfo = new ArrayList<Integer>(); /// List<String> releaseMachineSpecificInfoHex = new ArrayList<String>(); /// // Subytpe 4 /// Map<String, Double> OBSTypeValue = new LinkedHashMap<String, Double>(); /// Map<String, String> OBSTypeHexValue = new LinkedHashMap<String, String>(); //Subtype 11// w ww .j a v a 2 s . c om /// List<Integer> measurementLevel = new ArrayList<Integer>(); /// List<Integer> columnWidth = new ArrayList<Integer>(); /// List<Integer> alignment = new ArrayList<Integer>(); while (true) { try { if (stream == null) { throw new IllegalArgumentException("RT7: stream == null!"); } // first check the 4-byte header value //if (stream.markSupported()){ stream.mark(1000); //} // 7.0 check the first 4 bytes byte[] headerCodeRt7 = new byte[LENGTH_RECORD_TYPE7_CODE]; int nbytes_rt7 = stream.read(headerCodeRt7, 0, LENGTH_RECORD_TYPE7_CODE); // to-do check against nbytes //printHexDump(headerCodeRt7, "RT7 header test"); ByteBuffer bb_header_code_rt7 = ByteBuffer.wrap(headerCodeRt7, 0, LENGTH_RECORD_TYPE7_CODE); if (isLittleEndian) { bb_header_code_rt7.order(ByteOrder.LITTLE_ENDIAN); } int intRT7test = bb_header_code_rt7.getInt(); dbgLog.fine("RT7: header test value=" + intRT7test); if (intRT7test != 7) { //if (stream.markSupported()){ //out.print("iteration="+safteyCounter); //dbgLog.fine("iteration="+safteyCounter); dbgLog.fine("intRT7test failed=" + intRT7test); dbgLog.fine("counter=" + counter); stream.reset(); return; //} } // 7.1 check 4-byte integer Sub-Type Code byte[] length_sub_type_code = new byte[LENGTH_RT7_SUB_TYPE_CODE]; int nbytes_rt7_1 = stream.read(length_sub_type_code, 0, LENGTH_RT7_SUB_TYPE_CODE); // to-do check against nbytes //printHexDump(length_how_many_line_bytes, "RT7 how_many_line_bytes"); ByteBuffer bb_sub_type_code = ByteBuffer.wrap(length_sub_type_code, 0, LENGTH_RT7_SUB_TYPE_CODE); if (isLittleEndian) { bb_sub_type_code.order(ByteOrder.LITTLE_ENDIAN); } int subTypeCode = bb_sub_type_code.getInt(); dbgLog.fine("RT7: subTypeCode=" + subTypeCode); switch (subTypeCode) { case 3: // 3: Release andMachine-Specific Integer Information //parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; int numberOfUnits = headerSection[1]; for (int i = 0; i < numberOfUnits; i++) { dbgLog.finer(i + "-th fieldData"); byte[] work = new byte[unitLength]; int nb = stream.read(work); dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(work))); ByteBuffer bb_field = ByteBuffer.wrap(work); if (isLittleEndian) { bb_field.order(ByteOrder.LITTLE_ENDIAN); } String dataInHex = new String(Hex.encodeHex(bb_field.array())); /// releaseMachineSpecificInfoHex.add(dataInHex); dbgLog.finer("raw bytes in Hex:" + dataInHex); if (unitLength == 4) { int fieldData = bb_field.getInt(); dbgLog.finer("fieldData(int)=" + fieldData); dbgLog.finer("fieldData in Hex=0x" + Integer.toHexString(fieldData)); /// releaseMachineSpecificInfo.add(fieldData); } } /// dbgLog.fine("releaseMachineSpecificInfo="+releaseMachineSpecificInfo); /// dbgLog.fine("releaseMachineSpecificInfoHex="+releaseMachineSpecificInfoHex); } else { // throw new IOException } dbgLog.fine("***** end of subType 3 ***** \n"); break; case 4: // Release andMachine-SpecificOBS-Type Information headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; int numberOfUnits = headerSection[1]; for (int i = 0; i < numberOfUnits; i++) { dbgLog.finer(i + "-th fieldData:" + RecordType7SubType4Fields.get(i)); byte[] work = new byte[unitLength]; int nb = stream.read(work); dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(work))); ByteBuffer bb_field = ByteBuffer.wrap(work); dbgLog.finer("byte order=" + bb_field.order().toString()); if (isLittleEndian) { bb_field.order(ByteOrder.LITTLE_ENDIAN); } ByteBuffer bb_field_dup = bb_field.duplicate(); OBSTypeHexValue.put(RecordType7SubType4Fields.get(i), new String(Hex.encodeHex(bb_field.array()))); // dbgLog.finer("raw bytes in Hex:"+ // OBSTypeHexValue.get(RecordType7SubType4Fields.get(i))); if (unitLength == 8) { double fieldData = bb_field.getDouble(); /// OBSTypeValue.put(RecordType7SubType4Fields.get(i), fieldData); dbgLog.finer("fieldData(double)=" + fieldData); OBSTypeHexValue.put(RecordType7SubType4Fields.get(i), Double.toHexString(fieldData)); dbgLog.fine("fieldData in Hex=" + Double.toHexString(fieldData)); } } /// dbgLog.fine("OBSTypeValue="+OBSTypeValue); /// dbgLog.fine("OBSTypeHexValue="+OBSTypeHexValue); } else { // throw new IOException } dbgLog.fine("***** end of subType 4 ***** \n"); break; case 5: // Variable Sets Information parseRT7SubTypefield(stream); break; case 6: // Trends date information parseRT7SubTypefield(stream); break; case 7: // Multiple response groups parseRT7SubTypefield(stream); break; case 8: // Windows Data Entry data parseRT7SubTypefield(stream); break; case 9: // parseRT7SubTypefield(stream); break; case 10: // TextSmart data parseRT7SubTypefield(stream); break; case 11: // Msmt level, col width, & alignment //parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; int numberOfUnits = headerSection[1]; for (int i = 0; i < numberOfUnits; i++) { dbgLog.finer(i + "-th fieldData"); byte[] work = new byte[unitLength]; int nb = stream.read(work); dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(work))); ByteBuffer bb_field = ByteBuffer.wrap(work); if (isLittleEndian) { bb_field.order(ByteOrder.LITTLE_ENDIAN); } dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(bb_field.array()))); if (unitLength == 4) { int fieldData = bb_field.getInt(); dbgLog.finer("fieldData(int)=" + fieldData); dbgLog.finer("fieldData in Hex=0x" + Integer.toHexString(fieldData)); int remainder = i % 3; dbgLog.finer("remainder=" + remainder); if (remainder == 0) { /// measurementLevel.add(fieldData); } else if (remainder == 1) { /// columnWidth.add(fieldData); } else if (remainder == 2) { /// alignment.add(fieldData); } } } } else { // throw new IOException } /// dbgLog.fine("measurementLevel="+measurementLevel); /// dbgLog.fine("columnWidth="+columnWidth); /// dbgLog.fine("alignment="+alignment); dbgLog.fine("end of subType 11\n"); break; case 12: // Windows Data Entry GUID parseRT7SubTypefield(stream); break; case 13: // Extended variable names // parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; dbgLog.fine("RT7: unitLength=" + unitLength); int numberOfUnits = headerSection[1]; dbgLog.fine("RT7: numberOfUnits=" + numberOfUnits); byte[] work = new byte[unitLength * numberOfUnits]; int nbtyes13 = stream.read(work); String[] variableShortLongNamePairs = new String(work, "US-ASCII").split("\t"); for (int i = 0; i < variableShortLongNamePairs.length; i++) { dbgLog.fine("RT7: " + i + "-th pair" + variableShortLongNamePairs[i]); String[] pair = variableShortLongNamePairs[i].split("="); shortToLongVariableNameTable.put(pair[0], pair[1]); } dbgLog.fine("RT7: shortToLongVarialbeNameTable" + shortToLongVariableNameTable); // We are saving the short-to-long name map; at the // end of ingest, we'll go through the data variables and // change the names accordingly. // smd.setShortToLongVarialbeNameTable(shortToLongVarialbeNameTable); } else { // throw new IOException } break; case 14: // Extended strings //parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; dbgLog.fine("RT7.14: unitLength=" + unitLength); int numberOfUnits = headerSection[1]; dbgLog.fine("RT7.14: numberOfUnits=" + numberOfUnits); byte[] work = new byte[unitLength * numberOfUnits]; int nbtyes13 = stream.read(work); String[] extendedVariablesSizePairs = new String(work, defaultCharSet).split("\000\t"); for (int i = 0; i < extendedVariablesSizePairs.length; i++) { dbgLog.fine("RT7.14: " + i + "-th pair" + extendedVariablesSizePairs[i]); if (extendedVariablesSizePairs[i].indexOf("=") > 0) { String[] pair = extendedVariablesSizePairs[i].split("="); extendedVariablesSizeTable.put(pair[0], Integer.valueOf(pair[1])); } } dbgLog.fine("RT7.14: extendedVariablesSizeTable" + extendedVariablesSizeTable); } else { // throw new IOException } break; case 15: // Clementine Metadata parseRT7SubTypefield(stream); break; case 16: // 64 bit N of cases parseRT7SubTypefield(stream); break; case 17: // File level attributes parseRT7SubTypefield(stream); break; case 18: // Variable attributes parseRT7SubTypefield(stream); break; case 19: // Extended multiple response groups parseRT7SubTypefield(stream); break; case 20: // Character encoding, aka code page. // Must be a version 16+ feature (?). // Starting v.16, the default character encoding for SAV // files is UTF-8; but then it is possible to specify an // alternative encoding here. // A typical use case would be people setting it to "ISO-Latin" // or "windows-1252", or a similar 8-bit encoding to store // text with standard Western European accents. // -- L.A. headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; dbgLog.fine("RT7-20: unitLength=" + unitLength); int numberOfUnits = headerSection[1]; dbgLog.fine("RT7-20: numberOfUnits=" + numberOfUnits); byte[] rt7st20bytes = new byte[unitLength * numberOfUnits]; int nbytes20 = stream.read(rt7st20bytes); String dataCharSet = new String(rt7st20bytes, "US-ASCII"); if (dataCharSet != null && !(dataCharSet.equals(""))) { dbgLog.fine("RT7-20: data charset: " + dataCharSet); defaultCharSet = dataCharSet; } } /*else { // TODO: // decide if the exception should actually be thrown here! // -- L.A. 4.0 beta // throw new IOException }*/ break; case 21: // Value labels for long strings parseRT7SubTypefield(stream); break; case 22: // Missing values for long strings parseRT7SubTypefield(stream); break; default: parseRT7SubTypefield(stream); } } catch (IOException ex) { //ex.printStackTrace(); throw ex; } counter++; if (counter > 20) { break; } } dbgLog.fine("RT7: counter=" + counter); dbgLog.fine("RT7: decodeRecordType7(): end"); }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.sav.SAVFileReader.java
void decodeRecordType7(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordType7(): start *****"); int counter = 0; int[] headerSection = new int[2]; // the variables below may no longer needed; // but they may be useful for debugging/logging purposes. /// // RecordType 7 /// // Subtype 3 /// List<Integer> releaseMachineSpecificInfo = new ArrayList<Integer>(); /// List<String> releaseMachineSpecificInfoHex = new ArrayList<String>(); /// // Subytpe 4 /// Map<String, Double> OBSTypeValue = new LinkedHashMap<String, Double>(); /// Map<String, String> OBSTypeHexValue = new LinkedHashMap<String, String>(); //Subtype 11/*from w w w.ja v a2 s . com*/ /// List<Integer> measurementLevel = new ArrayList<Integer>(); /// List<Integer> columnWidth = new ArrayList<Integer>(); /// List<Integer> alignment = new ArrayList<Integer>(); Map<String, String> shortToLongVarialbeNameTable = new LinkedHashMap<String, String>(); while (true) { try { if (stream == null) { throw new IllegalArgumentException("RT7: stream == null!"); } // first check the 4-byte header value //if (stream.markSupported()){ stream.mark(1000); //} // 7.0 check the first 4 bytes byte[] headerCodeRt7 = new byte[LENGTH_RECORD_TYPE7_CODE]; int nbytes_rt7 = stream.read(headerCodeRt7, 0, LENGTH_RECORD_TYPE7_CODE); // to-do check against nbytes //printHexDump(headerCodeRt7, "RT7 header test"); ByteBuffer bb_header_code_rt7 = ByteBuffer.wrap(headerCodeRt7, 0, LENGTH_RECORD_TYPE7_CODE); if (isLittleEndian) { bb_header_code_rt7.order(ByteOrder.LITTLE_ENDIAN); } int intRT7test = bb_header_code_rt7.getInt(); dbgLog.fine("RT7: header test value=" + intRT7test); if (intRT7test != 7) { //if (stream.markSupported()){ //out.print("iteration="+safteyCounter); //dbgLog.fine("iteration="+safteyCounter); dbgLog.fine("intRT7test failed=" + intRT7test); dbgLog.fine("counter=" + counter); stream.reset(); return; //} } // 7.1 check 4-byte integer Sub-Type Code byte[] length_sub_type_code = new byte[LENGTH_RT7_SUB_TYPE_CODE]; int nbytes_rt7_1 = stream.read(length_sub_type_code, 0, LENGTH_RT7_SUB_TYPE_CODE); // to-do check against nbytes //printHexDump(length_how_many_line_bytes, "RT7 how_many_line_bytes"); ByteBuffer bb_sub_type_code = ByteBuffer.wrap(length_sub_type_code, 0, LENGTH_RT7_SUB_TYPE_CODE); if (isLittleEndian) { bb_sub_type_code.order(ByteOrder.LITTLE_ENDIAN); } int subTypeCode = bb_sub_type_code.getInt(); dbgLog.fine("RT7: subTypeCode=" + subTypeCode); switch (subTypeCode) { case 3: // 3: Release andMachine-Specific Integer Information //parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; int numberOfUnits = headerSection[1]; for (int i = 0; i < numberOfUnits; i++) { dbgLog.finer(i + "-th fieldData"); byte[] work = new byte[unitLength]; int nb = stream.read(work); dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(work))); ByteBuffer bb_field = ByteBuffer.wrap(work); if (isLittleEndian) { bb_field.order(ByteOrder.LITTLE_ENDIAN); } String dataInHex = new String(Hex.encodeHex(bb_field.array())); /// releaseMachineSpecificInfoHex.add(dataInHex); dbgLog.finer("raw bytes in Hex:" + dataInHex); if (unitLength == 4) { int fieldData = bb_field.getInt(); dbgLog.finer("fieldData(int)=" + fieldData); dbgLog.finer("fieldData in Hex=0x" + Integer.toHexString(fieldData)); /// releaseMachineSpecificInfo.add(fieldData); } } /// dbgLog.fine("releaseMachineSpecificInfo="+releaseMachineSpecificInfo); /// dbgLog.fine("releaseMachineSpecificInfoHex="+releaseMachineSpecificInfoHex); } else { // throw new IOException } dbgLog.fine("***** end of subType 3 ***** \n"); break; case 4: // Release andMachine-SpecificOBS-Type Information headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; int numberOfUnits = headerSection[1]; for (int i = 0; i < numberOfUnits; i++) { dbgLog.finer(i + "-th fieldData:" + RecordType7SubType4Fields.get(i)); byte[] work = new byte[unitLength]; int nb = stream.read(work); dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(work))); ByteBuffer bb_field = ByteBuffer.wrap(work); dbgLog.finer("byte order=" + bb_field.order().toString()); if (isLittleEndian) { bb_field.order(ByteOrder.LITTLE_ENDIAN); } ByteBuffer bb_field_dup = bb_field.duplicate(); OBSTypeHexValue.put(RecordType7SubType4Fields.get(i), new String(Hex.encodeHex(bb_field.array()))); // dbgLog.finer("raw bytes in Hex:"+ // OBSTypeHexValue.get(RecordType7SubType4Fields.get(i))); if (unitLength == 8) { double fieldData = bb_field.getDouble(); /// OBSTypeValue.put(RecordType7SubType4Fields.get(i), fieldData); dbgLog.finer("fieldData(double)=" + fieldData); OBSTypeHexValue.put(RecordType7SubType4Fields.get(i), Double.toHexString(fieldData)); dbgLog.fine("fieldData in Hex=" + Double.toHexString(fieldData)); } } /// dbgLog.fine("OBSTypeValue="+OBSTypeValue); /// dbgLog.fine("OBSTypeHexValue="+OBSTypeHexValue); } else { // throw new IOException } dbgLog.fine("***** end of subType 4 ***** \n"); break; case 5: // Variable Sets Information parseRT7SubTypefield(stream); break; case 6: // Trends date information parseRT7SubTypefield(stream); break; case 7: // Multiple response groups parseRT7SubTypefield(stream); break; case 8: // Windows Data Entry data parseRT7SubTypefield(stream); break; case 9: // parseRT7SubTypefield(stream); break; case 10: // TextSmart data parseRT7SubTypefield(stream); break; case 11: // Msmt level, col width, & alignment //parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; int numberOfUnits = headerSection[1]; for (int i = 0; i < numberOfUnits; i++) { dbgLog.finer(i + "-th fieldData"); byte[] work = new byte[unitLength]; int nb = stream.read(work); dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(work))); ByteBuffer bb_field = ByteBuffer.wrap(work); if (isLittleEndian) { bb_field.order(ByteOrder.LITTLE_ENDIAN); } dbgLog.finer("raw bytes in Hex:" + new String(Hex.encodeHex(bb_field.array()))); if (unitLength == 4) { int fieldData = bb_field.getInt(); dbgLog.finer("fieldData(int)=" + fieldData); dbgLog.finer("fieldData in Hex=0x" + Integer.toHexString(fieldData)); int remainder = i % 3; dbgLog.finer("remainder=" + remainder); if (remainder == 0) { /// measurementLevel.add(fieldData); } else if (remainder == 1) { /// columnWidth.add(fieldData); } else if (remainder == 2) { /// alignment.add(fieldData); } } } } else { // throw new IOException } /// dbgLog.fine("measurementLevel="+measurementLevel); /// dbgLog.fine("columnWidth="+columnWidth); /// dbgLog.fine("alignment="+alignment); dbgLog.fine("***** end of subType 11 ***** \n"); break; case 12: // Windows Data Entry GUID parseRT7SubTypefield(stream); break; case 13: // Extended variable names // parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; dbgLog.fine("RT7: unitLength=" + unitLength); int numberOfUnits = headerSection[1]; dbgLog.fine("RT7: numberOfUnits=" + numberOfUnits); byte[] work = new byte[unitLength * numberOfUnits]; int nbtyes13 = stream.read(work); String[] variableShortLongNamePairs = new String(work, "US-ASCII").split("\t"); for (int i = 0; i < variableShortLongNamePairs.length; i++) { dbgLog.fine("RT7: " + i + "-th pair" + variableShortLongNamePairs[i]); String[] pair = variableShortLongNamePairs[i].split("="); shortToLongVarialbeNameTable.put(pair[0], pair[1]); } dbgLog.fine("RT7: shortToLongVarialbeNameTable" + shortToLongVarialbeNameTable); smd.setShortToLongVarialbeNameTable(shortToLongVarialbeNameTable); } else { // throw new IOException } break; case 14: // Extended strings //parseRT7SubTypefield(stream); headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null) { int unitLength = headerSection[0]; dbgLog.fine("RT7.14: unitLength=" + unitLength); int numberOfUnits = headerSection[1]; dbgLog.fine("RT7.14: numberOfUnits=" + numberOfUnits); byte[] work = new byte[unitLength * numberOfUnits]; int nbtyes13 = stream.read(work); String[] extendedVariablesSizePairs = new String(work, defaultCharSet).split("\000\t"); for (int i = 0; i < extendedVariablesSizePairs.length; i++) { dbgLog.fine("RT7.14: " + i + "-th pair" + extendedVariablesSizePairs[i]); if (extendedVariablesSizePairs[i].indexOf("=") > 0) { String[] pair = extendedVariablesSizePairs[i].split("="); extendedVariablesSizeTable.put(pair[0], Integer.valueOf(pair[1])); } } dbgLog.fine("RT7.14: extendedVariablesSizeTable" + extendedVariablesSizeTable); } else { // throw new IOException } break; case 15: // Clementine Metadata parseRT7SubTypefield(stream); break; case 16: // 64 bit N of cases parseRT7SubTypefield(stream); break; case 17: // File level attributes parseRT7SubTypefield(stream); break; case 18: // Variable attributes parseRT7SubTypefield(stream); break; case 19: // Extended multiple response groups parseRT7SubTypefield(stream); break; case 20: // Encoding, aka code page parseRT7SubTypefield(stream); /* TODO: This needs to be researched; * Is this field really used, ever? headerSection = parseRT7SubTypefieldHeader(stream); if (headerSection != null){ int unitLength = headerSection[0]; dbgLog.fine("RT7-20: unitLength="+unitLength); int numberOfUnits = headerSection[1]; dbgLog.fine("RT7-20: numberOfUnits="+numberOfUnits); byte[] rt7st20bytes = new byte[unitLength*numberOfUnits]; int nbytes20 = stream.read(rt7st20bytes); String dataCharSet = new String(rt7st20bytes,"US-ASCII"); if (dataCharSet != null && !(dataCharSet.equals(""))) { dbgLog.fine("RT7-20: data charset: "+ dataCharSet); defaultCharSet = dataCharSet; } } else { // throw new IOException } * */ break; case 21: // Value labels for long strings parseRT7SubTypefield(stream); break; case 22: // Missing values for long strings parseRT7SubTypefield(stream); break; default: parseRT7SubTypefield(stream); } } catch (IOException ex) { //ex.printStackTrace(); throw ex; } counter++; if (counter > 20) { break; } } dbgLog.fine("RT7: counter=" + counter); dbgLog.fine("RT7: ***** decodeRecordType7(): end *****"); }
From source file:com.l2jfree.gameserver.network.L2ClientPacketHandlerFinal.java
@Override public L2ClientPacket handlePacket(ByteBuffer buf, L2Client client, final int opcode) { L2ClientPacket msg = null;/*w w w . j a v a 2s . com*/ GameClientState state = client.getState(); switch (state) { case CONNECTED: switch (opcode) { case 0x0e: msg = new ProtocolVersion(); break; case 0x2b: msg = new AuthLogin(); break; default: printDebug(buf, client, opcode); break; } break; case AUTHED: switch (opcode) { case 0x00: msg = new Logout(); break; case 0x0c: msg = new NewCharacter(); break; case 0x0d: msg = new CharacterDelete(); break; case 0x0f: // MoveBackwardsToLocation, lag issue break; case 0x12: msg = new CharacterSelected(); break; case 0x13: msg = new NewCharacterInit(); break; case 0x57: // RequestRestart, lag issue break; case 0x7b: msg = new CharacterRestore(); break; case 0xd0: int id2 = -1; if (buf.remaining() >= 2) { id2 = buf.getShort() & 0xffff; } else { if (Config.PACKET_HANDLER_DEBUG) _log.warn("Client: " + client.toString() + " sent a 0xd0 without the second opcode."); break; } switch (id2) { case 0x24: // RequestSaveInventoryOrder, lag issue break; case 0x36: msg = new CharacterPrevState(); break; case 0x39: // most probably using L2NET break; case 0x3d: // client definitely sends it right now, enable if supposed to be //msg = new RequestAllFortressInfo(); break; case 0x5a: int id3 = 0; if (buf.remaining() >= 4) { id3 = buf.getInt() & 0xffffffff; } else { if (Config.PACKET_HANDLER_DEBUG) _log.warn("Client: " + client + " sent a 0xd0:0x5a without the third opcode."); break; } switch (id3) { case 0x00: msg = new RequestExCubeGameChangeTeam(); break; default: printDebug(buf, client, opcode, id2, id3); break; } break; default: printDebug(buf, client, opcode, id2); } break; // to avoid unnecessary warning about invalid opcode (if the client lags a bit, then it starts spamming this packet) case 0x59: // ValidatePosition break; // default: printDebug(buf, client, opcode); break; } break; case IN_GAME: switch (opcode) { // to avoid unnecessary warning about invalid opcode (player clicked the button multiple times) case 0x12: // CharacterSelected break; // case 0x00: msg = new Logout(); break; case 0x01: msg = new AttackRequest(); break; case 0x03: msg = new RequestStartPledgeWar(); break; case 0x04: msg = new RequestReplyStartPledgeWar(); break; case 0x05: msg = new RequestStopPledgeWar(); break; case 0x06: // RequestSCCheck msg = new RequestReplyStopPledgeWar(); break; case 0x07: msg = new RequestSurrenderPledgeWar(); break; case 0x08: msg = new RequestReplySurrenderPledgeWar(); break; case 0x09: msg = new RequestSetPledgeCrest(); break; case 0x0b: msg = new RequestGiveNickName(); break; case 0x0f: msg = new MoveBackwardToLocation(); break; case 0x10: // Say break; case 0x11: msg = new EnterWorld(); break; case 0x14: msg = new RequestItemList(); break; case 0x15: // RequestEquipItem break; case 0x16: msg = new RequestUnEquipItem(); break; case 0x17: msg = new RequestDropItem(); break; case 0x19: msg = new UseItem(); break; case 0x1a: msg = new TradeRequest(); break; case 0x1b: msg = new AddTradeItem(); break; case 0x1c: msg = new TradeDone(); break; case 0x1f: msg = new Action(); break; case 0x22: msg = new RequestLinkHtml(); break; case 0x23: msg = new RequestBypassToServer(); break; case 0x24: msg = new RequestBBSwrite(); break; case 0x25: // RequestCreatePledge break; case 0x26: msg = new RequestJoinPledge(); break; case 0x27: msg = new RequestAnswerJoinPledge(); break; case 0x28: msg = new RequestWithdrawalPledge(); break; case 0x29: msg = new RequestOustPledgeMember(); break; case 0x2c: msg = new RequestGetItemFromPet(); break; case 0x2e: msg = new RequestAllyInfo(); break; case 0x2f: msg = new RequestCrystallizeItem(); break; case 0x30: msg = new RequestPrivateStoreManageSell(); break; case 0x31: msg = new SetPrivateStoreListSell(); break; case 0x32: msg = new AttackRequest(); break; case 0x33: // RequestTeleportPacket break; case 0x34: msg = new RequestSocialAction(); break; case 0x35: msg = new ChangeMoveType(); break; case 0x36: msg = new ChangeWaitType(); break; case 0x37: msg = new RequestSellItem(); break; case 0x38: // RequestMagicSkillList break; case 0x39: msg = new RequestMagicSkillUse(); break; case 0x3a: msg = new SendAppearing(); break; case 0x3b: if (Config.ALLOW_WAREHOUSE) msg = new SendWareHouseDepositList(); break; case 0x3c: msg = new SendWareHouseWithDrawList(); break; case 0x3d: msg = new RequestShortCutReg(); break; case 0x3f: msg = new RequestShortCutDel(); break; case 0x40: msg = new RequestBuyItem(); break; case 0x41: // RequestDismissPledge break; case 0x42: msg = new RequestJoinParty(); break; case 0x43: msg = new RequestAnswerJoinParty(); break; case 0x44: msg = new RequestWithDrawalParty(); break; case 0x45: msg = new RequestOustPartyMember(); break; case 0x46: // RequestDismissParty break; case 0x47: msg = new CannotMoveAnymore(); break; case 0x48: msg = new RequestTargetCanceld(); break; case 0x49: msg = new Say2(); break; case 0x4a: int id_2 = -1; if (buf.remaining() >= 2) { id_2 = buf.get() & 0xff; } else { if (Config.PACKET_HANDLER_DEBUG) _log.warn("Client: " + client.toString() + " sent a 0x4a without the second opcode."); break; } switch (id_2) { case 0x00: msg = new SuperCmdCharacterInfo(); break; case 0x01: msg = new SuperCmdSummonCmd(); break; case 0x02: msg = new SuperCmdServerStatus(); break; case 0x03: msg = new SendL2ParamSetting(); break; default: printDebug(buf, client, opcode, id_2); break; } break; case 0x4d: msg = new RequestPledgeMemberList(); break; case 0x4f: //RequestMagicList break; case 0x50: msg = new RequestSkillList(); break; case 0x52: msg = new MoveWithDelta(); break; case 0x53: msg = new RequestGetOnVehicle(); break; case 0x54: msg = new RequestGetOffVehicle(); break; case 0x55: msg = new AnswerTradeRequest(); break; case 0x56: msg = new RequestActionUse(); break; case 0x57: msg = new RequestRestart(); break; case 0x58: msg = new RequestSiegeInfo(); break; case 0x59: msg = new ValidatePosition(); break; case 0x5a: // RequestSEKCustom break; case 0x5b: // StartRotating break; case 0x5c: // FinishRotating break; case 0x5e: msg = new RequestShowBoard(); break; case 0x5f: msg = new RequestEnchantItem(); break; case 0x60: msg = new RequestDestroyItem(); break; case 0x62: msg = new RequestQuestList(); break; case 0x63: // RequestDestroyQuest msg = new RequestQuestAbort(); break; case 0x65: msg = new RequestPledgeInfo(); break; case 0x66: msg = new RequestPledgeExtendedInfo(); break; case 0x67: msg = new RequestPledgeCrest(); break; case 0x6b: msg = new RequestSendFriendMsg(); break; case 0x6c: msg = new RequestShowMiniMap(); break; case 0x6d: // RequestSendMsnChatLog break; case 0x6e: //RequestReload msg = new RequestRecordInfo(); break; case 0x6f: msg = new RequestHennaEquip(); break; case 0x70: msg = new RequestHennaRemoveList(); break; case 0x71: msg = new RequestHennaItemRemoveInfo(); break; case 0x72: msg = new RequestHennaRemove(); break; case 0x73: msg = new RequestAquireSkillInfo(); break; case 0x74: msg = new SendBypassBuildCmd(); break; case 0x75: msg = new RequestMoveToLocationInVehicle(); break; case 0x76: msg = new CannotMoveAnymoreInVehicle(); break; case 0x77: msg = new RequestFriendInvite(); break; case 0x78: msg = new RequestAnswerFriendInvite(); break; case 0x79: msg = new RequestFriendList(); break; case 0x7a: msg = new RequestFriendDel(); break; case 0x7c: // send when a skill to be learned is selected msg = new RequestAquireSkill(); break; case 0x7d: msg = new RequestRestartPoint(); break; case 0x7e: msg = new RequestGMCommand(); break; case 0x7f: msg = new RequestPartyMatchConfig(); break; case 0x80: msg = new RequestPartyMatchList(); break; case 0x81: msg = new RequestPartyMatchDetail(); break; case 0x83: msg = new RequestPrivateStoreBuy(); break; case 0x85: msg = new RequestTutorialLinkHtml(); break; case 0x86: msg = new RequestTutorialPassCmdToServer(); break; case 0x87: msg = new RequestTutorialQuestionMark(); break; case 0x88: msg = new RequestTutorialClientEvent(); break; case 0x89: msg = new RequestPetition(); break; case 0x8a: msg = new RequestPetitionCancel(); break; case 0x8b: msg = new RequestGmList(); break; case 0x8c: msg = new RequestJoinAlly(); break; case 0x8d: msg = new RequestAnswerJoinAlly(); break; case 0x8e: msg = new RequestWithdrawAlly(); break; case 0x8f: msg = new RequestOustAlly(); break; case 0x90: msg = new RequestDismissAlly(); break; case 0x91: msg = new RequestSetAllyCrest(); break; case 0x92: msg = new RequestAllyCrest(); break; case 0x93: msg = new RequestChangePetName(); break; case 0x94: msg = new RequestPetUseItem(); break; case 0x95: msg = new RequestGiveItemToPet(); break; case 0x96: msg = new RequestPrivateStoreQuitSell(); break; case 0x97: msg = new SetPrivateStoreMsgSell(); break; case 0x98: msg = new RequestPetGetItem(); break; case 0x99: msg = new RequestPrivateStoreManageBuy(); break; case 0x9a: msg = new SetPrivateStoreListBuy(); break; case 0x9c: msg = new RequestPrivateStoreQuitBuy(); break; case 0x9d: msg = new SetPrivateStoreMsgBuy(); break; case 0x9f: // SendPrivateStoreBuyList msg = new RequestPrivateStoreSell(); break; case 0xa0: //SendTimeCheckPacket break; case 0xa6: // RequestSkillCoolTime break; case 0xa7: msg = new RequestPackageSendableItemList(); break; case 0xa8: msg = new RequestPackageSend(); break; case 0xa9: msg = new RequestBlock(); break; case 0xaa: msg = new RequestSiegeInfo(); break; case 0xab: msg = new RequestSiegeAttackerList(); break; case 0xac: msg = new RequestSiegeDefenderList(); break; case 0xad: msg = new RequestJoinSiege(); break; case 0xae: msg = new RequestConfirmSiegeWaitingList(); break; case 0xaf: // RequestSetCastleSiegeTime break; case 0xb0: msg = new MultiSellChoose(); break; case 0xb1: // NetPing break; case 0xb2: msg = new RequestRemainTime(); break; case 0xb3: msg = new BypassUserCmd(); break; case 0xb4: msg = new SnoopQuit(); break; case 0xb5: msg = new RequestRecipeBookOpen(); break; case 0xb6: // RequestRecipeItemDelete msg = new RequestRecipeBookDestroy(); break; case 0xb7: msg = new RequestRecipeItemMakeInfo(); break; case 0xb8: msg = new RequestRecipeItemMakeSelf(); break; case 0xb9: // RequestRecipeShopManageList break; case 0xba: msg = new RequestRecipeShopMessageSet(); break; case 0xbb: msg = new RequestRecipeShopListSet(); break; case 0xbc: msg = new RequestRecipeShopManageQuit(); break; case 0xbd: // RequestRecipeShopManageCancel break; case 0xbe: msg = new RequestRecipeShopMakeInfo(); break; case 0xbf: msg = new RequestRecipeShopMakeItem(); break; case 0xc0: msg = new RequestRecipeShopSellList(); break; case 0xc1: msg = new RequestObserverEnd(); break; case 0xc2: // VoteSociality msg = new VoteSociality(); break; case 0xc3: // RequestHennaItemList msg = new RequestHennaDrawList(); break; case 0xc4: msg = new RequestHennaItemDrawInfo(); break; case 0xc5: msg = new RequestBuySeed(); break; case 0xc6: msg = new ConfirmDlgAnswer(); break; case 0xc7: msg = new RequestWearItem(); break; case 0xc8: msg = new RequestSSQStatus(); break; case 0xc9: // PetitionVote break; case 0xcb: msg = new GameGuardReply(); break; case 0xcc: // Clan Privileges msg = new RequestPledgePower(); break; case 0xcd: msg = new RequestMakeMacro(); break; case 0xce: msg = new RequestDeleteMacro(); break; // Manor case 0xcf: // RequestProcureCrop msg = new RequestBuyProcure(); break; case 0xd0: int id2 = -1; if (buf.remaining() >= 2) { id2 = buf.getShort() & 0xffff; } else { if (Config.PACKET_HANDLER_DEBUG) _log.warn("Client: " + client.toString() + " sent a 0xd0 without the second opcode."); break; } switch (id2) { case 0x01: msg = new RequestManorList(); break; case 0x02: msg = new RequestProcureCropList(); break; case 0x03: msg = new RequestSetSeed(); break; case 0x04: msg = new RequestSetCrop(); break; case 0x05: msg = new RequestWriteHeroWords(); break; case 0x06: msg = new RequestExAskJoinMPCC(); break; case 0x07: msg = new RequestExAcceptJoinMPCC(); break; case 0x08: msg = new RequestExOustFromMPCC(); break; case 0x09: msg = new RequestOustFromPartyRoom(); break; case 0x0a: msg = new RequestDismissPartyRoom(); break; case 0x0b: msg = new RequestWithdrawPartyRoom(); break; case 0x0c: msg = new RequestChangePartyLeader(); break; case 0x0d: msg = new RequestAutoSoulShot(); break; case 0x0e: msg = new RequestExEnchantSkillInfo(); break; case 0x0f: msg = new RequestExEnchantSkill(); break; case 0x10: msg = new RequestExPledgeCrestLarge(); break; case 0x11: msg = new RequestExSetPledgeCrestLarge(); break; case 0x12: msg = new RequestPledgeSetAcademyMaster(); break; case 0x13: msg = new RequestPledgePowerGradeList(); break; case 0x14: msg = new RequestPledgeMemberPowerInfo(); break; case 0x15: msg = new RequestPledgeSetMemberPowerGrade(); break; case 0x16: msg = new RequestPledgeMemberInfo(); break; case 0x17: msg = new RequestPledgeWarList(); break; case 0x18: msg = new RequestExFishRanking(); break; case 0x19: msg = new RequestPCCafeCouponUse(); break; case 0x1b: msg = new RequestDuelStart(); break; case 0x1c: msg = new RequestDuelAnswerStart(); break; case 0x1d: // RequestExSetTutorial break; case 0x1e: msg = new RequestExRqItemLink(); break; case 0x1f: // CanNotMoveAnymoreAirShip break; case 0x20: msg = new MoveToLocationInAirShip(); break; case 0x21: msg = new RequestKeyMapping(); break; case 0x22: // RequestSaveKeyMapping break; case 0x23: msg = new RequestExRemoveItemAttribute(); break; case 0x24: msg = new RequestSaveInventoryOrder(); break; case 0x25: msg = new RequestExitPartyMatchingWaitingRoom(); break; case 0x26: msg = new RequestConfirmTargetItem(); break; case 0x27: msg = new RequestConfirmRefinerItem(); break; case 0x28: msg = new RequestConfirmGemStone(); break; case 0x29: msg = new RequestOlympiadObserverEnd(); break; case 0x2a: msg = new RequestCursedWeaponList(); break; case 0x2b: msg = new RequestCursedWeaponLocation(); break; case 0x2c: msg = new RequestPledgeReorganizeMember(); break; case 0x2d: msg = new RequestExMPCCShowPartyMembersInfo(); break; case 0x2e: msg = new RequestOlympiadMatchList(); break; case 0x2f: msg = new RequestAskJoinPartyRoom(); break; case 0x30: msg = new AnswerJoinPartyRoom(); break; case 0x31: msg = new RequestListPartyMatchingWaitingRoom(); break; case 0x32: msg = new RequestExEnchantSkillSafe(); break; case 0x33: msg = new RequestExEnchantSkillUntrain(); break; case 0x34: msg = new RequestExEnchantSkillRouteChange(); break; case 0x35: msg = new RequestExEnchantItemAttribute(); break; case 0x36: msg = new ExGetOnAirShip(); break; case 0x38: // MoveToLocationAirShip break; case 0x39: msg = new RequestBidItemAuction(); break; case 0x3a: msg = new RequestInfoItemAuction(); break; case 0x3b: msg = new RequestExChangeName(); break; case 0x3c: msg = new RequestAllCastleInfo(); break; case 0x3d: msg = new RequestAllFortressInfo(); break; case 0x3e: msg = new RequestAllAgitInfo(); break; case 0x3f: msg = new RequestFortressSiegeInfo(); break; case 0x40: msg = new RequestGetBossRecord(); break; case 0x41: msg = new RequestRefine(); break; case 0x42: msg = new RequestConfirmCancelItem(); break; case 0x43: msg = new RequestRefineCancel(); break; case 0x44: msg = new RequestExMagicSkillUseGround(); break; case 0x45: msg = new RequestDuelSurrender(); break; case 0x46: msg = new RequestExEnchantSkillInfoDetail(); break; case 0x48: msg = new RequestFortressMapInfo(); break; case 0x49: msg = new RequestPVPMatchRecord(); break; case 0x4a: msg = new SetPrivateStoreWholeMsg(); break; case 0x4b: msg = new RequestDispel(); break; case 0x4c: msg = new RequestExTryToPutEnchantTargetItem(); break; case 0x4d: msg = new RequestExTryToPutEnchantSupportItem(); break; case 0x4e: msg = new RequestExCancelEnchantItem(); break; case 0x4f: msg = new RequestChangeNicknameColor(); break; case 0x50: msg = new RequestResetNickname(); break; case 0x51: int id3 = 0; if (buf.remaining() >= 4) { id3 = buf.getInt() & 0xffffffff; } else { if (Config.PACKET_HANDLER_DEBUG) _log.warn("Client: " + client + " sent a 0xd0:0x51 without the third opcode."); break; } switch (id3) { case 0x00: msg = new RequestBookMarkSlotInfo(); break; case 0x01: msg = new RequestSaveBookMarkSlot(); break; case 0x02: msg = new RequestModifyBookMarkSlot(); break; case 0x03: msg = new RequestDeleteBookMarkSlot(); break; case 0x04: msg = new RequestTeleportBookMark(); break; case 0x05: // RequestChangeBookMarkSlot break; default: printDebug(buf, client, opcode, id2, id3); break; } break; case 0x52: msg = new RequestWithDrawPremiumItem(); break; case 0x53: msg = new RequestJump(); break; case 0x54: msg = new RequestStartShowCrataeCubeRank(); break; case 0x55: msg = new RequestStopShowCrataeCubeRank(); break; case 0x56: msg = new NotifyStartMiniGame(); break; case 0x57: msg = new RequestJoinDominionWar(); break; case 0x58: msg = new RequestDominionInfo(); break; case 0x59: msg = new RequestExCleftEnter(); break; case 0x5a: id3 = 0; if (buf.remaining() >= 4) { id3 = buf.getInt() & 0xffffffff; } else { _log.warn("Client: " + client + " sent a 0xd0:0x5a without the third opcode."); break; } switch (id3) { case 0x00: msg = new RequestExCubeGameChangeTeam(); break; default: printDebug(buf, client, opcode, id2, id3); break; } break; case 0x5b: msg = new EndScenePlayer(); break; case 0x5c: msg = new RequestExBlockGameVote(); break; case 0x63: msg = new RequestSeedPhase(); break; case 0x65: msg = new BrGamePoint(); break; case 0x66: msg = new BrProductList(); break; case 0x67: msg = new BrProductInfo(); break; case 0x68: msg = new BrBuyProduct(); break; default: printDebug(buf, client, opcode, id2); break; } break; /* * case 0xee: msg = new RequestChangePartyLeader(data, * _client); break; */ default: printDebug(buf, client, opcode); break; } break; } return msg; }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeRecordType2(BufferedInputStream stream) throws IOException { dbgLog.fine("decodeRecordType2(): start"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }//from w w w. jav a2s . c o m Map<String, String> printFormatNameTable = new LinkedHashMap<String, String>(); Map<String, String> variableLabelMap = new LinkedHashMap<String, String>(); Map<String, List<String>> missingValueTable = new LinkedHashMap<String, List<String>>(); List<Integer> printFormatList = new ArrayList<Integer>(); String caseWeightVariableName = null; int caseWeightVariableIndex = 0; boolean lastVariableIsExtendable = false; boolean extendedVariableMode = false; boolean obs255 = false; String lastVariableName = null; String lastExtendedVariable = null; // this field repeats as many as the number of variables in // this sav file // (note that the above statement is not technically correct, this // record repeats not just for every variable in the file, but for // every OBS (8 byte unit); i.e., if a string is split into multiple // OBS units, each one will have its own RT2 record -- L.A.). // Each field constists of a fixed (32-byte) segment and // then a few variable segments: // if the variable has a label (3rd INT4 set to 1), then there's 4 more // bytes specifying the length of the label, and then that many bytes // holding the label itself (no more than 256). // Then if there are optional missing value units (4th INT4 set to 1) // there will be 3 more OBS units attached = 24 extra bytes. int variableCounter = 0; int obsSeqNumber = 0; int j; dbgLog.fine("RT2: Reading " + OBSUnitsPerCase + " OBS units."); for (j = 0; j < OBSUnitsPerCase; j++) { dbgLog.fine("RT2: " + j + "-th RT2 unit is being decoded."); // 2.0: read the fixed[=non-optional] 32-byte segment byte[] recordType2Fixed = new byte[LENGTH_RECORDTYPE2_FIXED]; try { int nbytes = stream.read(recordType2Fixed, 0, LENGTH_RECORDTYPE2_FIXED); //printHexDump(recordType2Fixed, "recordType2 part 1"); if (nbytes == 0) { throw new IOException("reading recordType2: no bytes read!"); } int offset = 0; // 2.1: create int-view of the bytebuffer for the first 16-byte segment int rt2_1st_4_units = 4; ByteBuffer[] bb_record_type2_fixed_part1 = new ByteBuffer[rt2_1st_4_units]; int[] recordType2FixedPart1 = new int[rt2_1st_4_units]; for (int i = 0; i < rt2_1st_4_units; i++) { bb_record_type2_fixed_part1[i] = ByteBuffer.wrap(recordType2Fixed, offset, LENGTH_SAV_INT_BLOCK); offset += LENGTH_SAV_INT_BLOCK; if (isLittleEndian) { bb_record_type2_fixed_part1[i].order(ByteOrder.LITTLE_ENDIAN); } recordType2FixedPart1[i] = bb_record_type2_fixed_part1[i].getInt(); } ///dbgLog.fine("recordType2FixedPart="+ /// ReflectionToStringBuilder.toString(recordType2FixedPart1, ToStringStyle.MULTI_LINE_STYLE)); // 1st ([0]) element must be 2 otherwise no longer Record Type 2 if (recordType2FixedPart1[0] != 2) { dbgLog.warning(j + "-th RT header value is no longet RT2! " + recordType2FixedPart1[0]); break; } dbgLog.fine("variable type[must be 2]=" + recordType2FixedPart1[0]); // 2.3 variable name: 8 byte(space[x20]-padded) // This field is located at the very end of the 32 byte // fixed-size RT2 header (bytes 24-31). // We are processing it now, so that // we can make the decision on whether this variable is part // of a compound variable: String RawVariableName = getNullStrippedString(new String( Arrays.copyOfRange(recordType2Fixed, 24, (24 + LENGTH_VARIABLE_NAME)), defaultCharSet)); //offset +=LENGTH_VARIABLE_NAME; String variableName = null; if (RawVariableName.indexOf(' ') >= 0) { variableName = RawVariableName.substring(0, RawVariableName.indexOf(' ')); } else { variableName = RawVariableName; } // 2nd ([1]) element: numeric variable = 0 :for string variable // this block indicates its datum-length, i.e, >0 ; // if -1, this RT2 unit is a non-1st RT2 unit for a string variable // whose value is longer than 8 character. boolean isNumericVariable = false; dbgLog.fine("variable type(0: numeric; > 0: String;-1 continue )=" + recordType2FixedPart1[1]); //OBSwiseTypelList.add(recordType2FixedPart1[1]); int HowManyRt2Units = 1; if (recordType2FixedPart1[1] == -1) { dbgLog.fine("this RT2 is an 8 bit continuation chunk of an earlier string variable"); if (obs255) { if (obsSeqNumber < 30) { OBSwiseTypelList.add(recordType2FixedPart1[1]); obsSeqNumber++; } else { OBSwiseTypelList.add(-2); obs255 = false; obsSeqNumber = 0; } } else { OBSwiseTypelList.add(recordType2FixedPart1[1]); } obsNonVariableBlockSet.add(j); continue; } else if (recordType2FixedPart1[1] == 0) { // This is a numeric variable extendedVariableMode = false; // And as such, it cannot be an extension of a // previous, long string variable. OBSwiseTypelList.add(recordType2FixedPart1[1]); variableCounter++; isNumericVariable = true; variableTypelList.add(recordType2FixedPart1[1]); } else if (recordType2FixedPart1[1] > 0) { // This looks like a regular string variable. However, // it may still be a part of a compound variable // (a String > 255 bytes that was split into 255 byte // chunks, stored as individual String variables). if (recordType2FixedPart1[1] == 255) { obs255 = true; } if (lastVariableIsExtendable) { String varNameBase = null; if (lastVariableName.length() > 5) { varNameBase = lastVariableName.substring(0, 5); } else { varNameBase = lastVariableName; } if (extendedVariableMode) { if (variableNameIsAnIncrement(varNameBase, lastExtendedVariable, variableName)) { OBSwiseTypelList.add(-1); lastExtendedVariable = variableName; // OK, we stay in the "extended variable" mode; // but we can't move on to the next OBS (hence the commented out // "continue" below: //continue; // see the next comment below for the explanation. // // Should we also set "extendable" flag to false at this point // if it's shorter than 255 bytes, i.e. the last extended chunk? } else { extendedVariableMode = false; } } else { if (variableNameIsAnIncrement(varNameBase, variableName)) { OBSwiseTypelList.add(-1); extendedVariableMode = true; dbgLog.fine("RT2: in extended variable mode; variable " + variableName); lastExtendedVariable = variableName; // Before we move on to the next OBS unit, we need to check // if this current extended variable has its own label specified; // If so, we need to determine its length, then read and skip // that many bytes. // Hence the commented out "continue" below: //continue; } } } if (!extendedVariableMode) { // OK, this is a "real" // string variable, and not a continuation chunk of a compound // string. OBSwiseTypelList.add(recordType2FixedPart1[1]); variableCounter++; if (recordType2FixedPart1[1] == 255) { // This variable is 255 bytes long, i.e. this is // either the single "atomic" variable of the // max allowed size, or it's a 255 byte segment // of a compound variable. So we will check // the next variable and see if it is the continuation // of this one. lastVariableIsExtendable = true; } else { lastVariableIsExtendable = false; } if (recordType2FixedPart1[1] % LENGTH_SAV_OBS_BLOCK == 0) { HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK; } else { HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK + 1; } variableTypelList.add(recordType2FixedPart1[1]); } } if (!extendedVariableMode) { // Again, we only want to do the following steps for the "real" // variables, not the chunks of split mega-variables: dbgLog.fine("RT2: HowManyRt2Units for this variable=" + HowManyRt2Units); lastVariableName = variableName; // caseWeightVariableOBSIndex starts from 1: 0 is used for does-not-exist cases if (j == (caseWeightVariableOBSIndex - 1)) { caseWeightVariableName = variableName; // TODO: do we need this "index"? -- 4.0 alpha caseWeightVariableIndex = variableCounter; ///smd.setCaseWeightVariableName(caseWeightVariableName); ///smd.getFileInformation().put("caseWeightVariableIndex", caseWeightVariableIndex); } OBSIndexToVariableName.put(j, variableName); //dbgLog.fine("\nvariable name="+variableName+"<-"); dbgLog.fine("RT2: " + j + "-th variable name=" + variableName + "<-"); dbgLog.fine("RT2: raw variable: " + RawVariableName); variableNameList.add(variableName); } // 3rd ([2]) element: = 1 variable-label block follows; 0 = no label // dbgLog.fine("RT: variable label follows?(1:yes; 0: no)=" + recordType2FixedPart1[2]); boolean hasVariableLabel = recordType2FixedPart1[2] == 1 ? true : false; if ((recordType2FixedPart1[2] != 0) && (recordType2FixedPart1[2] != 1)) { throw new IOException("RT2: reading error: value is neither 0 or 1" + recordType2FixedPart1[2]); } // 2.4 [optional]The length of a variable label followed: 4-byte int // 3rd element of 2.1 indicates whether this field exists // *** warning: The label block is padded to a multiple of the 4-byte // NOT the raw integer value of this 4-byte block if (hasVariableLabel) { byte[] length_variable_label = new byte[4]; int nbytes_2_4 = stream.read(length_variable_label); if (nbytes_2_4 == 0) { throw new IOException("RT 2: error reading recordType2.4: no bytes read!"); } else { dbgLog.fine("nbytes_2_4=" + nbytes_2_4); } ByteBuffer bb_length_variable_label = ByteBuffer.wrap(length_variable_label, 0, LENGTH_VARIABLE_LABEL); if (isLittleEndian) { bb_length_variable_label.order(ByteOrder.LITTLE_ENDIAN); } int rawVariableLabelLength = bb_length_variable_label.getInt(); dbgLog.fine("rawVariableLabelLength=" + rawVariableLabelLength); int variableLabelLength = getSAVintAdjustedBlockLength(rawVariableLabelLength); dbgLog.fine("RT2: variableLabelLength=" + variableLabelLength); // 2.5 [optional]variable label whose length is found at 2.4 String variableLabel = ""; if (rawVariableLabelLength > 0) { byte[] variable_label = new byte[variableLabelLength]; int nbytes_2_5 = stream.read(variable_label); if (nbytes_2_5 == 0) { throw new IOException("RT 2: error reading recordType2.5: " + variableLabelLength + " bytes requested, no bytes read!"); } else { dbgLog.fine("nbytes_2_5=" + nbytes_2_5); } variableLabel = getNullStrippedString(new String( Arrays.copyOfRange(variable_label, 0, rawVariableLabelLength), defaultCharSet)); dbgLog.fine("RT2: variableLabel=" + variableLabel + "<-"); dbgLog.fine(variableName + " => " + variableLabel); } else { dbgLog.fine("RT2: defaulting to empty variable label."); } if (!extendedVariableMode) { // We only have any use for this label if it's a "real" variable. // Thinking about it, it doesn't make much sense for the "fake" // variables that are actually chunks of large strings to store // their own labels. But in some files they do. Then failing to read // the bytes would result in getting out of sync with the RT record // borders. So we always read the bytes, but only use them for // the real variable entries. /*String variableLabel = new String(Arrays.copyOfRange(variable_label, 0, rawVariableLabelLength),"US-ASCII");*/ variableLabelMap.put(variableName, variableLabel); } } if (extendedVariableMode) { // there's nothing else left for us to do in this iteration of the loop. // Once again, this was not a real variable, but a dummy variable entry // created for a chunk of a string variable longer than 255 bytes -- // that's how SPSS stores them. continue; } // 4th ([3]) element: Missing value type code // 0[none], 1, 2, 3 [point-type],-2[range], -3 [range type+ point] dbgLog.fine("RT: missing value unit follows?(if 0, none)=" + recordType2FixedPart1[3]); boolean hasMissingValues = (validMissingValueCodeSet.contains(recordType2FixedPart1[3]) && (recordType2FixedPart1[3] != 0)) ? true : false; InvalidData invalidDataInfo = null; if (recordType2FixedPart1[3] != 0) { invalidDataInfo = new InvalidData(recordType2FixedPart1[3]); dbgLog.fine("RT: missing value type=" + invalidDataInfo.getType()); } // 2.2: print/write formats: 4-byte each = 8 bytes byte[] printFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_PRINT_FORMAT_CODE); dbgLog.fine("printFrmt=" + new String(Hex.encodeHex(printFormt))); offset += LENGTH_PRINT_FORMAT_CODE; int formatCode = isLittleEndian ? printFormt[2] : printFormt[1]; int formatWidth = isLittleEndian ? printFormt[1] : printFormt[2]; // TODO: // What should we be doing with these "format decimal positions" // in 4.0? // -- L.A. 4.0 alpha int formatDecimalPointPosition = isLittleEndian ? printFormt[0] : printFormt[3]; dbgLog.fine("RT2: format code{5=F, 1=A[String]}=" + formatCode); formatDecimalPointPositionList.add(formatDecimalPointPosition); if (!SPSSConstants.FORMAT_CODE_TABLE_SAV.containsKey(formatCode)) { throw new IOException("Unknown format code was found = " + formatCode); } else { printFormatList.add(formatCode); } byte[] writeFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_WRITE_FORMAT_CODE); dbgLog.fine("RT2: writeFrmt=" + new String(Hex.encodeHex(writeFormt))); if (writeFormt[3] != 0x00) { dbgLog.fine("byte-order(write format): reversal required"); } offset += LENGTH_WRITE_FORMAT_CODE; if (!SPSSConstants.ORDINARY_FORMAT_CODE_SET.contains(formatCode)) { StringBuilder sb = new StringBuilder( SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode) + formatWidth); if (formatDecimalPointPosition > 0) { sb.append("." + formatDecimalPointPosition); } dbgLog.fine("formattable[i] = " + variableName + " -> " + sb.toString()); printFormatNameTable.put(variableName, sb.toString()); } printFormatTable.put(variableName, SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode)); // 2.6 [optional] missing values:4-byte each if exists // 4th element of 2.1 indicates the structure of this sub-field // Should we perhaps check for this for the "fake" variables too? // if (hasMissingValues) { dbgLog.fine("RT2: decoding missing value: type=" + recordType2FixedPart1[3]); int howManyMissingValueUnits = missingValueCodeUnits.get(recordType2FixedPart1[3]); //int howManyMissingValueUnits = recordType2FixedPart1[3] > 0 ? recordType2FixedPart1[3] : 0; dbgLog.fine("RT2: howManyMissingValueUnits=" + howManyMissingValueUnits); byte[] missing_value_code_units = new byte[LENGTH_SAV_OBS_BLOCK * howManyMissingValueUnits]; int nbytes_2_6 = stream.read(missing_value_code_units); if (nbytes_2_6 == 0) { throw new IOException("RT 2: reading recordType2.6: no byte was read"); } else { dbgLog.fine("nbytes_2_6=" + nbytes_2_6); } //printHexDump(missing_value_code_units, "missing value"); if (isNumericVariable) { double[] missingValues = new double[howManyMissingValueUnits]; //List<String> mvp = new ArrayList<String>(); List<String> mv = new ArrayList<String>(); ByteBuffer[] bb_missig_value_code = new ByteBuffer[howManyMissingValueUnits]; int offset_start = 0; for (int i = 0; i < howManyMissingValueUnits; i++) { bb_missig_value_code[i] = ByteBuffer.wrap(missing_value_code_units, offset_start, LENGTH_SAV_OBS_BLOCK); offset_start += LENGTH_SAV_OBS_BLOCK; if (isLittleEndian) { bb_missig_value_code[i].order(ByteOrder.LITTLE_ENDIAN); } ByteBuffer temp = bb_missig_value_code[i].duplicate(); missingValues[i] = bb_missig_value_code[i].getDouble(); if (Double.toHexString(missingValues[i]).equals("-0x1.ffffffffffffep1023")) { dbgLog.fine("1st value is LOWEST"); mv.add(Double.toHexString(missingValues[i])); } else if (Double.valueOf(missingValues[i]).equals(Double.MAX_VALUE)) { dbgLog.fine("2nd value is HIGHEST"); mv.add(Double.toHexString(missingValues[i])); } else { mv.add(doubleNumberFormatter.format(missingValues[i])); } dbgLog.fine(i + "-th missing value=" + Double.toHexString(missingValues[i])); } dbgLog.fine("variableName=" + variableName); if (recordType2FixedPart1[3] > 0) { // point cases only dbgLog.fine("mv(>0)=" + mv); missingValueTable.put(variableName, mv); invalidDataInfo.setInvalidValues(mv); } else if (recordType2FixedPart1[3] == -2) { dbgLog.fine("mv(-2)=" + mv); // range invalidDataInfo.setInvalidRange(mv); } else if (recordType2FixedPart1[3] == -3) { // mixed case dbgLog.fine("mv(-3)=" + mv); invalidDataInfo.setInvalidRange(mv.subList(0, 2)); invalidDataInfo.setInvalidValues(mv.subList(2, 3)); missingValueTable.put(variableName, mv.subList(2, 3)); } dbgLog.fine("missing value=" + StringUtils.join(missingValueTable.get(variableName), "|")); dbgLog.fine("invalidDataInfo(Numeric):\n" + invalidDataInfo); invalidDataTable.put(variableName, invalidDataInfo); } else { // string variable case String[] missingValues = new String[howManyMissingValueUnits]; List<String> mv = new ArrayList<String>(); int offset_start = 0; int offset_end = LENGTH_SAV_OBS_BLOCK; for (int i = 0; i < howManyMissingValueUnits; i++) { missingValues[i] = StringUtils.stripEnd(new String( Arrays.copyOfRange(missing_value_code_units, offset_start, offset_end), defaultCharSet), " "); dbgLog.fine("missing value=" + missingValues[i] + "<-"); offset_start = offset_end; offset_end += LENGTH_SAV_OBS_BLOCK; mv.add(missingValues[i]); } invalidDataInfo.setInvalidValues(mv); missingValueTable.put(variableName, mv); invalidDataTable.put(variableName, invalidDataInfo); dbgLog.fine( "missing value(str)=" + StringUtils.join(missingValueTable.get(variableName), "|")); dbgLog.fine("invalidDataInfo(String):\n" + invalidDataInfo); } // string case dbgLog.fine("invalidDataTable:\n" + invalidDataTable); } // if msv } catch (IOException ex) { //ex.printStackTrace(); throw ex; } catch (Exception ex) { ex.printStackTrace(); // should we be throwing some exception here? } } // j-loop if (j != OBSUnitsPerCase) { dbgLog.fine("RT2: attention! didn't reach the end of the OBS list!"); throw new IOException("RT2: didn't reach the end of the OBS list!"); } dbgLog.fine("RT2 metadata-related exit-chores"); ///smd.getFileInformation().put("varQnty", variableCounter); dataTable.setVarQuantity(new Long(variableCounter)); dbgLog.fine("RT2: varQnty=" + variableCounter); // 4.0 Initialize variables: List<DataVariable> variableList = new ArrayList<DataVariable>(); for (int i = 0; i < variableCounter; i++) { DataVariable dv = new DataVariable(); String varName = variableNameList.get(i); dbgLog.fine("name: " + varName); dv.setName(varName); String varLabel = variableLabelMap.get(varName); if (varLabel != null && varLabel.length() > 255) { // TODO: // variable labels will be changed into type 'TEXT' in the // database - this will eliminate the 255 char. limit. // -- L.A. 4.0 beta11 dbgLog.fine("Have to truncate label: " + varLabel); varLabel = varLabel.substring(0, 255); } dbgLog.fine("label: " + varLabel); dv.setLabel(varLabel); dv.setInvalidRanges(new ArrayList<VariableRange>()); dv.setSummaryStatistics(new ArrayList<SummaryStatistic>()); dv.setUnf("UNF:6:"); dv.setCategories(new ArrayList<VariableCategory>()); variableList.add(dv); dv.setFileOrder(i); dv.setDataTable(dataTable); } dataTable.setDataVariables(variableList); ///smd.setVariableName(variableNameList.toArray(new String[variableNameList.size()])); ///smd.setVariableLabel(variableLabelMap); // TODO: // figure out what to do with the missing value table! // -- 4.0 alpha // well, they were used to generate merged summary statistics for // the variable. So need to verify what the DDI import was doing // with them and replicate the same in 4.0. // (add appropriate value labels?) ///TODO: 4.0 smd.setMissingValueTable(missingValueTable); ///smd.getFileInformation().put("caseWeightVariableName", caseWeightVariableName); dbgLog.fine("sumstat:long case=" + Arrays.deepToString(variableTypelList.toArray())); dbgLog.fine("RT2: OBSwiseTypelList=" + OBSwiseTypelList); dbgLog.fine("decodeRecordType2(): end"); }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.sav.SAVFileReader.java
void decodeRecordType2(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordType2(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }// w ww .j a va2 s . c om Map<String, String> variableLabelMap = new LinkedHashMap<String, String>(); Map<String, List<String>> missingValueTable = new LinkedHashMap<String, List<String>>(); List<Integer> printFormatList = new ArrayList<Integer>(); String caseWeightVariableName = null; int caseWeightVariableIndex = 0; boolean lastVariableIsExtendable = false; boolean extendedVariableMode = false; boolean obs255 = false; String lastVariableName = null; String lastExtendedVariable = null; // this field repeats as many as the number of variables in // this sav file // (note that the above statement is not technically correct, this // record repeats not just for every variable in the file, but for // every OBS (8 byte unit); i.e., if a string is split into multiple // OBS units, each one will have its own RT2 record -- L.A.). // Each field constists of a fixed (32-byte) segment and // then a few variable segments: // if the variable has a label (3rd INT4 set to 1), then there's 4 more // bytes specifying the length of the label, and then that many bytes // holding the label itself (no more than 256). // Then if there are optional missing value units (4th INT4 set to 1) // there will be 3 more OBS units attached = 24 extra bytes. int variableCounter = 0; int obsSeqNumber = 0; int j; dbgLog.fine("RT2: Reading " + OBSUnitsPerCase + " OBS units."); for (j = 0; j < OBSUnitsPerCase; j++) { dbgLog.fine("RT2: \n\n+++++++++++ " + j + "-th RT2 unit is to be decoded +++++++++++"); // 2.0: read the fixed[=non-optional] 32-byte segment byte[] recordType2Fixed = new byte[LENGTH_RECORDTYPE2_FIXED]; try { int nbytes = stream.read(recordType2Fixed, 0, LENGTH_RECORDTYPE2_FIXED); //printHexDump(recordType2Fixed, "recordType2 part 1"); if (nbytes == 0) { throw new IOException("reading recordType2: no bytes read!"); } int offset = 0; // 2.1: create int-view of the bytebuffer for the first 16-byte segment int rt2_1st_4_units = 4; ByteBuffer[] bb_record_type2_fixed_part1 = new ByteBuffer[rt2_1st_4_units]; int[] recordType2FixedPart1 = new int[rt2_1st_4_units]; for (int i = 0; i < rt2_1st_4_units; i++) { bb_record_type2_fixed_part1[i] = ByteBuffer.wrap(recordType2Fixed, offset, LENGTH_SAV_INT_BLOCK); offset += LENGTH_SAV_INT_BLOCK; if (isLittleEndian) { bb_record_type2_fixed_part1[i].order(ByteOrder.LITTLE_ENDIAN); } recordType2FixedPart1[i] = bb_record_type2_fixed_part1[i].getInt(); } dbgLog.fine("recordType2FixedPart=" + ReflectionToStringBuilder.toString(recordType2FixedPart1, ToStringStyle.MULTI_LINE_STYLE)); // 1st ([0]) element must be 2 otherwise no longer Record Type 2 if (recordType2FixedPart1[0] != 2) { dbgLog.info(j + "-th RT header value is no longet RT2! " + recordType2FixedPart1[0]); break; //throw new IOException("RT2 reading error: The current position is no longer Record Type 2"); } dbgLog.fine("variable type[must be 2]=" + recordType2FixedPart1[0]); // 2.3 variable name: 8 byte(space[x20]-padded) // This field is located at the very end of the 32 byte // fixed-size RT2 header (bytes 24-31). // We are processing it now, so that // we can make the decision on whether this variable is part // of a compound variable: String RawVariableName = new String( Arrays.copyOfRange(recordType2Fixed, 24, (24 + LENGTH_VARIABLE_NAME)), defaultCharSet); //offset +=LENGTH_VARIABLE_NAME; String variableName = null; if (RawVariableName.indexOf(' ') >= 0) { variableName = RawVariableName.substring(0, RawVariableName.indexOf(' ')); } else { variableName = RawVariableName; } // 2nd ([1]) element: numeric variable = 0 :for string variable // this block indicates its datum-length, i.e, >0 ; // if -1, this RT2 unit is a non-1st RT2 unit for a string variable // whose value is longer than 8 character. boolean isNumericVariable = false; dbgLog.fine("variable type(0: numeric; > 0: String;-1 continue )=" + recordType2FixedPart1[1]); //OBSwiseTypelList.add(recordType2FixedPart1[1]); int HowManyRt2Units = 1; if (recordType2FixedPart1[1] == -1) { dbgLog.fine("this RT2 is an 8 bit continuation chunk of an earlier string variable"); if (obs255) { if (obsSeqNumber < 30) { OBSwiseTypelList.add(recordType2FixedPart1[1]); obsSeqNumber++; } else { OBSwiseTypelList.add(-2); obs255 = false; obsSeqNumber = 0; } } else { OBSwiseTypelList.add(recordType2FixedPart1[1]); } obsNonVariableBlockSet.add(j); continue; } else if (recordType2FixedPart1[1] == 0) { // This is a numeric variable extendedVariableMode = false; // And as such, it cannot be an extension of a // previous, long string variable. OBSwiseTypelList.add(recordType2FixedPart1[1]); variableCounter++; isNumericVariable = true; variableTypelList.add(recordType2FixedPart1[1]); } else if (recordType2FixedPart1[1] > 0) { // This looks like a regular string variable. However, // it may still be a part of a compound variable // (a String > 255 bytes that was split into 255 byte // chunks, stored as individual String variables). if (recordType2FixedPart1[1] == 255) { obs255 = true; } if (lastVariableIsExtendable) { String varNameBase = null; if (lastVariableName.length() > 5) { varNameBase = lastVariableName.substring(0, 5); } else { varNameBase = lastVariableName; } if (extendedVariableMode) { if (variableNameIsAnIncrement(varNameBase, lastExtendedVariable, variableName)) { OBSwiseTypelList.add(-1); lastExtendedVariable = variableName; // OK, we stay in the "extended variable" mode; // but we can't move on to the next OBS (hence the commented out // "continue" below: //continue; // see the next comment below for the explanation. // // Should we also set "extendable" flag to false at this point // if it's shorter than 255 bytes, i.e. the last extended chunk? } else { extendedVariableMode = false; } } else { if (variableNameIsAnIncrement(varNameBase, variableName)) { OBSwiseTypelList.add(-1); extendedVariableMode = true; dbgLog.fine("RT2: in extended variable mode; variable " + variableName); lastExtendedVariable = variableName; // Before we move on to the next OBS unit, we need to check // if this current extended variable has its own label specified; // If so, we need to determine its length, then read and skip // that many bytes. // Hence the commented out "continue" below: //continue; } } } if (!extendedVariableMode) { // OK, this is a "real" // string variable, and not a continuation chunk of a compound // string. OBSwiseTypelList.add(recordType2FixedPart1[1]); variableCounter++; if (recordType2FixedPart1[1] == 255) { // This variable is 255 bytes long, i.e. this is // either the single "atomic" variable of the // max allowed size, or it's a 255 byte segment // of a compound variable. So we will check // the next variable and see if it is the continuation // of this one. lastVariableIsExtendable = true; } else { lastVariableIsExtendable = false; } if (recordType2FixedPart1[1] % LENGTH_SAV_OBS_BLOCK == 0) { HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK; } else { HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK + 1; } variableTypelList.add(recordType2FixedPart1[1]); } } if (!extendedVariableMode) { // Again, we only want to do the following steps for the "real" // variables, not the chunks of split mega-variables: dbgLog.fine("RT2: HowManyRt2Units for this variable=" + HowManyRt2Units); lastVariableName = variableName; // caseWeightVariableOBSIndex starts from 1: 0 is used for does-not-exist cases if (j == (caseWeightVariableOBSIndex - 1)) { caseWeightVariableName = variableName; caseWeightVariableIndex = variableCounter; smd.setCaseWeightVariableName(caseWeightVariableName); smd.getFileInformation().put("caseWeightVariableIndex", caseWeightVariableIndex); } OBSIndexToVariableName.put(j, variableName); //dbgLog.fine("\nvariable name="+variableName+"<-"); dbgLog.fine("RT2: " + j + "-th variable name=" + variableName + "<-"); dbgLog.fine("RT2: raw variable: " + RawVariableName); variableNameList.add(variableName); } // 3rd ([2]) element: = 1 variable-label block follows; 0 = no label // dbgLog.fine("RT: variable label follows?(1:yes; 0: no)=" + recordType2FixedPart1[2]); boolean hasVariableLabel = recordType2FixedPart1[2] == 1 ? true : false; if ((recordType2FixedPart1[2] != 0) && (recordType2FixedPart1[2] != 1)) { throw new IOException("RT2: reading error: value is neither 0 or 1" + recordType2FixedPart1[2]); } // 2.4 [optional]The length of a variable label followed: 4-byte int // 3rd element of 2.1 indicates whether this field exists // *** warning: The label block is padded to a multiple of the 4-byte // NOT the raw integer value of this 4-byte block if (hasVariableLabel) { byte[] length_variable_label = new byte[4]; int nbytes_2_4 = stream.read(length_variable_label); if (nbytes_2_4 == 0) { throw new IOException("RT 2: error reading recordType2.4: no bytes read!"); } else { dbgLog.fine("nbytes_2_4=" + nbytes_2_4); } ByteBuffer bb_length_variable_label = ByteBuffer.wrap(length_variable_label, 0, LENGTH_VARIABLE_LABEL); if (isLittleEndian) { bb_length_variable_label.order(ByteOrder.LITTLE_ENDIAN); } int rawVariableLabelLength = bb_length_variable_label.getInt(); dbgLog.fine("rawVariableLabelLength=" + rawVariableLabelLength); int variableLabelLength = getSAVintAdjustedBlockLength(rawVariableLabelLength); dbgLog.fine("RT2: variableLabelLength=" + variableLabelLength); // 2.5 [optional]variable label whose length is found at 2.4 String variableLabel = ""; if (rawVariableLabelLength > 0) { byte[] variable_label = new byte[variableLabelLength]; int nbytes_2_5 = stream.read(variable_label); if (nbytes_2_5 == 0) { throw new IOException("RT 2: error reading recordType2.5: " + variableLabelLength + " bytes requested, no bytes read!"); } else { dbgLog.fine("nbytes_2_5=" + nbytes_2_5); } variableLabel = new String(Arrays.copyOfRange(variable_label, 0, rawVariableLabelLength), defaultCharSet); dbgLog.fine("RT2: variableLabel=" + variableLabel + "<-"); dbgLog.info(variableName + " => " + variableLabel); } else { dbgLog.fine("RT2: defaulting to empty variable label."); } if (!extendedVariableMode) { // We only have any use for this label if it's a "real" variable. // Thinking about it, it doesn't make much sense for the "fake" // variables that are actually chunks of large strings to store // their own labels. But in some files they do. Then failing to read // the bytes would result in getting out of sync with the RT record // borders. So we always read the bytes, but only use them for // the real variable entries. /*String variableLabel = new String(Arrays.copyOfRange(variable_label, 0, rawVariableLabelLength),"US-ASCII");*/ variableLabelMap.put(variableName, variableLabel); } } if (extendedVariableMode) { // there's nothing else left for us to do in this iteration of the loop. // Once again, this was not a real variable, but a dummy variable entry // created for a chunk of a string variable longer than 255 bytes -- // that's how SPSS stores them. continue; } // 4th ([3]) element: Missing value type code // 0[none], 1, 2, 3 [point-type],-2[range], -3 [range type+ point] dbgLog.fine("RT: missing value unit follows?(if 0, none)=" + recordType2FixedPart1[3]); boolean hasMissingValues = (validMissingValueCodeSet.contains(recordType2FixedPart1[3]) && (recordType2FixedPart1[3] != 0)) ? true : false; InvalidData invalidDataInfo = null; if (recordType2FixedPart1[3] != 0) { invalidDataInfo = new InvalidData(recordType2FixedPart1[3]); dbgLog.fine("RT: missing value type=" + invalidDataInfo.getType()); } // 2.2: print/write formats: 4-byte each = 8 bytes byte[] printFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_PRINT_FORMAT_CODE); dbgLog.fine("printFrmt=" + new String(Hex.encodeHex(printFormt))); offset += LENGTH_PRINT_FORMAT_CODE; int formatCode = isLittleEndian ? printFormt[2] : printFormt[1]; int formatWidth = isLittleEndian ? printFormt[1] : printFormt[2]; int formatDecimalPointPosition = isLittleEndian ? printFormt[0] : printFormt[3]; dbgLog.fine("RT2: format code{5=F, 1=A[String]}=" + formatCode); formatDecimalPointPositionList.add(formatDecimalPointPosition); if (!SPSSConstants.FORMAT_CODE_TABLE_SAV.containsKey(formatCode)) { throw new IOException("Unknown format code was found = " + formatCode); } else { printFormatList.add(formatCode); } byte[] writeFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_WRITE_FORMAT_CODE); dbgLog.fine("RT2: writeFrmt=" + new String(Hex.encodeHex(writeFormt))); if (writeFormt[3] != 0x00) { dbgLog.fine("byte-order(write format): reversal required"); } offset += LENGTH_WRITE_FORMAT_CODE; if (!SPSSConstants.ORDINARY_FORMAT_CODE_SET.contains(formatCode)) { StringBuilder sb = new StringBuilder( SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode) + formatWidth); if (formatDecimalPointPosition > 0) { sb.append("." + formatDecimalPointPosition); } dbgLog.info("formattable[i] = " + variableName + " -> " + sb.toString()); printFormatNameTable.put(variableName, sb.toString()); } printFormatTable.put(variableName, SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode)); // 2.6 [optional] missing values:4-byte each if exists // 4th element of 2.1 indicates the structure of this sub-field // Should we perhaps check for this for the "fake" variables too? // if (hasMissingValues) { dbgLog.fine("RT2: decoding missing value: type=" + recordType2FixedPart1[3]); int howManyMissingValueUnits = missingValueCodeUnits.get(recordType2FixedPart1[3]); //int howManyMissingValueUnits = recordType2FixedPart1[3] > 0 ? recordType2FixedPart1[3] : 0; dbgLog.fine("RT2: howManyMissingValueUnits=" + howManyMissingValueUnits); byte[] missing_value_code_units = new byte[LENGTH_SAV_OBS_BLOCK * howManyMissingValueUnits]; int nbytes_2_6 = stream.read(missing_value_code_units); if (nbytes_2_6 == 0) { throw new IOException("RT 2: reading recordType2.6: no byte was read"); } else { dbgLog.fine("nbytes_2_6=" + nbytes_2_6); } //printHexDump(missing_value_code_units, "missing value"); if (isNumericVariable) { double[] missingValues = new double[howManyMissingValueUnits]; //List<String> mvp = new ArrayList<String>(); List<String> mv = new ArrayList<String>(); ByteBuffer[] bb_missig_value_code = new ByteBuffer[howManyMissingValueUnits]; int offset_start = 0; for (int i = 0; i < howManyMissingValueUnits; i++) { bb_missig_value_code[i] = ByteBuffer.wrap(missing_value_code_units, offset_start, LENGTH_SAV_OBS_BLOCK); offset_start += LENGTH_SAV_OBS_BLOCK; if (isLittleEndian) { bb_missig_value_code[i].order(ByteOrder.LITTLE_ENDIAN); } ByteBuffer temp = bb_missig_value_code[i].duplicate(); missingValues[i] = bb_missig_value_code[i].getDouble(); if (Double.toHexString(missingValues[i]).equals("-0x1.ffffffffffffep1023")) { dbgLog.fine("1st value is LOWEST"); mv.add(Double.toHexString(missingValues[i])); } else if (Double.valueOf(missingValues[i]).equals(Double.MAX_VALUE)) { dbgLog.fine("2nd value is HIGHEST"); mv.add(Double.toHexString(missingValues[i])); } else { mv.add(doubleNumberFormatter.format(missingValues[i])); } dbgLog.fine(i + "-th missing value=" + Double.toHexString(missingValues[i])); } dbgLog.fine("variableName=" + variableName); if (recordType2FixedPart1[3] > 0) { // point cases only dbgLog.fine("mv(>0)=" + mv); missingValueTable.put(variableName, mv); invalidDataInfo.setInvalidValues(mv); } else if (recordType2FixedPart1[3] == -2) { dbgLog.fine("mv(-2)=" + mv); // range invalidDataInfo.setInvalidRange(mv); } else if (recordType2FixedPart1[3] == -3) { // mixed case dbgLog.fine("mv(-3)=" + mv); invalidDataInfo.setInvalidRange(mv.subList(0, 2)); invalidDataInfo.setInvalidValues(mv.subList(2, 3)); missingValueTable.put(variableName, mv.subList(2, 3)); } dbgLog.fine("missing value=" + StringUtils.join(missingValueTable.get(variableName), "|")); dbgLog.fine("invalidDataInfo(Numeric):\n" + invalidDataInfo); invalidDataTable.put(variableName, invalidDataInfo); } else { // string variable case String[] missingValues = new String[howManyMissingValueUnits]; List<String> mv = new ArrayList<String>(); int offset_start = 0; int offset_end = LENGTH_SAV_OBS_BLOCK; for (int i = 0; i < howManyMissingValueUnits; i++) { missingValues[i] = StringUtils.stripEnd(new String( Arrays.copyOfRange(missing_value_code_units, offset_start, offset_end), defaultCharSet), " "); dbgLog.fine("missing value=" + missingValues[i] + "<-"); offset_start = offset_end; offset_end += LENGTH_SAV_OBS_BLOCK; mv.add(missingValues[i]); } invalidDataInfo.setInvalidValues(mv); missingValueTable.put(variableName, mv); invalidDataTable.put(variableName, invalidDataInfo); dbgLog.fine( "missing value(str)=" + StringUtils.join(missingValueTable.get(variableName), "|")); dbgLog.fine("invalidDataInfo(String):\n" + invalidDataInfo); } // string case dbgLog.fine("invalidDataTable:\n" + invalidDataTable); } // if msv } catch (IOException ex) { //ex.printStackTrace(); throw ex; } catch (Exception ex) { ex.printStackTrace(); // should we be throwing some exception here? } } // j-loop if (j == OBSUnitsPerCase) { dbgLog.fine("RT2 metadata-related exit-chores"); smd.getFileInformation().put("varQnty", variableCounter); varQnty = variableCounter; dbgLog.fine("RT2: varQnty=" + varQnty); smd.setVariableName(variableNameList.toArray(new String[variableNameList.size()])); smd.setVariableLabel(variableLabelMap); smd.setMissingValueTable(missingValueTable); smd.getFileInformation().put("caseWeightVariableName", caseWeightVariableName); dbgLog.info("sumstat:long case=" + Arrays.deepToString(variableTypelList.toArray())); smd.setVariableFormat(printFormatList); smd.setVariableFormatName(printFormatNameTable); dbgLog.info("<<<<<<"); dbgLog.info("printFormatList = " + printFormatList); dbgLog.info("printFormatNameTable = " + printFormatNameTable); // dbgLog.info("formatCategoryTable = " + formatCategoryTable); dbgLog.info(">>>>>>"); dbgLog.fine("RT2: OBSwiseTypelList=" + OBSwiseTypelList); // variableType is determined after the valueTable is finalized } else { dbgLog.info("RT2: attention! didn't reach the end of the OBS list!"); throw new IOException("RT2: didn't reach the end of the OBS list!"); } dbgLog.fine("***** decodeRecordType2(): end *****"); }
From source file:org.alfresco.repo.search.impl.lucene.index.IndexInfo.java
private void setStatusFromFile(FileChannel channel) throws IOException { if (channel.size() > 0) { channel.position(0);//from w ww.j a v a2 s.c o m ByteBuffer buffer; if (useNIOMemoryMapping) { MappedByteBuffer mbb = channel.map(MapMode.READ_ONLY, 0, channel.size()); mbb.load(); buffer = mbb; } else { buffer = ByteBuffer.wrap(new byte[(int) channel.size()]); channel.read(buffer); buffer.position(0); } buffer.position(0); long onDiskVersion = buffer.getLong(); if (version != onDiskVersion) { CRC32 crc32 = new CRC32(); crc32.update((int) (onDiskVersion >>> 32) & 0xFFFFFFFF); crc32.update((int) (onDiskVersion >>> 0) & 0xFFFFFFFF); int size = buffer.getInt(); crc32.update(size); LinkedHashMap<String, IndexEntry> newIndexEntries = new LinkedHashMap<String, IndexEntry>(); // Not all state is saved some is specific to this index so we // need to add the transient stuff. // Until things are committed they are not shared unless it is // prepared for (int i = 0; i < size; i++) { String indexTypeString = readString(buffer, crc32); IndexType indexType; try { indexType = IndexType.valueOf(indexTypeString); } catch (IllegalArgumentException e) { throw new IOException("Invalid type " + indexTypeString); } String name = readString(buffer, crc32); String parentName = readString(buffer, crc32); String txStatus = readString(buffer, crc32); TransactionStatus status; try { status = TransactionStatus.valueOf(txStatus); } catch (IllegalArgumentException e) { throw new IOException("Invalid status " + txStatus); } String mergeId = readString(buffer, crc32); long documentCount = buffer.getLong(); crc32.update((int) (documentCount >>> 32) & 0xFFFFFFFF); crc32.update((int) (documentCount >>> 0) & 0xFFFFFFFF); long deletions = buffer.getLong(); crc32.update((int) (deletions >>> 32) & 0xFFFFFFFF); crc32.update((int) (deletions >>> 0) & 0xFFFFFFFF); byte deleteOnlyNodesFlag = buffer.get(); crc32.update(deleteOnlyNodesFlag); boolean isDeletOnlyNodes = deleteOnlyNodesFlag == 1; if (!status.isTransient()) { newIndexEntries.put(name, new IndexEntry(indexType, name, parentName, status, mergeId, documentCount, deletions, isDeletOnlyNodes)); } } long onDiskCRC32 = buffer.getLong(); if (crc32.getValue() == onDiskCRC32) { for (IndexEntry entry : indexEntries.values()) { if (entry.getStatus().isTransient()) { newIndexEntries.put(entry.getName(), entry); } } version = onDiskVersion; indexEntries = newIndexEntries; } else { throw new IOException("Invalid file check sum"); } } } }