List of usage examples for org.apache.commons.lang StringUtils stripEnd
public static String stripEnd(String str, String stripChars)
Strips any of a set of characters from the end of a String.
From source file:com.blm.orc.RecordReaderImpl.java
private static Object getConvertedStatsObj(Object statsObj, Object predObj) { // converting between date and other types is not implicit, so convert to // text for comparison if (((predObj instanceof DateWritable) && !(statsObj instanceof DateWritable)) || ((statsObj instanceof DateWritable) && !(predObj instanceof DateWritable))) { return StringUtils.stripEnd(statsObj.toString(), null); }//from www . j a v a2 s . c o m if (statsObj instanceof String) { return StringUtils.stripEnd(statsObj.toString(), null); } return statsObj; }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.sav.SAVFileReader.java
void decodeRecordTypeDataCompressed(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordTypeDataCompressed(): start *****"); if (stream == null) { throw new IllegalArgumentException("decodeRecordTypeDataCompressed: stream == null!"); }/*from w w w. j a v a2 s. co m*/ Map<String, String> formatCategoryTable = new LinkedHashMap<String, String>(); PrintWriter pwout = createOutputWriter(stream); boolean hasStringVarContinuousBlock = obsNonVariableBlockSet.size() > 0 ? true : false; dbgLog.fine("hasStringVarContinuousBlock=" + hasStringVarContinuousBlock); int ii = 0; int OBS = LENGTH_SAV_OBS_BLOCK; int nOBS = OBSUnitsPerCase; dbgLog.fine("OBSUnitsPerCase=" + OBSUnitsPerCase); int caseIndex = 0; dbgLog.fine("printFormatTable:\n" + printFormatTable); dbgLog.fine("printFormatNameTable:\n" + printFormatNameTable); variableFormatTypeList = new String[varQnty]; dbgLog.fine("varQnty: " + varQnty); for (int i = 0; i < varQnty; i++) { variableFormatTypeList[i] = SPSSConstants.FORMAT_CATEGORY_TABLE .get(printFormatTable.get(variableNameList.get(i))); dbgLog.fine("i=" + i + "th variableFormatTypeList=" + variableFormatTypeList[i]); formatCategoryTable.put(variableNameList.get(i), variableFormatTypeList[i]); } dbgLog.fine("variableFormatType:\n" + Arrays.deepToString(variableFormatTypeList)); dbgLog.fine("formatCategoryTable:\n" + formatCategoryTable); // contents (variable) checker concering decimals variableTypeFinal = new int[varQnty]; Arrays.fill(variableTypeFinal, 0); List<String> casewiseRecordForUNF = new ArrayList<String>(); String[] caseWiseDateFormatForUNF = null; List<String> casewiseRecordForTabFile = new ArrayList<String>(); // missing values are written to the tab-delimited file by // using the default or user-specified missing-value strings; // however, to calculate UNF/summary statistics, // classes for these calculations require their specific // missing values that differ from the above missing-value // strings; therefore, after row data for the tab-delimited // file are written, missing values in a row are changed to // UNF/summary-statistics-OK ones. // data-storage object for sumStat dataTable2 = new Object[varQnty][caseQnty]; // storage of date formats to pass to UNF dateFormats = new String[varQnty][caseQnty]; try { // this compression is applied only to non-float data, i.e. integer; // 8-byte float datum is kept in tact boolean hasReachedEOF = false; OBSERVATION: while (true) { dbgLog.fine("SAV Reader: compressed: ii=" + ii + "-th iteration"); byte[] octate = new byte[LENGTH_SAV_OBS_BLOCK]; int nbytes = stream.read(octate); // processCompressedOBSblock () // (this means process a block of 8 compressed OBS // values -- should result in 64 bytes of data total) for (int i = 0; i < LENGTH_SAV_OBS_BLOCK; i++) { dbgLog.finer("i=" + i + "-th iteration"); int octate_i = octate[i]; //dbgLog.fine("octate="+octate_i); if (octate_i < 0) { octate_i += 256; } int byteCode = octate_i;//octate_i & 0xF; //out.println("byeCode="+byteCode); // processCompressedOBS switch (byteCode) { case 252: // end of the file dbgLog.fine("SAV Reader: compressed: end of file mark [FC] was found"); hasReachedEOF = true; break; case 253: // FD: uncompressed data follows after this octate // long string datum or float datum // read the following octate byte[] uncompressedByte = new byte[LENGTH_SAV_OBS_BLOCK]; int ucbytes = stream.read(uncompressedByte); int typeIndex = (ii * OBS + i) % nOBS; if ((OBSwiseTypelList.get(typeIndex) > 0) || (OBSwiseTypelList.get(typeIndex) == -1)) { // code= >0 |-1: string or its conitiguous block // decode as a string object String strdatum = new String( Arrays.copyOfRange(uncompressedByte, 0, LENGTH_SAV_OBS_BLOCK), defaultCharSet); //out.println("str_datum="+strdatum+"<-"); // add this non-missing-value string datum casewiseRecordForTabFile.add(strdatum); //out.println("casewiseRecordForTabFile(String)="+casewiseRecordForTabFile); } else if (OBSwiseTypelList.get(typeIndex) == -2) { String strdatum = new String( Arrays.copyOfRange(uncompressedByte, 0, LENGTH_SAV_OBS_BLOCK - 1), defaultCharSet); casewiseRecordForTabFile.add(strdatum); //out.println("casewiseRecordForTabFile(String)="+casewiseRecordForTabFile); } else if (OBSwiseTypelList.get(typeIndex) == 0) { // code= 0: numeric ByteBuffer bb_double = ByteBuffer.wrap(uncompressedByte, 0, LENGTH_SAV_OBS_BLOCK); if (isLittleEndian) { bb_double.order(ByteOrder.LITTLE_ENDIAN); } Double ddatum = bb_double.getDouble(); // out.println("ddatum="+ddatum); // add this non-missing-value numeric datum casewiseRecordForTabFile.add(doubleNumberFormatter.format(ddatum)); dbgLog.fine("SAV Reader: compressed: added value to dataLine: " + ddatum); } else { dbgLog.fine("SAV Reader: out-of-range exception"); throw new IOException("out-of-range value was found"); } /* // EOF-check after reading this octate if (stream.available() == 0){ hasReachedEOF = true; dbgLog.fine( "SAV Reader: *** After reading an uncompressed octate," + " reached the end of the file at "+ii +"th iteration and i="+i+"th octate position [0-start] *****"); } */ break; case 254: // FE: used as the missing value for string variables // an empty case in a string variable also takes this value // string variable does not accept space-only data // cf: uncompressed case // 20 20 20 20 20 20 20 20 // add the string missing value // out.println("254: String missing data"); casewiseRecordForTabFile.add(" "); // add "." here? // Note that technically this byte flag (254/xFE) means // that *eight* white space characters should be // written to the output stream. This caused me // a great amount of confusion, because it appeared // to me that there was a mismatch between the number // of bytes advertised in the variable metadata and // the number of bytes actually found in the data // section of a compressed SAV file; this is because // these 8 bytes "come out of nowhere"; they are not // written in the data section, but this flag specifies // that they should be added to the output. // Also, as I pointed out above, we are only writing // out one whitespace character, not 8 as instructed. // This appears to be legit; these blocks of 8 spaces // seem to be only used for padding, and all such // multiple padding spaces are stripped anyway during // the post-processing. break; case 255: // FF: system missing value for numeric variables // cf: uncompressed case (sysmis) // FF FF FF FF FF FF eF FF(little endian) // add the numeric missing value dbgLog.fine("SAV Reader: compressed: Missing Value, numeric"); casewiseRecordForTabFile.add(MissingValueForTextDataFileNumeric); break; case 0: // 00: do nothing dbgLog.fine("SAV Reader: compressed: doing nothing (zero); "); break; default: //out.println("byte code(default)="+ byteCode); if ((byteCode > 0) && (byteCode < 252)) { // datum is compressed //Integer unCompressed = Integer.valueOf(byteCode -100); // add this uncompressed numeric datum Double unCompressed = Double.valueOf(byteCode - 100); dbgLog.fine("SAV Reader: compressed: default case: " + unCompressed); casewiseRecordForTabFile.add(doubleNumberFormatter.format(unCompressed)); // out.println("uncompressed="+unCompressed); // out.println("dataline="+casewiseRecordForTabFile); } }// end of switch // out.println("end of switch"); // The-end-of-a-case(row)-processing // this line that follows, and the code around it // is really confusing: int varCounter = (ii * OBS + i + 1) % nOBS; // while both OBS and LENGTH_SAV_OBS_BLOCK = 8 // (OBS was initialized as OBS=LENGTH_SAV_OBS_BLOCK), // the 2 values mean different things: // LENGTH_SAV_OBS_BLOCK is the number of bytes in one OBS; // and OBS is the number of OBS blocks that we process // at a time. I.e., we process 8 chunks of 8 bytes at a time. // This is how data is organized inside an SAV file: // 8 bytes of compression flags, followd by 8x8 or fewer // (depending on the flags) bytes of compressed data. // I should rename this OBS variable something more // meaningful. // // Also, the "varCounter" variable name is entirely // misleading -- it counts not variables, but OBS blocks. dbgLog.fine("SAV Reader: compressed: OBS counter=" + varCounter + "(ii=" + ii + ")"); if ((ii * OBS + i + 1) % nOBS == 0) { //out.println("casewiseRecordForTabFile(before)="+casewiseRecordForTabFile); // out.println("all variables in a case are parsed == nOBS"); // out.println("hasStringVarContinuousBlock="+hasStringVarContinuousBlock); // check whether a string-variable's continuous block exits // if so, they must be joined if (hasStringVarContinuousBlock) { // string-variable's continuous-block-concatenating-processing //out.println("concatenating process starts"); //out.println("casewiseRecordForTabFile(before)="+casewiseRecordForTabFile); //out.println("casewiseRecordForTabFile(before:size)="+casewiseRecordForTabFile.size()); StringBuilder sb = new StringBuilder(""); int firstPosition = 0; Set<Integer> removeJset = new HashSet<Integer>(); for (int j = 0; j < nOBS; j++) { dbgLog.fine("RTD: j=" + j + "-th type =" + OBSwiseTypelList.get(j)); if ((OBSwiseTypelList.get(j) == -1) || (OBSwiseTypelList.get(j) == -2)) { // Continued String variable found at j-th // position. look back the j-1 firstPosition = j - 1; int lastJ = j; String concatenated = null; removeJset.add(j); sb.append(casewiseRecordForTabFile.get(j - 1)); sb.append(casewiseRecordForTabFile.get(j)); for (int jc = 1;; jc++) { if ((j + jc == nOBS) || ((OBSwiseTypelList.get(j + jc) != -1) && (OBSwiseTypelList.get(j + jc) != -2))) { // j is the end unit of this string variable concatenated = sb.toString(); sb.setLength(0); lastJ = j + jc; break; } else { sb.append(casewiseRecordForTabFile.get(j + jc)); removeJset.add(j + jc); } } casewiseRecordForTabFile.set(j - 1, concatenated); //out.println(j-1+"th concatenated="+concatenated); j = lastJ - 1; } // end-of-if: continuous-OBS only } // end of loop-j //out.println("removeJset="+removeJset); // a new list that stores a new case with concatanated string data List<String> newDataLine = new ArrayList<String>(); for (int jl = 0; jl < casewiseRecordForTabFile.size(); jl++) { //out.println("jl="+jl+"-th datum =["+casewiseRecordForTabFile.get(jl)+"]"); if (!removeJset.contains(jl)) { // if (casewiseRecordForTabFile.get(jl).equals(MissingValueForTextDataFileString)){ // out.println("NA-S jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals(MissingValueForTextDataFileNumeric)){ // out.println("NA-N jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl)==null){ // out.println("null case jl="+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals("NaN")){ // out.println("NaN jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals("")){ // out.println("blank jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals(" ")){ // out.println("space jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } newDataLine.add(casewiseRecordForTabFile.get(jl)); } else { // out.println("Excluded: jl="+jl+"-th datum=["+casewiseRecordForTabFile.get(jl)+"]"); } } // end of loop-jl //out.println("new casewiseRecordForTabFile="+newDataLine); //out.println("new casewiseRecordForTabFile(size)="+newDataLine.size()); casewiseRecordForTabFile = newDataLine; } // end-if: stringContinuousVar-exist case for (int el = 0; el < casewiseRecordForTabFile.size(); el++) { casewiseRecordForUNF.add(casewiseRecordForTabFile.get(el)); } caseWiseDateFormatForUNF = new String[casewiseRecordForTabFile.size()]; // caseIndex starts from 1 not 0 caseIndex = (ii * OBS + i + 1) / nOBS; for (int k = 0; k < casewiseRecordForTabFile.size(); k++) { dbgLog.fine("k=" + k + "-th variableTypelList=" + variableTypelList.get(k)); if (variableTypelList.get(k) > 0) { // String variable case: set to -1 variableTypeFinal[k] = -1; // Strip the String variables off the // whitespace padding: // [ snipped ] // I've removed the block of code above where // String values were substring()-ed to the // length specified in the variable metadata; // Doing that was not enough, since a string // can still be space-padded inside its // advertised capacity. (note that extended // variables can have many kylobytes of such // padding in them!) Plus it was completely // redundant, since we are stripping all the // trailing white spaces with // StringUtils.stripEnd() below: String paddRemoved = StringUtils .stripEnd(casewiseRecordForTabFile.get(k).toString(), null); // TODO: clean this up. For now, just make sure that strings contain at least one blank space. if (paddRemoved.equals("")) { paddRemoved = " "; } casewiseRecordForUNF.set(k, paddRemoved); casewiseRecordForTabFile.set(k, "\"" + paddRemoved.replaceAll("\"", Matcher.quoteReplacement("\\\"")) + "\""); // end of String var case } else { // numeric var case if (casewiseRecordForTabFile.get(k).equals(MissingValueForTextDataFileNumeric)) { casewiseRecordForUNF.set(k, null); } } // end of variable-type check if (casewiseRecordForTabFile.get(k) != null && !casewiseRecordForTabFile.get(k) .equals(MissingValueForTextDataFileNumeric)) { String variableFormatType = variableFormatTypeList[k]; dbgLog.finer("k=" + k + "th printFormatTable format=" + printFormatTable.get(variableNameList.get(k))); int formatDecimalPointPosition = formatDecimalPointPositionList.get(k); if (variableFormatType.equals("date")) { dbgLog.finer("date case"); long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymd.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); caseWiseDateFormatForUNF[k] = sdf_ymd.toPattern(); /* saving date format */ dbgLog.finer("setting caseWiseDateFormatForUNF[k] = " + sdf_ymd.toPattern()); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); //formatCategoryTable.put(variableNameList.get(k), "date"); } else if (variableFormatType.equals("time")) { dbgLog.finer("time case:DTIME or DATETIME or TIME"); //formatCategoryTable.put(variableNameList.get(k), "time"); if (printFormatTable.get(variableNameList.get(k)).equals("DTIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long .parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_BIAS; String newDatum = sdf_dhms.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString() .split("\\."); dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_BIAS; StringBuilder sb_time = new StringBuilder( sdf_dhms.format(new Date(dateDatum))); dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append( "." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); casewiseRecordForUNF.set(k, sb_time.toString()); } } else if (printFormatTable.get(variableNameList.get(k)).equals("DATETIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long .parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymdhms.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); caseWiseDateFormatForUNF[k] = sdf_ymdhms.toPattern(); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString() .split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_OFFSET; StringBuilder sb_time = new StringBuilder( sdf_ymdhms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append( "." + timeData[1].substring(0, formatDecimalPointPosition)); } caseWiseDateFormatForUNF[k] = sdf_ymdhms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); casewiseRecordForUNF.set(k, sb_time.toString()); } } else if (printFormatTable.get(variableNameList.get(k)).equals("TIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long .parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L; String newDatum = sdf_hms.format(new Date(dateDatum)); caseWiseDateFormatForUNF[k] = sdf_hms.toPattern(); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString() .split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L; StringBuilder sb_time = new StringBuilder( sdf_hms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append( "." + timeData[1].substring(0, formatDecimalPointPosition)); } caseWiseDateFormatForUNF[k] = this.sdf_hms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); casewiseRecordForUNF.set(k, sb_time.toString()); } } } else if (variableFormatType.equals("other")) { dbgLog.finer("other non-date/time case:=" + i); if (printFormatTable.get(variableNameList.get(k)).equals("WKDAY")) { // day of week dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.WEEKDAY_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.WEEKDAY_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); dbgLog.finer("wkday:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } else if (printFormatTable.get(variableNameList.get(k)).equals("MONTH")) { // month dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.MONTH_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.MONTH_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); dbgLog.finer("month:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } } } // end: date-time-datum check } // end: loop-k(2nd: variable-wise-check) // write to tab file if (casewiseRecordForTabFile.size() > 0) { pwout.println(StringUtils.join(casewiseRecordForTabFile, "\t")); } if (casewiseRecordForTabFile.size() > 0) { for (int ij = 0; ij < varQnty; ij++) { dataTable2[ij][caseIndex - 1] = casewiseRecordForUNF.get(ij); if (variableFormatTypeList[ij].equals("date") || variableFormatTypeList[ij].equals("time")) { this.dateFormats[ij][caseIndex - 1] = caseWiseDateFormatForUNF[ij]; } } } // numeric contents-check for (int l = 0; l < casewiseRecordForTabFile.size(); l++) { if (variableFormatTypeList[l].equals("date") || variableFormatTypeList[l].equals("time") || printFormatTable.get(variableNameList.get(l)).equals("WKDAY") || printFormatTable.get(variableNameList.get(l)).equals("MONTH")) { variableTypeFinal[l] = -1; } if (variableTypeFinal[l] == 0) { if (casewiseRecordForTabFile.get(l).toString().indexOf(".") >= 0) { // TODO - check for large numbers // l-th variable is not integer variableTypeFinal[l] = 1; decimalVariableSet.add(l); } } } // reset the case-wise working objects casewiseRecordForUNF.clear(); casewiseRecordForTabFile.clear(); if (caseQnty > 0) { if (caseIndex == caseQnty) { hasReachedEOF = true; } } if (hasReachedEOF) { break; } } // if(The-end-of-a-case(row)-processing) } // loop-i (OBS unit) if ((hasReachedEOF) || (stream.available() == 0)) { // reached the end of this file // do exit-processing dbgLog.fine("***** reached the end of the file at " + ii + "th iteration *****"); break OBSERVATION; } ii++; } // while loop pwout.close(); } catch (IOException ex) { throw ex; } smd.setDecimalVariables(decimalVariableSet); smd.getFileInformation().put("caseQnty", caseQnty); smd.setVariableFormatCategory(formatCategoryTable); dbgLog.info("<<<<<<"); //dbgLog.info("printFormatList = " + printFormatList); //dbgLog.info("printFormatNameTable = " + printFormatNameTable); dbgLog.info("formatCategoryTable = " + formatCategoryTable); dbgLog.info(">>>>>>"); // contents check //out.println("variableType="+ArrayUtils.toString(variableTypeFinal)); dbgLog.fine("decimalVariableSet=" + decimalVariableSet); //out.println("variableTypelList=\n"+ variableTypelList.toString()); // out.println("dataTable2:\n"+Arrays.deepToString(dataTable2)); dbgLog.fine("***** decodeRecordTypeDataCompressed(): end *****"); }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeRecordTypeDataCompressed(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordTypeDataCompressed(): start *****"); if (stream == null) { throw new IllegalArgumentException("decodeRecordTypeDataCompressed: stream == null!"); }/*from www . ja v a 2s . c om*/ PrintWriter pwout = createOutputWriter(stream); int varQnty = dataTable.getVarQuantity().intValue(); int caseQnty = dataTable.getCaseQuantity().intValue(); dbgLog.fine("varQnty: " + varQnty); dateFormatList = new String[varQnty]; boolean hasStringVarContinuousBlock = obsNonVariableBlockSet.size() > 0 ? true : false; dbgLog.fine("hasStringVarContinuousBlock=" + hasStringVarContinuousBlock); int ii = 0; int OBS = LENGTH_SAV_OBS_BLOCK; int nOBS = OBSUnitsPerCase; dbgLog.fine("OBSUnitsPerCase=" + OBSUnitsPerCase); int caseIndex = 0; dbgLog.fine("printFormatTable:\n" + printFormatTable); variableFormatTypeList = new String[varQnty]; for (int i = 0; i < varQnty; i++) { variableFormatTypeList[i] = SPSSConstants.FORMAT_CATEGORY_TABLE .get(printFormatTable.get(variableNameList.get(i))); dbgLog.fine("i=" + i + "th variableFormatTypeList=" + variableFormatTypeList[i]); formatCategoryTable.put(variableNameList.get(i), variableFormatTypeList[i]); } dbgLog.fine("variableFormatType:\n" + Arrays.deepToString(variableFormatTypeList)); dbgLog.fine("formatCategoryTable:\n" + formatCategoryTable); // TODO: // Make sure the date formats are actually preserved! // (this is something that was collected in the code below and passed // to the UNF calculator). // -- L.A. 4.0 alpha List<String> casewiseRecordForTabFile = new ArrayList<String>(); try { // this compression is applied only to non-float data, i.e. integer; // 8-byte float datum is kept in tact boolean hasReachedEOF = false; OBSERVATION: while (true) { dbgLog.fine("SAV Reader: compressed: ii=" + ii + "-th iteration"); byte[] octate = new byte[LENGTH_SAV_OBS_BLOCK]; int nbytes = stream.read(octate); // processCompressedOBSblock () // (this means process a block of 8 compressed OBS // values -- should result in 64 bytes of data total) for (int i = 0; i < LENGTH_SAV_OBS_BLOCK; i++) { dbgLog.finer("i=" + i + "-th iteration"); int octate_i = octate[i]; //dbgLog.fine("octate="+octate_i); if (octate_i < 0) { octate_i += 256; } int byteCode = octate_i;//octate_i & 0xF; //out.println("byeCode="+byteCode); // processCompressedOBS switch (byteCode) { case 252: // end of the file dbgLog.fine("SAV Reader: compressed: end of file mark [FC] was found"); hasReachedEOF = true; break; case 253: // FD: uncompressed data follows after this octate // long string datum or float datum // read the following octate byte[] uncompressedByte = new byte[LENGTH_SAV_OBS_BLOCK]; int ucbytes = stream.read(uncompressedByte); int typeIndex = (ii * OBS + i) % nOBS; if ((OBSwiseTypelList.get(typeIndex) > 0) || (OBSwiseTypelList.get(typeIndex) == -1)) { // code= >0 |-1: string or its conitiguous block // decode as a string object String strdatum = new String( Arrays.copyOfRange(uncompressedByte, 0, LENGTH_SAV_OBS_BLOCK), defaultCharSet); //out.println("str_datum="+strdatum+"<-"); // add this non-missing-value string datum casewiseRecordForTabFile.add(strdatum); //out.println("casewiseRecordForTabFile(String)="+casewiseRecordForTabFile); } else if (OBSwiseTypelList.get(typeIndex) == -2) { String strdatum = new String( Arrays.copyOfRange(uncompressedByte, 0, LENGTH_SAV_OBS_BLOCK - 1), defaultCharSet); casewiseRecordForTabFile.add(strdatum); //out.println("casewiseRecordForTabFile(String)="+casewiseRecordForTabFile); } else if (OBSwiseTypelList.get(typeIndex) == 0) { // code= 0: numeric ByteBuffer bb_double = ByteBuffer.wrap(uncompressedByte, 0, LENGTH_SAV_OBS_BLOCK); if (isLittleEndian) { bb_double.order(ByteOrder.LITTLE_ENDIAN); } Double ddatum = bb_double.getDouble(); // out.println("ddatum="+ddatum); // add this non-missing-value numeric datum casewiseRecordForTabFile.add(doubleNumberFormatter.format(ddatum)); dbgLog.fine("SAV Reader: compressed: added value to dataLine: " + ddatum); } else { dbgLog.fine("SAV Reader: out-of-range exception"); throw new IOException("out-of-range value was found"); } /* // EOF-check after reading this octate if (stream.available() == 0){ hasReachedEOF = true; dbgLog.fine( "SAV Reader: *** After reading an uncompressed octate," + " reached the end of the file at "+ii +"th iteration and i="+i+"th octate position [0-start] *****"); } */ break; case 254: // FE: used as the missing value for string variables // an empty case in a string variable also takes this value // string variable does not accept space-only data // cf: uncompressed case // 20 20 20 20 20 20 20 20 // add the string missing value // out.println("254: String missing data"); casewiseRecordForTabFile.add(" "); // add "." here? // Note that technically this byte flag (254/xFE) means // that *eight* white space characters should be // written to the output stream. This caused me // a great amount of confusion, because it appeared // to me that there was a mismatch between the number // of bytes advertised in the variable metadata and // the number of bytes actually found in the data // section of a compressed SAV file; this is because // these 8 bytes "come out of nowhere"; they are not // written in the data section, but this flag specifies // that they should be added to the output. // Also, as I pointed out above, we are only writing // out one whitespace character, not 8 as instructed. // This appears to be legit; these blocks of 8 spaces // seem to be only used for padding, and all such // multiple padding spaces are stripped anyway during // the post-processing. break; case 255: // FF: system missing value for numeric variables // cf: uncompressed case (sysmis) // FF FF FF FF FF FF eF FF(little endian) // add the numeric missing value dbgLog.fine("SAV Reader: compressed: Missing Value, numeric"); casewiseRecordForTabFile.add(MissingValueForTextDataFileNumeric); break; case 0: // 00: do nothing dbgLog.fine("SAV Reader: compressed: doing nothing (zero); "); break; default: //out.println("byte code(default)="+ byteCode); if ((byteCode > 0) && (byteCode < 252)) { // datum is compressed //Integer unCompressed = Integer.valueOf(byteCode -100); // add this uncompressed numeric datum Double unCompressed = Double.valueOf(byteCode - 100); dbgLog.fine("SAV Reader: compressed: default case: " + unCompressed); casewiseRecordForTabFile.add(doubleNumberFormatter.format(unCompressed)); // out.println("uncompressed="+unCompressed); // out.println("dataline="+casewiseRecordForTabFile); } }// end of switch // out.println("end of switch"); // The-end-of-a-case(row)-processing // this line that follows, and the code around it // is really confusing: int varCounter = (ii * OBS + i + 1) % nOBS; // while both OBS and LENGTH_SAV_OBS_BLOCK = 8 // (OBS was initialized as OBS=LENGTH_SAV_OBS_BLOCK), // the 2 values mean different things: // LENGTH_SAV_OBS_BLOCK is the number of bytes in one OBS; // and OBS is the number of OBS blocks that we process // at a time. I.e., we process 8 chunks of 8 bytes at a time. // This is how data is organized inside an SAV file: // 8 bytes of compression flags, followd by 8x8 or fewer // (depending on the flags) bytes of compressed data. // I should rename this OBS variable something more // meaningful. // // Also, the "varCounter" variable name is entirely // misleading -- it counts not variables, but OBS blocks. dbgLog.fine("SAV Reader: compressed: OBS counter=" + varCounter + "(ii=" + ii + ")"); if ((ii * OBS + i + 1) % nOBS == 0) { //out.println("casewiseRecordForTabFile(before)="+casewiseRecordForTabFile); // out.println("all variables in a case are parsed == nOBS"); // out.println("hasStringVarContinuousBlock="+hasStringVarContinuousBlock); // check whether a string-variable's continuous block exits // if so, they must be joined if (hasStringVarContinuousBlock) { // string-variable's continuous-block-concatenating-processing //out.println("concatenating process starts"); //out.println("casewiseRecordForTabFile(before)="+casewiseRecordForTabFile); //out.println("casewiseRecordForTabFile(before:size)="+casewiseRecordForTabFile.size()); StringBuilder sb = new StringBuilder(""); int firstPosition = 0; Set<Integer> removeJset = new HashSet<Integer>(); for (int j = 0; j < nOBS; j++) { dbgLog.fine("RTD: j=" + j + "-th type =" + OBSwiseTypelList.get(j)); if ((OBSwiseTypelList.get(j) == -1) || (OBSwiseTypelList.get(j) == -2)) { // Continued String variable found at j-th // position. look back the j-1 firstPosition = j - 1; int lastJ = j; String concatenated = null; removeJset.add(j); sb.append(casewiseRecordForTabFile.get(j - 1)); sb.append(casewiseRecordForTabFile.get(j)); for (int jc = 1;; jc++) { if ((j + jc == nOBS) || ((OBSwiseTypelList.get(j + jc) != -1) && (OBSwiseTypelList.get(j + jc) != -2))) { // j is the end unit of this string variable concatenated = sb.toString(); sb.setLength(0); lastJ = j + jc; break; } else { sb.append(casewiseRecordForTabFile.get(j + jc)); removeJset.add(j + jc); } } casewiseRecordForTabFile.set(j - 1, concatenated); //out.println(j-1+"th concatenated="+concatenated); j = lastJ - 1; } // end-of-if: continuous-OBS only } // end of loop-j //out.println("removeJset="+removeJset); // a new list that stores a new case with concatanated string data List<String> newDataLine = new ArrayList<String>(); for (int jl = 0; jl < casewiseRecordForTabFile.size(); jl++) { //out.println("jl="+jl+"-th datum =["+casewiseRecordForTabFile.get(jl)+"]"); if (!removeJset.contains(jl)) { // if (casewiseRecordForTabFile.get(jl).equals(MissingValueForTextDataFileString)){ // out.println("NA-S jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals(MissingValueForTextDataFileNumeric)){ // out.println("NA-N jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl)==null){ // out.println("null case jl="+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals("NaN")){ // out.println("NaN jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals("")){ // out.println("blank jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } else if (casewiseRecordForTabFile.get(jl).equals(" ")){ // out.println("space jl= "+jl+"=["+casewiseRecordForTabFile.get(jl)+"]"); // } newDataLine.add(casewiseRecordForTabFile.get(jl)); } else { // out.println("Excluded: jl="+jl+"-th datum=["+casewiseRecordForTabFile.get(jl)+"]"); } } // end of loop-jl //out.println("new casewiseRecordForTabFile="+newDataLine); //out.println("new casewiseRecordForTabFile(size)="+newDataLine.size()); casewiseRecordForTabFile = newDataLine; } // end-if: stringContinuousVar-exist case // caseIndex starts from 1 not 0 caseIndex = (ii * OBS + i + 1) / nOBS; for (int k = 0; k < casewiseRecordForTabFile.size(); k++) { dbgLog.fine("k=" + k + "-th variableTypelList=" + variableTypelList.get(k)); if (variableTypelList.get(k) > 0) { // Strip the String variables off the // whitespace padding: // [ snipped ] // I've removed the block of code above where // String values were substring()-ed to the // length specified in the variable metadata; // Doing that was not enough, since a string // can still be space-padded inside its // advertised capacity. (note that extended // variables can have many kylobytes of such // padding in them!) Plus it was completely // redundant, since we are stripping all the // trailing white spaces with // StringUtils.stripEnd() below: String paddRemoved = StringUtils .stripEnd(casewiseRecordForTabFile.get(k).toString(), null); // TODO: clean this up. For now, just make sure that strings contain at least one blank space. if (paddRemoved.equals("")) { paddRemoved = " "; } //casewiseRecordForTabFile.set(k, "\"" + paddRemoved.replaceAll("\"", Matcher.quoteReplacement("\\\"")) + "\""); casewiseRecordForTabFile.set(k, escapeCharacterString(paddRemoved)); // end of String var case } // end of variable-type check if (casewiseRecordForTabFile.get(k) != null && !casewiseRecordForTabFile.get(k) .equals(MissingValueForTextDataFileNumeric)) { String variableFormatType = variableFormatTypeList[k]; dbgLog.finer("k=" + k + "th printFormatTable format=" + printFormatTable.get(variableNameList.get(k))); int formatDecimalPointPosition = formatDecimalPointPositionList.get(k); if (variableFormatType.equals("date")) { dbgLog.finer("date case"); long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymd.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); /* saving date format */ dbgLog.finer("saving dateFormat[k] = " + sdf_ymd.toPattern()); casewiseRecordForTabFile.set(k, newDatum); dateFormatList[k] = sdf_ymd.toPattern(); //formatCategoryTable.put(variableNameList.get(k), "date"); } else if (variableFormatType.equals("time")) { dbgLog.finer("time case:DTIME or DATETIME or TIME"); //formatCategoryTable.put(variableNameList.get(k), "time"); if (printFormatTable.get(variableNameList.get(k)).equals("DTIME")) { // We're not even going to try to handle "DTIME" // values as time/dates in dataverse; this is a weird // format that nobody uses outside of SPSS. // (but we do need to remember to treat the resulting values // as character strings, not numerics!) if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long .parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_BIAS; String newDatum = sdf_dhms.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString() .split("\\."); dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_BIAS; StringBuilder sb_time = new StringBuilder( sdf_dhms.format(new Date(dateDatum))); dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append( "." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); } } else if (printFormatTable.get(variableNameList.get(k)).equals("DATETIME")) { // TODO: // (for both datetime and "dateless" time) // keep the longest of the matching formats - i.e., if there are *some* // values in the vector that have thousands of a second, that should be // part of the saved format! // -- L.A. Aug. 12 2014 if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long .parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymdhms.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); dateFormatList[k] = sdf_ymdhms.toPattern(); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString() .split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_OFFSET; StringBuilder sb_time = new StringBuilder( sdf_ymdhms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append( "." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); dateFormatList[k] = sdf_ymdhms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); } } else if (printFormatTable.get(variableNameList.get(k)).equals("TIME")) { // TODO: // double-check that we are handling "dateless" time correctly... -- L.A. Aug. 2014 if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long .parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L; String newDatum = sdf_hms.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); if (dateFormatList[k] == null) { dateFormatList[k] = sdf_hms.toPattern(); } } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString() .split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L; StringBuilder sb_time = new StringBuilder( sdf_hms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append( "." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); String format_hmsS = sdf_hms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); if (dateFormatList[k] == null || (format_hmsS.length() > dateFormatList[k].length())) { dateFormatList[k] = format_hmsS; } } } } else if (variableFormatType.equals("other")) { dbgLog.finer("other non-date/time case:=" + i); if (printFormatTable.get(variableNameList.get(k)).equals("WKDAY")) { // day of week dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.WEEKDAY_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.WEEKDAY_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); dbgLog.finer("wkday:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } else if (printFormatTable.get(variableNameList.get(k)).equals("MONTH")) { // month dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.MONTH_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.MONTH_LIST.get( Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); dbgLog.finer("month:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } } } // end: date-time-datum check } // end: loop-k(2nd: variable-wise-check) // write to tab file if (casewiseRecordForTabFile.size() > 0) { pwout.println(StringUtils.join(casewiseRecordForTabFile, "\t")); } // numeric contents-check for (int l = 0; l < casewiseRecordForTabFile.size(); l++) { if (variableFormatTypeList[l].equals("date") || variableFormatTypeList[l].equals("time") || printFormatTable.get(variableNameList.get(l)).equals("WKDAY") || printFormatTable.get(variableNameList.get(l)).equals("MONTH")) { // TODO: // figure out if any special handling is still needed here in 4.0. // -- L.A. - Aug. 2014 } else { if (variableTypelList.get(l) <= 0) { if (casewiseRecordForTabFile.get(l).toString().indexOf(".") >= 0) { decimalVariableSet.add(l); } } } } // reset the case-wise working objects casewiseRecordForTabFile.clear(); if (caseQnty > 0) { if (caseIndex == caseQnty) { hasReachedEOF = true; } } if (hasReachedEOF) { break; } } // if(The-end-of-a-case(row)-processing) } // loop-i (OBS unit) if ((hasReachedEOF) || (stream.available() == 0)) { // reached the end of this file // do exit-processing dbgLog.fine("***** reached the end of the file at " + ii + "th iteration *****"); break OBSERVATION; } ii++; } // while loop pwout.close(); } catch (IOException ex) { throw ex; } dbgLog.fine("<<<<<<"); dbgLog.fine("formatCategoryTable = " + formatCategoryTable); dbgLog.fine(">>>>>>"); dbgLog.fine("decimalVariableSet=" + decimalVariableSet); dbgLog.fine("decodeRecordTypeDataCompressed(): end"); }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.sav.SAVFileReader.java
void decodeRecordTypeDataUnCompressed(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordTypeDataUnCompressed(): start *****"); if (stream == null) { throw new IllegalArgumentException("decodeRecordTypeDataUnCompressed: stream == null!"); }/*from w ww .ja v a 2 s . c om*/ Map<String, String> formatCategoryTable = new LinkedHashMap<String, String>(); // // set-up tab file PrintWriter pwout = createOutputWriter(stream); boolean hasStringVarContinuousBlock = obsNonVariableBlockSet.size() > 0 ? true : false; dbgLog.fine("hasStringVarContinuousBlock=" + hasStringVarContinuousBlock); int ii = 0; int OBS = LENGTH_SAV_OBS_BLOCK; int nOBS = OBSUnitsPerCase; dbgLog.fine("OBSUnitsPerCase=" + OBSUnitsPerCase); int caseIndex = 0; dbgLog.fine("printFormatTable:\n" + printFormatTable); dbgLog.fine("printFormatNameTable:\n" + printFormatNameTable); variableFormatTypeList = new String[varQnty]; for (int i = 0; i < varQnty; i++) { variableFormatTypeList[i] = SPSSConstants.FORMAT_CATEGORY_TABLE .get(printFormatTable.get(variableNameList.get(i))); dbgLog.fine("i=" + i + "th variableFormatTypeList=" + variableFormatTypeList[i]); formatCategoryTable.put(variableNameList.get(i), variableFormatTypeList[i]); } dbgLog.fine("variableFormatType:\n" + Arrays.deepToString(variableFormatTypeList)); dbgLog.fine("formatCategoryTable:\n" + formatCategoryTable); // contents (variable) checker concering decimals variableTypeFinal = new int[varQnty]; Arrays.fill(variableTypeFinal, 0); int numberOfDecimalVariables = 0; List<String> casewiseRecordForTabFile = new ArrayList<String>(); String[] caseWiseDateFormatForUNF = null; List<String> casewiseRecordForUNF = new ArrayList<String>(); // missing values are written to the tab-delimited file by // using the default or user-specified missing-value strings; // however, to calculate UNF/summary statistics, // classes for these calculations require their specific // missing values that differ from the above missing-value // strings; therefore, after row data for the tab-delimited // file are written, missing values in a row are changed to // UNF/summary-statistics-OK ones. // data-storage object for sumStat dataTable2 = new Object[varQnty][caseQnty]; // storage of date formats to pass to UNF dateFormats = new String[varQnty][caseQnty]; try { for (int i = 0;; i++) { // case-wise loop byte[] buffer = new byte[OBS * nOBS]; int nbytesuc = stream.read(buffer); StringBuilder sb_stringStorage = new StringBuilder(""); for (int k = 0; k < nOBS; k++) { int offset = OBS * k; // uncompressed case // numeric missing value == sysmis // FF FF FF FF FF FF eF FF(little endian) // string missing value // 20 20 20 20 20 20 20 20 // cf: compressed case // numeric type:sysmis == 0xFF // string type: missing value == 0xFE // boolean isNumeric = OBSwiseTypelList.get(k) == 0 ? true : false; if (isNumeric) { dbgLog.finer(k + "-th variable is numeric"); // interprete as double ByteBuffer bb_double = ByteBuffer.wrap(buffer, offset, LENGTH_SAV_OBS_BLOCK); if (isLittleEndian) { bb_double.order(ByteOrder.LITTLE_ENDIAN); } //char[] hexpattern = String dphex = new String(Hex.encodeHex( Arrays.copyOfRange(bb_double.array(), offset, offset + LENGTH_SAV_OBS_BLOCK))); dbgLog.finer("dphex=" + dphex); if ((dphex.equals("ffffffffffffefff")) || (dphex.equals("ffefffffffffffff"))) { //casewiseRecordForTabFile.add(systemMissingValue); // add the numeric missing value dbgLog.fine("SAV Reader: adding: Missing Value (numeric)"); casewiseRecordForTabFile.add(MissingValueForTextDataFileNumeric); } else { Double ddatum = bb_double.getDouble(); dbgLog.fine("SAV Reader: adding: ddatum=" + ddatum); // add this non-missing-value numeric datum casewiseRecordForTabFile.add(doubleNumberFormatter.format(ddatum)); } } else { dbgLog.finer(k + "-th variable is string"); // string case // strip space-padding // do not trim: string might have spaces within it // the missing value (hex) for a string variable is: // "20 20 20 20 20 20 20 20" String strdatum = new String( Arrays.copyOfRange(buffer, offset, (offset + LENGTH_SAV_OBS_BLOCK)), defaultCharSet); dbgLog.finer("str_datum=" + strdatum); // add this non-missing-value string datum casewiseRecordForTabFile.add(strdatum); } // if isNumeric } // k-loop // String-variable's continuous block exits: if (hasStringVarContinuousBlock) { // continuous blocks: string case // concatenating process //dbgLog.fine("concatenating process starts"); //dbgLog.fine("casewiseRecordForTabFile(before)="+casewiseRecordForTabFile); //dbgLog.fine("casewiseRecordForTabFile(before:size)="+casewiseRecordForTabFile.size()); StringBuilder sb = new StringBuilder(""); int firstPosition = 0; Set<Integer> removeJset = new HashSet<Integer>(); for (int j = 0; j < nOBS; j++) { dbgLog.finer("j=" + j + "-th type =" + OBSwiseTypelList.get(j)); if (OBSwiseTypelList.get(j) == -1) { // String continued fount at j-th // look back the j-1 firstPosition = j - 1; int lastJ = j; String concatanated = null; removeJset.add(j); sb.append(casewiseRecordForTabFile.get(j - 1)); sb.append(casewiseRecordForTabFile.get(j)); for (int jc = 1;; jc++) { if (OBSwiseTypelList.get(j + jc) != -1) { // j is the end unit of this string variable concatanated = sb.toString(); sb.setLength(0); lastJ = j + jc; break; } else { sb.append(casewiseRecordForTabFile.get(j + jc)); removeJset.add(j + jc); } } casewiseRecordForTabFile.set(j - 1, concatanated); //out.println(j-1+"th concatanated="+concatanated); j = lastJ - 1; } // end-of-if: continuous-OBS only } // end of loop-j List<String> newDataLine = new ArrayList<String>(); for (int jl = 0; jl < casewiseRecordForTabFile.size(); jl++) { //out.println("jl="+jl+"-th datum =["+casewiseRecordForTabFile.get(jl)+"]"); if (!removeJset.contains(jl)) { newDataLine.add(casewiseRecordForTabFile.get(jl)); } } dbgLog.fine("new casewiseRecordForTabFile=" + newDataLine); dbgLog.fine("new casewiseRecordForTabFile(size)=" + newDataLine.size()); casewiseRecordForTabFile = newDataLine; } // end-if: stringContinuousVar-exist case for (int el = 0; el < casewiseRecordForTabFile.size(); el++) { casewiseRecordForUNF.add(casewiseRecordForTabFile.get(el)); } caseWiseDateFormatForUNF = new String[casewiseRecordForTabFile.size()]; caseIndex++; dbgLog.finer("caseIndex=" + caseIndex); for (int k = 0; k < casewiseRecordForTabFile.size(); k++) { if (variableTypelList.get(k) > 0) { // String variable case: set to -1 variableTypeFinal[k] = -1; // See my comments for this padding removal logic // in the "compressed" method -- L.A. String paddRemoved = StringUtils.stripEnd(casewiseRecordForTabFile.get(k).toString(), null); // TODO: clean this up. For now, just make sure that strings contain at least one blank space. if (paddRemoved.equals("")) { paddRemoved = " "; } casewiseRecordForUNF.set(k, paddRemoved); casewiseRecordForTabFile.set(k, "\"" + paddRemoved.replaceAll("\"", Matcher.quoteReplacement("\\\"")) + "\""); // end of String var case } else { // numeric var case if (casewiseRecordForTabFile.get(k).equals(MissingValueForTextDataFileNumeric)) { casewiseRecordForUNF.set(k, null); } } // end of variable-type check if (casewiseRecordForTabFile.get(k) != null && !casewiseRecordForTabFile.get(k).equals(MissingValueForTextDataFileNumeric)) { // to do date conversion String variableFormatType = variableFormatTypeList[k]; dbgLog.finer("k=" + k + "th variable format=" + variableFormatType); int formatDecimalPointPosition = formatDecimalPointPositionList.get(k); if (variableFormatType.equals("date")) { dbgLog.finer("date case"); long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymd.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); caseWiseDateFormatForUNF[k] = sdf_ymd.toPattern(); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); //formatCategoryTable.put(variableNameList.get(k), "date"); } else if (variableFormatType.equals("time")) { dbgLog.finer("time case:DTIME or DATETIME or TIME"); //formatCategoryTable.put(variableNameList.get(k), "time"); if (printFormatTable.get(variableNameList.get(k)).equals("DTIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_BIAS; String newDatum = sdf_dhms.format(new Date(dateDatum)); // Note: DTIME is not a complete date, so we don't save a date format with it dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString().split("\\."); dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_BIAS; StringBuilder sb_time = new StringBuilder(sdf_dhms.format(new Date(dateDatum))); if (formatDecimalPointPosition > 0) { sb_time.append("." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); casewiseRecordForUNF.set(k, sb_time.toString()); } } else if (printFormatTable.get(variableNameList.get(k)).equals("DATETIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymdhms.format(new Date(dateDatum)); caseWiseDateFormatForUNF[k] = sdf_ymdhms.toPattern(); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString().split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_OFFSET; StringBuilder sb_time = new StringBuilder( sdf_ymdhms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append("." + timeData[1].substring(0, formatDecimalPointPosition)); } caseWiseDateFormatForUNF[k] = sdf_ymdhms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); casewiseRecordForUNF.set(k, sb_time.toString()); } } else if (printFormatTable.get(variableNameList.get(k)).equals("TIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L; String newDatum = sdf_hms.format(new Date(dateDatum)); caseWiseDateFormatForUNF[k] = sdf_hms.toPattern(); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString().split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L; StringBuilder sb_time = new StringBuilder(sdf_hms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append("." + timeData[1].substring(0, formatDecimalPointPosition)); } caseWiseDateFormatForUNF[k] = this.sdf_hms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); casewiseRecordForUNF.set(k, sb_time.toString()); } } } else if (variableFormatType.equals("other")) { dbgLog.finer("other non-date/time case"); if (printFormatTable.get(variableNameList.get(k)).equals("WKDAY")) { // day of week dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.WEEKDAY_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.WEEKDAY_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); dbgLog.finer("wkday:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } else if (printFormatTable.get(variableNameList.get(k)).equals("MONTH")) { // month dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.MONTH_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.MONTH_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); casewiseRecordForUNF.set(k, newDatum); dbgLog.finer("month:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } } // end of date/time block } // end: date-time-datum check } // end: loop-k(2nd: variablte-wise-check) // write to tab file if (casewiseRecordForTabFile.size() > 0) { pwout.println(StringUtils.join(casewiseRecordForTabFile, "\t")); } if (casewiseRecordForTabFile.size() > 0) { for (int ij = 0; ij < varQnty; ij++) { dataTable2[ij][caseIndex - 1] = casewiseRecordForUNF.get(ij); if (variableFormatTypeList[ij].equals("date") || variableFormatTypeList[ij].equals("time")) { this.dateFormats[ij][caseIndex - 1] = caseWiseDateFormatForUNF[ij]; } } } // numeric contents-check for (int l = 0; l < casewiseRecordForTabFile.size(); l++) { if (variableFormatTypeList[l].equals("date") || variableFormatTypeList[l].equals("time") || printFormatTable.get(variableNameList.get(l)).equals("WKDAY") || printFormatTable.get(variableNameList.get(l)).equals("MONTH")) { variableTypeFinal[l] = -1; } if (variableTypeFinal[l] == 0) { if (casewiseRecordForTabFile.get(l).toString().indexOf(".") >= 0) { // l-th variable is not integer variableTypeFinal[l] = 1; decimalVariableSet.add(l); } } } // reset the case-wise working objects casewiseRecordForTabFile.clear(); casewiseRecordForUNF.clear(); if (stream.available() == 0) { // reached the end of this file // do exit-processing dbgLog.fine("***** reached the end of the file at " + ii + "th iteration *****"); break; } // if eof processing } //i-loop: case(row) iteration // close the writer pwout.close(); } catch (IOException ex) { throw ex; } smd.getFileInformation().put("caseQnty", caseQnty); smd.setDecimalVariables(decimalVariableSet); smd.setVariableFormatCategory(formatCategoryTable); // contents check dbgLog.fine("variableType=" + ArrayUtils.toString(variableTypeFinal)); dbgLog.fine("numberOfDecimalVariables=" + numberOfDecimalVariables); dbgLog.fine("decimalVariableSet=" + decimalVariableSet); dbgLog.fine("***** decodeRecordTypeDataUnCompressed(): end *****"); }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeRecordTypeDataUnCompressed(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordTypeDataUnCompressed(): start *****"); if (stream == null) { throw new IllegalArgumentException("decodeRecordTypeDataUnCompressed: stream == null!"); }// w w w .j a va 2 s. c o m int varQnty = dataTable.getVarQuantity().intValue(); // // set-up tab file PrintWriter pwout = createOutputWriter(stream); boolean hasStringVarContinuousBlock = obsNonVariableBlockSet.size() > 0 ? true : false; dbgLog.fine("hasStringVarContinuousBlock=" + hasStringVarContinuousBlock); int ii = 0; int OBS = LENGTH_SAV_OBS_BLOCK; int nOBS = OBSUnitsPerCase; dbgLog.fine("OBSUnitsPerCase=" + OBSUnitsPerCase); int caseIndex = 0; dbgLog.fine("printFormatTable:\n" + printFormatTable); variableFormatTypeList = new String[varQnty]; dateFormatList = new String[varQnty]; for (int i = 0; i < varQnty; i++) { variableFormatTypeList[i] = SPSSConstants.FORMAT_CATEGORY_TABLE .get(printFormatTable.get(variableNameList.get(i))); dbgLog.fine("i=" + i + "th variableFormatTypeList=" + variableFormatTypeList[i]); formatCategoryTable.put(variableNameList.get(i), variableFormatTypeList[i]); } dbgLog.fine("variableFormatType:\n" + Arrays.deepToString(variableFormatTypeList)); dbgLog.fine("formatCategoryTable:\n" + formatCategoryTable); int numberOfDecimalVariables = 0; // TODO: // Make sure the date formats are actually preserved! // (this is something that was collected in the code below and passed // to the UNF calculator). // -- L.A. 4.0 alpha List<String> casewiseRecordForTabFile = new ArrayList<String>(); // missing values are written to the tab-delimited file by // using the default or user-specified missing-value strings; // however, to calculate UNF/summary statistics, // classes for these calculations require their specific // missing values that differ from the above missing-value // strings; therefore, after row data for the tab-delimited // file are written, missing values in a row are changed to // UNF/summary-statistics-OK ones. // data-storage object for sumStat ///dataTable2 = new Object[varQnty][caseQnty]; // storage of date formats to pass to UNF ///dateFormats = new String[varQnty][caseQnty]; try { for (int i = 0;; i++) { // case-wise loop byte[] buffer = new byte[OBS * nOBS]; int nbytesuc = stream.read(buffer); StringBuilder sb_stringStorage = new StringBuilder(""); for (int k = 0; k < nOBS; k++) { int offset = OBS * k; // uncompressed case // numeric missing value == sysmis // FF FF FF FF FF FF eF FF(little endian) // string missing value // 20 20 20 20 20 20 20 20 // cf: compressed case // numeric type:sysmis == 0xFF // string type: missing value == 0xFE // boolean isNumeric = OBSwiseTypelList.get(k) == 0 ? true : false; if (isNumeric) { dbgLog.finer(k + "-th variable is numeric"); // interprete as double ByteBuffer bb_double = ByteBuffer.wrap(buffer, offset, LENGTH_SAV_OBS_BLOCK); if (isLittleEndian) { bb_double.order(ByteOrder.LITTLE_ENDIAN); } //char[] hexpattern = String dphex = new String(Hex.encodeHex( Arrays.copyOfRange(bb_double.array(), offset, offset + LENGTH_SAV_OBS_BLOCK))); dbgLog.finer("dphex=" + dphex); if ((dphex.equals("ffffffffffffefff")) || (dphex.equals("ffefffffffffffff"))) { //casewiseRecordForTabFile.add(systemMissingValue); // add the numeric missing value dbgLog.fine("SAV Reader: adding: Missing Value (numeric)"); casewiseRecordForTabFile.add(MissingValueForTextDataFileNumeric); } else { Double ddatum = bb_double.getDouble(); dbgLog.fine("SAV Reader: adding: ddatum=" + ddatum); // add this non-missing-value numeric datum casewiseRecordForTabFile.add(doubleNumberFormatter.format(ddatum)); } } else { dbgLog.finer(k + "-th variable is string"); // string case // strip space-padding // do not trim: string might have spaces within it // the missing value (hex) for a string variable is: // "20 20 20 20 20 20 20 20" String strdatum = new String( Arrays.copyOfRange(buffer, offset, (offset + LENGTH_SAV_OBS_BLOCK)), defaultCharSet); dbgLog.finer("str_datum=" + strdatum); // add this non-missing-value string datum casewiseRecordForTabFile.add(strdatum); } // if isNumeric } // k-loop // String-variable's continuous block exits: if (hasStringVarContinuousBlock) { // continuous blocks: string case // concatenating process //dbgLog.fine("concatenating process starts"); //dbgLog.fine("casewiseRecordForTabFile(before)="+casewiseRecordForTabFile); //dbgLog.fine("casewiseRecordForTabFile(before:size)="+casewiseRecordForTabFile.size()); StringBuilder sb = new StringBuilder(""); int firstPosition = 0; Set<Integer> removeJset = new HashSet<Integer>(); for (int j = 0; j < nOBS; j++) { dbgLog.finer("j=" + j + "-th type =" + OBSwiseTypelList.get(j)); if (OBSwiseTypelList.get(j) == -1) { // String continued fount at j-th // look back the j-1 firstPosition = j - 1; int lastJ = j; String concatanated = null; removeJset.add(j); sb.append(casewiseRecordForTabFile.get(j - 1)); sb.append(casewiseRecordForTabFile.get(j)); for (int jc = 1;; jc++) { if (OBSwiseTypelList.get(j + jc) != -1) { // j is the end unit of this string variable concatanated = sb.toString(); sb.setLength(0); lastJ = j + jc; break; } else { sb.append(casewiseRecordForTabFile.get(j + jc)); removeJset.add(j + jc); } } casewiseRecordForTabFile.set(j - 1, concatanated); //out.println(j-1+"th concatanated="+concatanated); j = lastJ - 1; } // end-of-if: continuous-OBS only } // end of loop-j List<String> newDataLine = new ArrayList<String>(); for (int jl = 0; jl < casewiseRecordForTabFile.size(); jl++) { //out.println("jl="+jl+"-th datum =["+casewiseRecordForTabFile.get(jl)+"]"); if (!removeJset.contains(jl)) { newDataLine.add(casewiseRecordForTabFile.get(jl)); } } dbgLog.fine("new casewiseRecordForTabFile=" + newDataLine); dbgLog.fine("new casewiseRecordForTabFile(size)=" + newDataLine.size()); casewiseRecordForTabFile = newDataLine; } // end-if: stringContinuousVar-exist case caseIndex++; dbgLog.finer("caseIndex=" + caseIndex); for (int k = 0; k < casewiseRecordForTabFile.size(); k++) { if (variableTypelList.get(k) > 0) { // See my comments for this padding removal logic // in the "compressed" method -- L.A. String paddRemoved = StringUtils.stripEnd(casewiseRecordForTabFile.get(k).toString(), null); // TODO: clean this up. For now, just make sure that strings contain at least one blank space. if (paddRemoved.equals("")) { paddRemoved = " "; } //casewiseRecordForTabFile.set(k, "\"" + paddRemoved.replaceAll("\"", Matcher.quoteReplacement("\\\"")) + "\""); casewiseRecordForTabFile.set(k, escapeCharacterString(paddRemoved)); // end of String var case } // end of variable-type check if (casewiseRecordForTabFile.get(k) != null && !casewiseRecordForTabFile.get(k).equals(MissingValueForTextDataFileNumeric)) { // to do date conversion String variableFormatType = variableFormatTypeList[k]; dbgLog.finer("k=" + k + "th variable format=" + variableFormatType); int formatDecimalPointPosition = formatDecimalPointPositionList.get(k); if (variableFormatType.equals("date")) { dbgLog.finer("date case"); long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymd.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); dateFormatList[k] = sdf_ymd.toPattern(); } else if (variableFormatType.equals("time")) { dbgLog.finer("time case:DTIME or DATETIME or TIME"); //formatCategoryTable.put(variableNameList.get(k), "time"); // not treating DTIME as date/time; see comment elsewhere in // the code; // (but we do need to remember to treat the resulting values // as character strings, not numerics!) if (printFormatTable.get(variableNameList.get(k)).equals("DTIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_BIAS; String newDatum = sdf_dhms.format(new Date(dateDatum)); // Note: DTIME is not a complete date, so we don't save a date format with it dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString().split("\\."); dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_BIAS; StringBuilder sb_time = new StringBuilder(sdf_dhms.format(new Date(dateDatum))); if (formatDecimalPointPosition > 0) { sb_time.append("." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); } } else if (printFormatTable.get(variableNameList.get(k)).equals("DATETIME")) { // TODO: // (for both datetime and "dateless" time) // keep the longest of the matching formats - i.e., if there are *some* // values in the vector that have thousands of a second, that should be // part of the saved format! // -- L.A. Aug. 12 2014 if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L - SPSS_DATE_OFFSET; String newDatum = sdf_ymdhms.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); dateFormatList[k] = sdf_ymdhms.toPattern(); } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString().split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L - SPSS_DATE_OFFSET; StringBuilder sb_time = new StringBuilder( sdf_ymdhms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append("." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); // datetime with milliseconds: dateFormatList[k] = sdf_ymdhms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); } } else if (printFormatTable.get(variableNameList.get(k)).equals("TIME")) { if (casewiseRecordForTabFile.get(k).toString().indexOf(".") < 0) { long dateDatum = Long.parseLong(casewiseRecordForTabFile.get(k).toString()) * 1000L; String newDatum = sdf_hms.format(new Date(dateDatum)); dbgLog.finer("k=" + k + ":" + newDatum); casewiseRecordForTabFile.set(k, newDatum); if (dateFormatList[k] == null) { dateFormatList[k] = sdf_hms.toPattern(); } } else { // decimal point included String[] timeData = casewiseRecordForTabFile.get(k).toString().split("\\."); //dbgLog.finer(StringUtils.join(timeData, "|")); long dateDatum = Long.parseLong(timeData[0]) * 1000L; StringBuilder sb_time = new StringBuilder(sdf_hms.format(new Date(dateDatum))); //dbgLog.finer(sb_time.toString()); if (formatDecimalPointPosition > 0) { sb_time.append("." + timeData[1].substring(0, formatDecimalPointPosition)); } dbgLog.finer("k=" + k + ":" + sb_time.toString()); casewiseRecordForTabFile.set(k, sb_time.toString()); // time, possibly with milliseconds: String format_hmsS = sdf_hms.toPattern() + (formatDecimalPointPosition > 0 ? ".S" : ""); if (dateFormatList[k] == null || (format_hmsS.length() > dateFormatList[k].length())) { dateFormatList[k] = format_hmsS; } } } } else if (variableFormatType.equals("other")) { dbgLog.finer("other non-date/time case"); if (printFormatTable.get(variableNameList.get(k)).equals("WKDAY")) { // day of week dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.WEEKDAY_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.WEEKDAY_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); dbgLog.finer("wkday:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } else if (printFormatTable.get(variableNameList.get(k)).equals("MONTH")) { // month dbgLog.finer("data k=" + k + ":" + casewiseRecordForTabFile.get(k)); dbgLog.finer("data k=" + k + ":" + SPSSConstants.MONTH_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1)); String newDatum = SPSSConstants.MONTH_LIST .get(Integer.valueOf(casewiseRecordForTabFile.get(k).toString()) - 1); casewiseRecordForTabFile.set(k, newDatum); dbgLog.finer("month:k=" + k + ":" + casewiseRecordForTabFile.get(k)); } } // end of date/time block } // end: date-time-datum check } // end: loop-k(2nd: variablte-wise-check) // write to tab file if (casewiseRecordForTabFile.size() > 0) { pwout.println(StringUtils.join(casewiseRecordForTabFile, "\t")); } // numeric contents-check for (int l = 0; l < casewiseRecordForTabFile.size(); l++) { if (variableFormatTypeList[l].equals("date") || variableFormatTypeList[l].equals("time") || printFormatTable.get(variableNameList.get(l)).equals("WKDAY") || printFormatTable.get(variableNameList.get(l)).equals("MONTH")) { } else { if (variableTypelList.get(l) <= 0) { if (casewiseRecordForTabFile.get(l).toString().indexOf(".") >= 0) { decimalVariableSet.add(l); } } } } // reset the case-wise working objects casewiseRecordForTabFile.clear(); if (stream.available() == 0) { // reached the end of this file // do exit-processing dbgLog.fine("reached the end of the file at " + ii + "th iteration"); break; } // if eof processing } //i-loop: case(row) iteration // close the writer pwout.close(); } catch (IOException ex) { throw ex; } // contents check dbgLog.fine("numberOfDecimalVariables=" + numberOfDecimalVariables); dbgLog.fine("decimalVariableSet=" + decimalVariableSet); dbgLog.fine("***** decodeRecordTypeDataUnCompressed(): end *****"); }
From source file:net.vnpt.tr069.handler.ADSLHandler.java
@Override public void processGetParameterValuesResponse(HttpServletRequest request, HttpServletResponse response, Message msg, ByteArrayOutputStream out) throws Exception { HttpSession session = request.getSession(); Inform lastInform = (Inform) session.getAttribute(ATTR_LASTINFORM); StringBuilder sb = new StringBuilder(100); GetParameterValuesResponse dataReceive = (GetParameterValuesResponse) msg; String lastEvent = (String) session.getAttribute(ATTR_LAST_EVENT); if (TR069StaticParameter.EVENT_0.equals(lastEvent) || TR069StaticParameter.EVENT_1.equals(lastEvent)) { System.out.println("SAVED.........."); saveDeviceInfo(session, dataReceive, lastInform); } else if (TR069StaticParameter.EVENT_2.equals(lastEvent)) { // periodic session Command cm = CommandConfigurationFactory.getCommand(lastInform.sn); if (cm == null) { return; }//w w w . j a v a 2 s . com logger.info("GetParameterValuesResponse with CommandType: " + cm.getType()); if (cm.getType().equals(Command.TYPE_PERIODIC)) { enqueueUpdateFirmware(lastInform); PeriodicCommand periodicCmd = (PeriodicCommand) cm; HashMap hm_data = new HashMap(dataReceive.values); String KeyCheck = "InternetGatewayDevice.Layer2Bridging.AvailableInterface."; String ValueCheck = "InternetGatewayDevice.WANDevice.1.WANConnectionDevice."; List<String> ListWanGet = new ArrayList<String>(16); if (periodicCmd.orderCmd == 0) { //periodicCmd.orderCmd = 1; periodicCmd.orderCmd = 3; //detect version String ontModelName = dataReceive.values.get(TR069StaticParameter.DeviceInfoModelName); // list chua cac tham so tr069 can lay // performance cua lan + wire lan + wan + optical List<String> listParams = new ArrayList<String>(150); // lay danh sach cac wan interface for (Object variableName : hm_data.keySet()) { String Key = variableName.toString(); String Value = hm_data.get(variableName).toString(); if (Value.contains(ValueCheck) && Key.contains(KeyCheck)) { ListWanGet.add(Value); } } // //Kiem tra trong file GET_WANPERFORMANCE lay ra tat ca cac gia tri DataFileModel wanDataFileModel = DataFileModel.createDataFileModel(FilePath.GET_WANPERFORMANCE, cm.getModelName()); List<SimpleObject> WanData = wanDataFileModel.getDataFile(); //Tong hop cac gia tri va cho vao ListWanFull for (String strWanItem_First : ListWanGet) { for (SimpleObject WanData1 : WanData) { String strWanItem_End = WanData1.getParameter(); // posfix of wan parameter String strWanFull = strWanItem_First + "." + strWanItem_End; // full path wan parameter of specific wan interface listParams.add(strWanFull); } } cm.setModelName(ontModelName); //add lan performance periodicCmd.addDataToRoot(dataReceive.values); // get Performance DataFileModel dataModel = DataFileModel.createDataFileModel(FilePath.Get_DevicePerformance, cm.getModelName()); ArrayList<SimpleObject> listDevicePerformance = dataModel.getDataModel(); for (SimpleObject item : listDevicePerformance) { listParams.add(item.getParameter()); } //get wlan performance if (listParams.size() > 0) { logger.info("Get_DevicePerformance: " + listParams.size()); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out); } else { logger.warn("Get_DevicePerformance: Not run"); } } else if (periodicCmd.orderCmd == 1) { //periodicCmd.orderCmd = 2; // DSL Performance not belong to GPON Device! --> Skip step 2 periodicCmd.orderCmd = 3; //add wlan performance periodicCmd.addDataToRoot(dataReceive.values); // checkWifi is enabled ? String dataTree = "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.Enable"; String wifiIsEnabled = dataReceive.values.get(dataTree); periodicCmd.getKeyMap().put(UpdateFirmwareCommand.UPDATE_FW_WIFI_STATE, wifiIsEnabled); //get wan performance & opticalInfo //String treeIndex = AppConfig.isDEV_V2(cm.getModelName()) ? "5" : "3"; DataFileModel dataModel = DataFileModel.createDataFileModel(FilePath.GET_WAN_SERVICE, cm.getModelName()); SimpleObject root = dataModel.getRoot(); String wanPerformancePath = StringUtils.stripEnd(root.getParameter(), ".") + "."; sb.setLength(0); sb.append("PeriodicCommand: DeviceModelName=").append(cm.getModelName()) .append(", SerialNumber=").append(lastInform.sn).append(", wanPerformancePath=") .append(wanPerformancePath); logger.info(sb); Map<String, String> listParams = new HashMap<String, String>(); if (AppConfig.isDEV_V2(cm.getModelName())) { // only get OpticalInfo in V2 that supported logger.info(cm.getType() + ": ONT V2 is supported !"); listParams.put(TR069StaticParameter.OpticalInfo, "OpticalInfo"); } listParams.put(wanPerformancePath, "wanPerformancePath"); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out); } else if (periodicCmd.orderCmd == 2) { periodicCmd.orderCmd = 3; //add wan performance periodicCmd.addDataToRoot(dataReceive.values); //get dsl performance GetParameterValues getParaVl = new GetParameterValues(TR069StaticParameter.DslPerformancePath); getParaVl.writeTo(out); } else if (periodicCmd.orderCmd == 3) { periodicCmd.orderCmd = 4; //add dsl performance try { ///hoangtuan check update alarm // long beforeEnque = System.currentTimeMillis(); HashMap hm_datasend = new HashMap(dataReceive.values); AlarmData alarm = new AlarmData(); alarm.hashdata = hm_datasend; alarm.serial = lastInform.sn; Global.enqueueAlarm(alarm); } catch (Exception e) { logger.error("error insert data performance to database", e); } // hoangtuan: bo update Firmware chuyen sang jms + firmware Policy // Neu co yeu cau Update Firmware trong // hoangtuan check // Command cmUpdate = JMSRequestUpdateFirmware.getUpdateFirmwareRequest().remove(lastInform.sn); // if (cmUpdate != null) { // UpdateFirmwareCommand updateFirmwareInfo = (UpdateFirmwareCommand) cmUpdate; // String urlFileFirmware = updateFirmwareInfo.getUrlFileServer(); // // Send message command // Download recm = new Download("", urlFileFirmware, Download.FT_FIRMWARE); // recm.writeTo(out); // // // Remove from request // } else { // AdslDevice deviceInfo = new AdslDevice(); // deviceInfo.setAdslSerialNumber(lastInform.sn); // deviceInfo.setSerialNumber(lastInform.sn); // deviceInfo.setManufacturer(lastInform.Manufacturer); // deviceInfo.setOui(lastInform.getOui()); // deviceInfo.setProductClass(lastInform.ProductClass); // deviceInfo.setHardwareVersion(lastInform.getHardwareVersion()); // deviceInfo.setSoftwareVersion(lastInform.getSoftwareVersion()); // deviceInfo.setProvisioningCode(lastInform.getProvisiongCode()); // deviceInfo.setConnectionRequest(lastInform.getConnectionRequestURL()); // String ipAddress = DataFileUtils.getIPfromConnectionRequest(lastInform.getConnectionRequestURL()); // deviceInfo.setIpAddress(ipAddress); // deviceInfo.setName(lastInform.sn); // deviceInfo.setShortName(lastInform.sn); // // Global.enqueueUpdateFw(deviceInfo); // } } } else if (cm.getType().equals(Command.TYPE_ZERO_TOUCH)) { ZeroTouchCommand_GPON zeroCmd = (ZeroTouchCommand_GPON) cm; if (zeroCmd.orderCmd == 0) { // Get device info (pre-step) // DONOT: Increase OrderCmd AdslDevice device = ADSL_DEVICE_DAO.getByAdslSerialNumber(lastInform.sn); String ontModelName = dataReceive.values.get(TR069StaticParameter.DeviceInfoModelName); cm.setModelName(ontModelName); if (device != null) { int ontVersion = AppConfig.getDeviceVersion(ontModelName); sb.setLength(0); sb.append("ZeroTouch Update Device Version: ONT_VERSION=").append(ontVersion) .append(", MODEL_NAME=").append(ontModelName); logger.info(sb); device.setModelName(ontModelName); device.setDeviceVersion(ontVersion); ADSL_DEVICE_DAO.save(device); } // Config PeriodicInformEnable String intervalInform = zeroCmd.getIntervalInform(); SetParameterValues recm = new SetParameterValues(); recm.AddValue(TR069StaticParameter.PeriodicInformEnable, "1", "xsd:boolean"); recm.AddValue(TR069StaticParameter.PeriodicInformInterval, intervalInform, "xsd:unsignedInt"); recm.writeTo(out); } else if (zeroCmd.orderCmd == 1) { //zeroCmd.orderCmd = 2; //--> not create Layer 2 --> use default instance 1 zeroCmd.orderCmd = 4; //add ATM object try { zeroCmd.setRootDevice(dataReceive.values); // zeroCmd.setACSServer(dataReceive.getParam(TR069StaticParameter.ManagementURL)); } catch (Exception e) { logger.error("ZeroTouch, Step 2: " + e.getMessage(), e); } //AddObject recm = new AddObject(zeroCmd.getRootNameLAN() + ".", ""); //recm.writeTo(out); sb.setLength(0); sb.append(cm.getType()).append(", create WANPPPConnection").append(", serialNumber=") .append(lastInform.sn).append(", modelName=").append(cm.getModelName()); logger.info(sb); AddObject recm = new AddObject(zeroCmd.getRootNameLAN() + ".1." + "WANPPPConnection.", ""); recm.writeTo(out); } else if (zeroCmd.orderCmd == 6) { zeroCmd.orderCmd = 7; try { // zeroCmd.setACSServer(dataReceive.getParam(TR069StaticParameter.ManagementURL)); zeroCmd.setACSManagementServer(dataReceive.values); } catch (Exception e) { logger.error("ZeroTouch, Step 7: " + e.getMessage(), e); } //add static route // Vunb 2015-03-04: Remove static route --> add only default gateway AddObject recm = new AddObject(zeroCmd.rootNameForwarding + ".", ""); recm.writeTo(out); } } // } else if (TR069StaticParameter.EVENT_6.equals(lastEvent)) { // connection request sessionF Command cm = CommandRequestFactory.getCommand(lastInform.sn); if (cm != null) { logger.info("GetParameterValuesResponse with CommandType: " + cm.getType()); if (cm.getType().equals(Command.TYPE_GETTREEVALUE)) { GetValueCommand getVlCmd = (GetValueCommand) cm; getVlCmd.setReturnValue(dataReceive.values); getVlCmd.receiveResult(); } else if (cm.getType().equals(Command.TYPE_ADDOBJECT)) { AddObjectCommand addCmd = (AddObjectCommand) cm; addCmd.ifnameObject.setValue(dataReceive.getParam(addCmd.ifnameObject.getParameter())); addCmd.receiveResult(); } } } }
From source file:net.vnpt.tr069.handler.AONHandler.java
@Override public void processGetParameterValuesResponse(HttpServletRequest request, HttpServletResponse response, Message msg, ByteArrayOutputStream out) throws Exception { HttpSession session = request.getSession(); Inform lastInform = (Inform) session.getAttribute(ATTR_LASTINFORM); StringBuilder sb = new StringBuilder(100); GetParameterValuesResponse dataReceive = (GetParameterValuesResponse) msg; String lastEvent = (String) session.getAttribute(ATTR_LAST_EVENT); if (TR069StaticParameter.EVENT_0.equals(lastEvent) || TR069StaticParameter.EVENT_1.equals(lastEvent)) { saveDeviceInfo(session, dataReceive, lastInform); } else if (TR069StaticParameter.EVENT_2.equals(lastEvent)) { // periodic session Command cm = CommandConfigurationFactory.getCommand(lastInform.sn); if (cm == null) { return; }//ww w . java2 s . c om logger.info("GetParameterValuesResponse with CommandType: " + cm.getType()); if (cm.getType().equals(Command.TYPE_PERIODIC)) { enqueueUpdateFirmware(lastInform); PeriodicCommand periodicCmd = (PeriodicCommand) cm; HashMap<String, String> hm_data = new HashMap(dataReceive.values); String KeyCheck = "InternetGatewayDevice.Layer2Bridging.AvailableInterface."; String ValueCheck_Wan = "InternetGatewayDevice.WANDevice.3.WANConnectionDevice."; List<String> ListWanGet = new ArrayList<String>(16); String ValueCheck_Lan = "InternetGatewayDevice.LANDevice.1.LANEthernetInterfaceConfig."; List<String> ListLanGet = new ArrayList<String>(10); String ValueCheck_Wlan = "InternetGatewayDevice.LANDevice.1.WLANConfiguration."; List<String> ListWlanGet = new ArrayList<String>(10); if (periodicCmd.orderCmd == 0) { //periodicCmd.orderCmd = 1; periodicCmd.orderCmd = 3; //detect version String ontModelName = dataReceive.values.get(TR069StaticParameter.DeviceInfoModelName); // list chua cac tham so tr069 can lay // performance cua lan + wire lan + wan + optical List<String> listParams = new ArrayList<String>(150); // lay danh sach cac wan interface for (Map.Entry<String, String> entry : hm_data.entrySet()) { String Key = entry.getKey(); String Value = entry.getValue(); //Wan if (Value.contains(ValueCheck_Wan) && Key.contains(KeyCheck)) { ListWanGet.add(Value); } //Lan if (Value.contains(ValueCheck_Lan) && Key.contains(KeyCheck)) { ListLanGet.add(Value); } //Wlan if (Value.contains(ValueCheck_Wlan) && Key.contains(KeyCheck)) { ListWlanGet.add(Value); } } // List<String> listFullParam = getFullParam(ListLanGet, ListWlanGet, ListWanGet); for (String strValue : listFullParam) { listParams.add(strValue); } cm.setModelName(ontModelName); //add lan performance periodicCmd.addDataToRoot(dataReceive.values); // get Performance DataFileModel dataModel = DataFileModel.createDataFileModel(FilePath.Get_DevicePerformance, cm.getModelName()); ArrayList<SimpleObject> listDevicePerformance = dataModel.getDataModel(); for (SimpleObject item : listDevicePerformance) { listParams.add(item.getParameter()); } //get wlan performance if (listParams.size() > 0) { logger.info("Get_DevicePerformance: " + listParams.size()); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out); } else { logger.warn("Get_DevicePerformance: Not run"); } } else if (periodicCmd.orderCmd == 1) { //periodicCmd.orderCmd = 2; // DSL Performance not belong to GPON Device! --> Skip step 2 periodicCmd.orderCmd = 3; //add wlan performance periodicCmd.addDataToRoot(dataReceive.values); // checkWifi is enabled ? String dataTree = "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.Enable"; String wifiIsEnabled = dataReceive.values.get(dataTree); periodicCmd.getKeyMap().put(UpdateFirmwareCommand.UPDATE_FW_WIFI_STATE, wifiIsEnabled); //get wan performance & opticalInfo //String treeIndex = AppConfig.isDEV_V2(cm.getModelName()) ? "5" : "3"; DataFileModel dataModel = DataFileModel.createDataFileModel(FilePath.GET_WAN_SERVICE, cm.getModelName()); SimpleObject root = dataModel.getRoot(); String wanPerformancePath = StringUtils.stripEnd(root.getParameter(), ".") + "."; sb.setLength(0); sb.append("PeriodicCommand: DeviceModelName=").append(cm.getModelName()) .append(", SerialNumber=").append(lastInform.sn).append(", wanPerformancePath=") .append(wanPerformancePath); logger.info(sb); Map<String, String> listParams = new HashMap<String, String>(); if (AppConfig.isDEV_V2(cm.getModelName())) { // only get OpticalInfo in V2 that supported logger.info(cm.getType() + ": ONT V2 is supported !"); listParams.put(TR069StaticParameter.OpticalInfo, "OpticalInfo"); } listParams.put(wanPerformancePath, "wanPerformancePath"); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out); } else if (periodicCmd.orderCmd == 2) { periodicCmd.orderCmd = 3; //add wan performance periodicCmd.addDataToRoot(dataReceive.values); //get dsl performance GetParameterValues getParaVl = new GetParameterValues(TR069StaticParameter.DslPerformancePath); getParaVl.writeTo(out); } else if (periodicCmd.orderCmd == 3) { periodicCmd.orderCmd = 4; //add dsl performance try { ///hoangtuan check update alarm // long beforeEnque = System.currentTimeMillis(); HashMap hm_datasend = new HashMap(dataReceive.values); AlarmData alarm = new AlarmData(); alarm.hashdata = hm_datasend; alarm.serial = lastInform.sn; Global.enqueueAlarm(alarm); } catch (Exception e) { logger.error("error insert data performance to database", e); } // hoangtuan: bo update Firmware chuyen sang jms + firmware Policy // Neu co yeu cau Update Firmware trong // hoangtuan check // Command cmUpdate = JMSRequestUpdateFirmware.getUpdateFirmwareRequest().remove(lastInform.sn); // if (cmUpdate != null) { // UpdateFirmwareCommand updateFirmwareInfo = (UpdateFirmwareCommand) cmUpdate; // String urlFileFirmware = updateFirmwareInfo.getUrlFileServer(); // // Send message command // Download recm = new Download("", urlFileFirmware, Download.FT_FIRMWARE); // recm.writeTo(out); // // // Remove from request // } else { // AdslDevice deviceInfo = new AdslDevice(); // deviceInfo.setAdslSerialNumber(lastInform.sn); // deviceInfo.setSerialNumber(lastInform.sn); // deviceInfo.setManufacturer(lastInform.Manufacturer); // deviceInfo.setOui(lastInform.getOui()); // deviceInfo.setProductClass(lastInform.ProductClass); // deviceInfo.setHardwareVersion(lastInform.getHardwareVersion()); // deviceInfo.setSoftwareVersion(lastInform.getSoftwareVersion()); // deviceInfo.setProvisioningCode(lastInform.getProvisiongCode()); // deviceInfo.setConnectionRequest(lastInform.getConnectionRequestURL()); // String ipAddress = DataFileUtils.getIPfromConnectionRequest(lastInform.getConnectionRequestURL()); // deviceInfo.setIpAddress(ipAddress); // deviceInfo.setName(lastInform.sn); // deviceInfo.setShortName(lastInform.sn); // // Global.enqueueUpdateFw(deviceInfo); // } } } else if (cm.getType().equals(Command.TYPE_ZERO_TOUCH)) { ZeroTouchCommand_GPON zeroCmd = (ZeroTouchCommand_GPON) cm; if (zeroCmd.orderCmd == 0) { // Get device info (pre-step) // DONOT: Increase OrderCmd AdslDevice device = DatabaseManager.ADSL_DEVICE_DAO.getByAdslSerialNumber(lastInform.sn); String ontModelName = dataReceive.values.get(TR069StaticParameter.DeviceInfoModelName); cm.setModelName(ontModelName); if (device != null) { int ontVersion = AppConfig.getDeviceVersion(ontModelName); sb.setLength(0); sb.append("ZeroTouch Update Device Version: ONT_VERSION=").append(ontVersion) .append(", MODEL_NAME=").append(ontModelName); logger.info(sb); device.setModelName(ontModelName); device.setDeviceVersion(ontVersion); DatabaseManager.ADSL_DEVICE_DAO.save(device); } // Config PeriodicInformEnable String intervalInform = zeroCmd.getIntervalInform(); SetParameterValues recm = new SetParameterValues(); recm.AddValue(TR069StaticParameter.PeriodicInformEnable, "1", "xsd:boolean"); recm.AddValue(TR069StaticParameter.PeriodicInformInterval, intervalInform, "xsd:unsignedInt"); recm.writeTo(out); } else if (zeroCmd.orderCmd == 1) { //zeroCmd.orderCmd = 2; //--> not create Layer 2 --> use default instance 1 zeroCmd.orderCmd = 4; //add ATM object try { zeroCmd.setRootDevice(dataReceive.values); // zeroCmd.setACSServer(dataReceive.getParam(TR069StaticParameter.ManagementURL)); } catch (Exception e) { logger.error("ZeroTouch, Step 2: " + e.getMessage(), e); } //AddObject recm = new AddObject(zeroCmd.getRootNameLAN() + ".", ""); //recm.writeTo(out); sb.setLength(0); sb.append(cm.getType()).append(", create WANPPPConnection").append(", serialNumber=") .append(lastInform.sn).append(", modelName=").append(cm.getModelName()); logger.info(sb); AddObject recm = new AddObject(zeroCmd.getRootNameLAN() + ".1." + "WANPPPConnection.", ""); recm.writeTo(out); } else if (zeroCmd.orderCmd == 6) { zeroCmd.orderCmd = 7; try { // zeroCmd.setACSServer(dataReceive.getParam(TR069StaticParameter.ManagementURL)); zeroCmd.setACSManagementServer(dataReceive.values); } catch (Exception e) { logger.error("ZeroTouch, Step 7: " + e.getMessage(), e); } //add static route // Vunb 2015-03-04: Remove static route --> add only default gateway AddObject recm = new AddObject(zeroCmd.rootNameForwarding + ".", ""); recm.writeTo(out); } } // } else if (TR069StaticParameter.EVENT_6.equals(lastEvent)) { // connection request sessionF Command cm = CommandRequestFactory.getCommand(lastInform.sn); if (cm != null) { logger.info("GetParameterValuesResponse with CommandType: " + cm.getType()); if (cm.getType().equals(Command.TYPE_GETTREEVALUE)) { GetValueCommand getVlCmd = (GetValueCommand) cm; getVlCmd.setReturnValue(dataReceive.values); getVlCmd.receiveResult(); } else if (cm.getType().equals(Command.TYPE_ADDOBJECT)) { AddObjectCommand addCmd = (AddObjectCommand) cm; addCmd.ifnameObject.setValue(dataReceive.getParam(addCmd.ifnameObject.getParameter())); addCmd.receiveResult(); } } } }
From source file:net.vnpt.tr069.handler.ONTv2Handler.java
@Override public void processGetParameterValuesResponse(HttpServletRequest request, HttpServletResponse response, Message msg, ByteArrayOutputStream out) throws Exception { HttpSession session = request.getSession(); Inform lastInform = (Inform) session.getAttribute(ATTR_LASTINFORM); StringBuilder sb = new StringBuilder(100); GetParameterValuesResponse dataReceive = (GetParameterValuesResponse) msg; String lastEvent = (String) session.getAttribute(ATTR_LAST_EVENT); if (TR069StaticParameter.EVENT_0.equals(lastEvent)) { String stepOrder = (String) session.getAttribute(ATTR_STEP_ORDER); if (stepOrder == null) { saveDeviceInfo(session, dataReceive, lastInform); List<String> listParams = new ArrayList<String>(1); listParams.add(TR069StaticParameter.GPON_INFO_MEN_NUMBER); session.setAttribute(ATTR_STEP_ORDER, "getMenNumber"); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out);/*w w w.ja v a 2s . co m*/ } else { updateCPEInfo(session, dataReceive, lastInform); startConfigDevice(session, dataReceive, lastInform); } } else if (TR069StaticParameter.EVENT_1.equals(lastEvent)) { String stepOrder = (String) session.getAttribute(ATTR_STEP_ORDER); if (stepOrder == null) { saveDeviceInfo(session, dataReceive, lastInform); List<String> listParams = new ArrayList<String>(1); listParams.add(TR069StaticParameter.GPON_INFO_MEN_NUMBER); session.setAttribute(ATTR_STEP_ORDER, "getMenNumber"); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out); } else { updateCPEInfo(session, dataReceive, lastInform); } } else if (TR069StaticParameter.EVENT_2.equals(lastEvent)) { // periodic session Command cm = CommandConfigurationFactory.getCommand(lastInform.sn); if (cm == null) { return; } logger.info("GetParameterValuesResponse with CommandType: " + cm.getType()); if (cm.getType().equals(Command.TYPE_PERIODIC)) { enqueueUpdateFirmware(lastInform); PeriodicCommand periodicCmd = (PeriodicCommand) cm; HashMap<String, String> hm_data = new HashMap(dataReceive.values); String KeyCheck = "InternetGatewayDevice.Layer2Bridging.AvailableInterface."; String ValueCheck_Wan = "InternetGatewayDevice.WANDevice.5.WANConnectionDevice."; List<String> ListWanGet = new ArrayList<String>(16); String ValueCheck_Lan = "InternetGatewayDevice.LANDevice.1.LANEthernetInterfaceConfig."; List<String> ListLanGet = new ArrayList<String>(10); String ValueCheck_Wlan = "InternetGatewayDevice.LANDevice.1.WLANConfiguration."; List<String> ListWlanGet = new ArrayList<String>(10); if (periodicCmd.orderCmd == 0) { //periodicCmd.orderCmd = 1; periodicCmd.orderCmd = 3; //detect version String ontModelName = dataReceive.values.get(TR069StaticParameter.DeviceInfoModelName); // list chua cac tham so tr069 can lay // performance cua lan + wire lan + wan + optical List<String> listParams = new ArrayList<String>(150); // lay danh sach cac wan interface for (Map.Entry<String, String> entry : hm_data.entrySet()) { String Key = entry.getKey(); String Value = entry.getValue(); //Wan if (Value.contains(ValueCheck_Wan) && Key.contains(KeyCheck)) { ListWanGet.add(Value); } //Lan if (Value.contains(ValueCheck_Lan) && Key.contains(KeyCheck)) { ListLanGet.add(Value); } //Wlan if (Value.contains(ValueCheck_Wlan) && Key.contains(KeyCheck)) { ListWlanGet.add(Value); } } // List<String> listFullParam = getFullParam(ListLanGet, ListWlanGet, ListWanGet); for (String strValue : listFullParam) { listParams.add(strValue); } cm.setModelName(ontModelName); //add lan performance periodicCmd.addDataToRoot(dataReceive.values); // get Performance DataFileModel dataModel = DataFileModel.createDataFileModel(FilePath.Get_DevicePerformance, cm.getModelName()); ArrayList<SimpleObject> listDevicePerformance = dataModel.getDataModel(); for (SimpleObject item : listDevicePerformance) { listParams.add(item.getParameter()); } //get wlan performance if (listParams.size() > 0) { logger.info("Get_DevicePerformance: " + listParams.size()); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out); } else { logger.warn("Get_DevicePerformance: Not run"); } } else if (periodicCmd.orderCmd == 1) { //periodicCmd.orderCmd = 2; // DSL Performance not belong to GPON Device! --> Skip step 2 periodicCmd.orderCmd = 3; //add wlan performance periodicCmd.addDataToRoot(dataReceive.values); // checkWifi is enabled ? String dataTree = "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.Enable"; String wifiIsEnabled = dataReceive.values.get(dataTree); periodicCmd.getKeyMap().put(UpdateFirmwareCommand.UPDATE_FW_WIFI_STATE, wifiIsEnabled); //get wan performance & opticalInfo //String treeIndex = AppConfig.isDEV_V2(cm.getModelName()) ? "5" : "3"; DataFileModel dataModel = DataFileModel.createDataFileModel(FilePath.GET_WAN_SERVICE, cm.getModelName()); SimpleObject root = dataModel.getRoot(); String wanPerformancePath = StringUtils.stripEnd(root.getParameter(), ".") + "."; sb.setLength(0); sb.append("PeriodicCommand: DeviceModelName=").append(cm.getModelName()) .append(", SerialNumber=").append(lastInform.sn).append(", wanPerformancePath=") .append(wanPerformancePath); logger.info(sb); Map<String, String> listParams = new HashMap<String, String>(); if (AppConfig.isDEV_V2(cm.getModelName())) { // only get OpticalInfo in V2 that supported logger.info(cm.getType() + ": ONT V2 is supported !"); listParams.put(TR069StaticParameter.OpticalInfo, "OpticalInfo"); } listParams.put(wanPerformancePath, "wanPerformancePath"); GetParameterValues getParaVl = new GetParameterValues(listParams); getParaVl.writeTo(out); } else if (periodicCmd.orderCmd == 2) { periodicCmd.orderCmd = 3; //add wan performance periodicCmd.addDataToRoot(dataReceive.values); //get dsl performance GetParameterValues getParaVl = new GetParameterValues(TR069StaticParameter.DslPerformancePath); getParaVl.writeTo(out); } else if (periodicCmd.orderCmd == 3) { periodicCmd.orderCmd = 4; //add dsl performance try { ///hoangtuan check update alarm // long beforeEnque = System.currentTimeMillis(); HashMap hm_datasend = new HashMap(dataReceive.values); AlarmData alarm = new AlarmData(); alarm.hashdata = hm_datasend; alarm.serial = lastInform.sn; Global.enqueueAlarm(alarm); } catch (Exception e) { logger.error("error insert data performance to database", e); } // hoangtuan: bo update Firmware chuyen sang jms + firmware Policy // Neu co yeu cau Update Firmware trong // hoangtuan check // Command cmUpdate = JMSRequestUpdateFirmware.getUpdateFirmwareRequest().remove(lastInform.sn); // if (cmUpdate != null) { // UpdateFirmwareCommand updateFirmwareInfo = (UpdateFirmwareCommand) cmUpdate; // String urlFileFirmware = updateFirmwareInfo.getUrlFileServer(); // // Send message command // Download recm = new Download("", urlFileFirmware, Download.FT_FIRMWARE); // recm.writeTo(out); // // // Remove from request // } else { // } } } else if (cm.getType().equals(Command.TYPE_ZERO_TOUCH)) { ZeroTouchCommand_GPON zeroCmd = (ZeroTouchCommand_GPON) cm; if (zeroCmd.orderCmd == 0) { // Get device info (pre-step) // DONOT: Increase OrderCmd AdslDevice device = DatabaseManager.ADSL_DEVICE_DAO.getByAdslSerialNumber(lastInform.sn); String ontModelName = dataReceive.values.get(TR069StaticParameter.DeviceInfoModelName); cm.setModelName(ontModelName); if (device != null) { int ontVersion = AppConfig.getDeviceVersion(ontModelName); sb.setLength(0); sb.append("ZeroTouch Update Device Version: ONT_VERSION=").append(ontVersion) .append(", MODEL_NAME=").append(ontModelName); logger.info(sb); device.setModelName(ontModelName); device.setDeviceVersion(ontVersion); DatabaseManager.ADSL_DEVICE_DAO.save(device); } // Config PeriodicInformEnable String intervalInform = zeroCmd.getIntervalInform(); SetParameterValues recm = new SetParameterValues(); recm.AddValue(TR069StaticParameter.PeriodicInformEnable, "1", "xsd:boolean"); recm.AddValue(TR069StaticParameter.PeriodicInformInterval, intervalInform, "xsd:unsignedInt"); recm.writeTo(out); } else if (zeroCmd.orderCmd == 1) { //zeroCmd.orderCmd = 2; //--> not create Layer 2 --> use default instance 1 zeroCmd.orderCmd = 4; //add ATM object try { zeroCmd.setRootDevice(dataReceive.values); // zeroCmd.setACSServer(dataReceive.getParam(TR069StaticParameter.ManagementURL)); } catch (Exception e) { logger.error("ZeroTouch, Step 2: " + e.getMessage(), e); } //AddObject recm = new AddObject(zeroCmd.getRootNameLAN() + ".", ""); //recm.writeTo(out); sb.setLength(0); sb.append(cm.getType()).append(", create WANPPPConnection").append(", serialNumber=") .append(lastInform.sn).append(", modelName=").append(cm.getModelName()); logger.info(sb); AddObject recm = new AddObject(zeroCmd.getRootNameLAN() + ".1." + "WANPPPConnection.", ""); recm.writeTo(out); } else if (zeroCmd.orderCmd == 6) { zeroCmd.orderCmd = 7; try { // zeroCmd.setACSServer(dataReceive.getParam(TR069StaticParameter.ManagementURL)); zeroCmd.setACSManagementServer(dataReceive.values); } catch (Exception e) { logger.error("ZeroTouch, Step 7: " + e.getMessage(), e); } //add static route // Vunb 2015-03-04: Remove static route --> add only default gateway AddObject recm = new AddObject(zeroCmd.rootNameForwarding + ".", ""); recm.writeTo(out); } } // } else if (TR069StaticParameter.EVENT_6.equals(lastEvent)) { // connection request sessionF Command cm = CommandRequestFactory.getCommand(lastInform.sn); if (cm != null) { logger.info("GetParameterValuesResponse with CommandType: " + cm.getType()); if (cm.getType().equals(Command.TYPE_GETTREEVALUE)) { GetValueCommand getVlCmd = (GetValueCommand) cm; getVlCmd.setReturnValue(dataReceive.values); getVlCmd.receiveResult(); } else if (cm.getType().equals(Command.TYPE_ADDOBJECT)) { AddObjectCommand addCmd = (AddObjectCommand) cm; addCmd.ifnameObject.setValue(dataReceive.getParam(addCmd.ifnameObject.getParameter())); addCmd.receiveResult(); } } } }
From source file:nl.tranquilizedquality.itest.cargo.AbstractInstalledContainerUtil.java
/** * Installs the container and the application configuration. It also sets * some system properties so the container can startup properly. Finally it * sets up additional configuration like jndi.proprties files etc. */// www . j ava 2 s . c om protected void setupContainer() { if (LOGGER.isInfoEnabled()) { LOGGER.info("Cleaning up " + containerName + "..."); } // In windows the renaming causes problem when: // - The zip file has not the same name of the installed directory. // - The ZipURLInstaller fails. final String operatingSystem = System.getProperty("os.name"); if (operatingSystem != null && !operatingSystem.startsWith("Windows")) { try { new File(containerHome).mkdir(); } catch (final Exception exceptionOnMkDir) { if (LOGGER.isErrorEnabled()) { LOGGER.error("Failed to create the directory: " + containerHome + ". Details: " + exceptionOnMkDir.getMessage(), exceptionOnMkDir); } throw new ConfigurationException("Failed to create the directory: " + containerHome + ". Details: " + exceptionOnMkDir.getMessage(), exceptionOnMkDir); } } if (LOGGER.isInfoEnabled()) { LOGGER.info("Installing " + containerName + "..."); LOGGER.info("Downloading container from: " + remoteLocation); LOGGER.info("Container file: " + containerFile); } /* * Download and configure the container. */ final String installDir = StringUtils.substringBeforeLast(StringUtils.chomp(containerHome, "/"), "/"); if (StringUtils.contains(this.remoteLocation, "http")) { try { final URL remoteLocationUrl = new URL(this.remoteLocation + containerFile); final ZipURLInstaller installer = new ZipURLInstaller(remoteLocationUrl, installDir, installDir); installer.install(); } catch (final MalformedURLException e) { throw new DeployException("Failed to download container!", e); } /* * Rename the install directory to the container home directory so * it doesn't matter what the name is of the zip file and avoid case * sensitive issues on Linux. */ final String containerDir = StringUtils.stripEnd(containerFile, ".zip"); final File installedDir = new File(installDir + "/" + containerDir + "/"); final File destenationDir = new File(containerHome); if (LOGGER.isInfoEnabled()) { LOGGER.info("Renaming: " + installedDir.getPath()); LOGGER.info("To: " + destenationDir.getPath()); } final boolean renamed = installedDir.renameTo(destenationDir); if (!renamed) { final String msg = "Failed to rename container install directory to home directory name!"; if (LOGGER.isErrorEnabled()) { LOGGER.error(msg); } throw new ConfigurationException(msg); } } else { final Unzip unzipper = new Unzip(); unzipper.setSrc(new File(this.remoteLocation + containerFile)); unzipper.setDest(new File(containerHome)); unzipper.execute(); } /* * Setup the system properties. */ systemProperties.put("cargo.server.port", containerPort.toString()); }
From source file:org.apache.archiva.admin.model.beans.RemoteRepository.java
public RemoteRepository(Locale locale, String id, String name, String url, String layout, String userName, String password, int timeout) { super(locale, id, name, layout); this.url = StringUtils.stripEnd(url, "/"); this.userName = userName; this.password = password; this.timeout = timeout; }