List of usage examples for java.nio ByteBuffer clear
public final Buffer clear()
From source file:org.lnicholls.galleon.togo.ToGo.java
public boolean Download(Video video, CancelDownload cancelDownload) { ServerConfiguration serverConfiguration = Server.getServer().getServerConfiguration(); GoBackConfiguration goBackConfiguration = Server.getServer().getGoBackConfiguration(); ArrayList videos = new ArrayList(); GetMethod get = null;//ww w . j av a2 s .c o m try { URL url = new URL(video.getUrl()); Protocol protocol = new Protocol("https", new TiVoSSLProtocolSocketFactory(), 443); HttpClient client = new HttpClient(); // TODO How to get TiVo address?? client.getHostConfiguration().setHost(url.getHost(), 443, protocol); String password = Tools.decrypt(serverConfiguration.getMediaAccessKey()); if (video.getParentalControls() != null && video.getParentalControls().booleanValue()) { if (serverConfiguration.getPassword() == null) throw new NullPointerException("Parental Controls Password is null"); password = password + Tools.decrypt(serverConfiguration.getPassword()); } Credentials credentials = new UsernamePasswordCredentials("tivo", password); //client.getState().setCredentials("TiVo DVR", url.getHost(), credentials); client.getState().setCredentials(null, url.getHost(), credentials); get = new GetMethod(video.getUrl()); client.executeMethod(get); if (get.getStatusCode() != 200) { log.debug("Status code: " + get.getStatusCode()); return false; } InputStream input = get.getResponseBodyAsStream(); String path = serverConfiguration.getRecordingsPath(); File dir = new File(path); if (!dir.exists()) { dir.mkdirs(); } String name = getFilename(video); File file = null; if (goBackConfiguration.isGroupByShow()) { if (video.getSeriesTitle() != null && video.getSeriesTitle().trim().length() > 0) { path = path + File.separator + clean(video.getSeriesTitle()); File filePath = new File(path); if (!filePath.exists()) filePath.mkdirs(); file = new File(path + File.separator + name); } else file = new File(path + File.separator + name); } else { file = new File(path + File.separator + name); } // TODO Handle retransfers /* if (file.exists() && video.getStatus()!=Video.STATUS_DOWNLOADING) { log.debug("duplicate file: "+file); try { List list = VideoManager.findByPath(file.getCanonicalPath()); if (list!=null && list.size()>0) { video.setDownloadSize(file.length()); video.setDownloadTime(0); video.setPath(file.getCanonicalPath()); video.setStatus(Video.STATUS_DELETED); VideoManager.updateVideo(video); return true; } } catch (HibernateException ex) { log.error("Video update failed", ex); } } */ log.info("Downloading: " + name); WritableByteChannel channel = new FileOutputStream(file, false).getChannel(); long total = 0; double diff = 0.0; ByteBuffer buf = ByteBuffer.allocateDirect(1024 * 4); byte[] bytes = new byte[1024 * 4]; int amount = 0; int index = 0; long target = video.getSize(); long start = System.currentTimeMillis(); long last = start; while (amount == 0 && total < target) { while (amount >= 0 && !cancelDownload.cancel()) { if (index == amount) { amount = input.read(bytes); index = 0; total = total + amount; } while (index < amount && buf.hasRemaining()) { buf.put(bytes[index++]); } buf.flip(); int numWritten = channel.write(buf); if (buf.hasRemaining()) { buf.compact(); } else { buf.clear(); } if ((System.currentTimeMillis() - last > 10000) && (total > 0)) { try { video = VideoManager.retrieveVideo(video.getId()); if (video.getStatus() == Video.STATUS_DOWNLOADING) { diff = (System.currentTimeMillis() - start) / 1000.0; if (diff > 0) { video.setDownloadSize(total); video.setDownloadTime((int) diff); VideoManager.updateVideo(video); } } } catch (HibernateException ex) { log.error("Video update failed", ex); } last = System.currentTimeMillis(); } } if (cancelDownload.cancel()) { channel.close(); return false; } } diff = (System.currentTimeMillis() - start) / 1000.0; channel.close(); if (diff != 0) log.info("Download rate=" + (total / 1024) / diff + " KBps"); try { video.setPath(file.getCanonicalPath()); VideoManager.updateVideo(video); } catch (HibernateException ex) { log.error("Video update failed", ex); } } catch (MalformedURLException ex) { Tools.logException(ToGo.class, ex, video.getUrl()); return false; } catch (Exception ex) { Tools.logException(ToGo.class, ex, video.getUrl()); return false; } finally { if (get != null) get.releaseConnection(); } return true; }
From source file:org.wso2.andes.tools.messagestore.commands.Dump.java
protected List<List> createMessageData(java.util.List<Long> msgids, List<QueueEntry> messages, boolean showHeaders, boolean showRouting, boolean showMessageHeaders) { List<List> display = new LinkedList<List>(); List<String> hex = new LinkedList<String>(); List<String> ascii = new LinkedList<String>(); display.add(hex);/*from w ww . j a v a 2s .co m*/ display.add(ascii); for (QueueEntry entry : messages) { ServerMessage msg = entry.getMessage(); if (!includeMsg(msg, msgids)) { continue; } //Add divider between messages hex.add(Console.ROW_DIVIDER); ascii.add(Console.ROW_DIVIDER); // Show general message information hex.add(Show.Columns.ID.name()); ascii.add(msg.getMessageNumber().toString()); hex.add(Console.ROW_DIVIDER); ascii.add(Console.ROW_DIVIDER); if (showRouting) { addShowInformation(hex, ascii, msg, "Routing Details", true, false, false); } if (showHeaders) { addShowInformation(hex, ascii, msg, "Headers", false, true, false); } if (showMessageHeaders) { addShowInformation(hex, ascii, msg, null, false, false, true); } // Add Content Body section hex.add("Content Body"); ascii.add(""); hex.add(Console.ROW_DIVIDER); ascii.add(Console.ROW_DIVIDER); final int messageSize = (int) msg.getSize(); if (messageSize != 0) { hex.add("Hex"); hex.add(Console.ROW_DIVIDER); ascii.add("ASCII"); ascii.add(Console.ROW_DIVIDER); java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocate(64 * 1024); int position = 0; while (position < messageSize) { position += msg.getContent(buf, position); buf.flip(); //Duplicate so we don't destroy original data :) java.nio.ByteBuffer hexBuffer = buf; java.nio.ByteBuffer charBuffer = hexBuffer.duplicate(); Hex hexencoder = new Hex(); while (hexBuffer.hasRemaining()) { byte[] line = new byte[LINE_SIZE]; int bufsize = hexBuffer.remaining(); if (bufsize < LINE_SIZE) { hexBuffer.get(line, 0, bufsize); } else { bufsize = line.length; hexBuffer.get(line); } byte[] encoded = hexencoder.encode(line); try { String encStr = new String(encoded, 0, bufsize * 2, DEFAULT_ENCODING); String hexLine = ""; int strLength = encStr.length(); for (int c = 0; c < strLength; c++) { hexLine += encStr.charAt(c); if ((c & 1) == 1 && SPACE_BYTES) { hexLine += BYTE_SPACER; } } hex.add(hexLine); } catch (UnsupportedEncodingException e) { _console.println(e.getMessage()); return null; } } while (charBuffer.hasRemaining()) { String asciiLine = ""; for (int pos = 0; pos < LINE_SIZE; pos++) { if (charBuffer.hasRemaining()) { byte ch = charBuffer.get(); if (isPrintable(ch)) { asciiLine += (char) ch; } else { asciiLine += NON_PRINTING_ASCII_CHAR; } if (SPACE_BYTES) { asciiLine += BYTE_SPACER; } } else { break; } } ascii.add(asciiLine); } buf.clear(); } } else { List<String> result = new LinkedList<String>(); display.add(result); result.add("No ContentBodies"); } } // if hex is empty then we have no data to display if (hex.size() == 0) { return null; } return display; }
From source file:jp.queuelinker.system.net.SelectorThread.java
@Override public void run() { ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); // I believe the inner try-catches does not cause overhead. // http://stackoverflow.com/questions/141560/ long sendCount = 0; SocketChannel currentChannel; SelectionKey key = null;//from w w w. java 2 s .c om while (true) { try { selector.select(); // selector.selectNow(); } catch (ClosedSelectorException e) { logger.fatal("BUG: The selector is closed."); return; } catch (IOException e) { logger.fatal("An IOException occured while calling select()."); // Fatal Error. Notify the error to the users and leave the matter to them. for (ChannelState state : channels) { state.callBack.fatalError(); } return; } if (!requests.isEmpty()) { handleRequest(); continue; } if (stopRequested) { return; } Set<SelectionKey> keys = selector.selectedKeys(); Iterator<SelectionKey> iter = keys.iterator(); iter_loop: while (iter.hasNext()) { key = iter.next(); iter.remove(); // Required. Don't remove. if (key.isReadable()) { currentChannel = (SocketChannel) key.channel(); final ChannelState state = (ChannelState) key.attachment(); int valid; try { valid = currentChannel.read(buffer); } catch (IOException e) { logger.warn("An IOException happened while reading from a channel."); state.callBack.exceptionOccured(state.channelId, state.attachment); key.cancel(); continue; } if (valid == -1) { // Normal socket close? state.callBack.exceptionOccured(state.channelId, state.attachment); // cleanUpChannel(state.channelId); key.cancel(); continue; } buffer.rewind(); if (state.callBack.receive(state.channelId, buffer, valid, state.attachment) == false) { state.key.interestOps(state.key.interestOps() & ~SelectionKey.OP_READ); } buffer.clear(); } else if (key.isWritable()) { currentChannel = (SocketChannel) key.channel(); final ChannelState state = (ChannelState) key.attachment(); while (state.sendBuffer.readableSize() < WRITE_SIZE) { ByteBuffer newBuffer = state.callBack.send(state.channelId, state.attachment); if (newBuffer != null) { state.sendBuffer.write(newBuffer); if (++sendCount % 50000 == 0) { logger.info("Send Count: " + sendCount); } } else if (state.sendBuffer.readableSize() == 0) { key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE); continue iter_loop; } else { break; } } final int available = state.sendBuffer.readableSize(); if (available >= WRITE_SIZE || ++state.noopCount >= HEURISTIC_WAIT) { int done; try { done = currentChannel.write(state.sendBuffer.getByteBuffer()); } catch (IOException e) { logger.warn("An IOException occured while writing to a channel."); state.callBack.exceptionOccured(state.channelId, state.attachment); key.cancel(); continue; } if (done < available) { state.sendBuffer.rollback(available - done); } state.sendBuffer.compact(); state.noopCount = 0; } } else if (key.isAcceptable()) { ServerSocketChannel channel = (ServerSocketChannel) key.channel(); ChannelState state = (ChannelState) key.attachment(); SocketChannel socketChannel; try { socketChannel = channel.accept(); socketChannel.configureBlocking(false); } catch (IOException e) { continue; // Do nothing. } state.callBack.newConnection(state.channelId, socketChannel, state.attachment); } } } }
From source file:com.healthmarketscience.jackcess.Column.java
/** * @param lvalDefinition Column value that points to an LVAL record * @return The LVAL data//from w w w. jav a2 s . c o m */ private byte[] readLongValue(byte[] lvalDefinition) throws IOException { ByteBuffer def = ByteBuffer.wrap(lvalDefinition).order(PageChannel.DEFAULT_BYTE_ORDER); int lengthWithFlags = def.getInt(); int length = lengthWithFlags & (~LONG_VALUE_TYPE_MASK); byte[] rtn = new byte[length]; byte type = (byte) ((lengthWithFlags & LONG_VALUE_TYPE_MASK) >>> 24); if (type == LONG_VALUE_TYPE_THIS_PAGE) { // inline long value def.getInt(); //Skip over lval_dp def.getInt(); //Skip over unknown def.get(rtn); } else { // long value on other page(s) if (lvalDefinition.length != getFormat().SIZE_LONG_VALUE_DEF) { throw new IOException("Expected " + getFormat().SIZE_LONG_VALUE_DEF + " bytes in long value definition, but found " + lvalDefinition.length); } int rowNum = ByteUtil.getUnsignedByte(def); int pageNum = ByteUtil.get3ByteInt(def, def.position()); ByteBuffer lvalPage = getPageChannel().createPageBuffer(); switch (type) { case LONG_VALUE_TYPE_OTHER_PAGE: { getPageChannel().readPage(lvalPage, pageNum); short rowStart = Table.findRowStart(lvalPage, rowNum, getFormat()); short rowEnd = Table.findRowEnd(lvalPage, rowNum, getFormat()); if ((rowEnd - rowStart) != length) { throw new IOException("Unexpected lval row length"); } lvalPage.position(rowStart); lvalPage.get(rtn); } break; case LONG_VALUE_TYPE_OTHER_PAGES: ByteBuffer rtnBuf = ByteBuffer.wrap(rtn); int remainingLen = length; while (remainingLen > 0) { lvalPage.clear(); getPageChannel().readPage(lvalPage, pageNum); short rowStart = Table.findRowStart(lvalPage, rowNum, getFormat()); short rowEnd = Table.findRowEnd(lvalPage, rowNum, getFormat()); // read next page information lvalPage.position(rowStart); rowNum = ByteUtil.getUnsignedByte(lvalPage); pageNum = ByteUtil.get3ByteInt(lvalPage); // update rowEnd and remainingLen based on chunkLength int chunkLength = (rowEnd - rowStart) - 4; if (chunkLength > remainingLen) { rowEnd = (short) (rowEnd - (chunkLength - remainingLen)); chunkLength = remainingLen; } remainingLen -= chunkLength; lvalPage.limit(rowEnd); rtnBuf.put(lvalPage); } break; default: throw new IOException("Unrecognized long value type: " + type); } } return rtn; }
From source file:edu.hawaii.soest.hioos.storx.StorXParser.java
/** * Parses the binary STOR-X file. The binary file format is a sequence of * 'frames' that all begin with 'SAT'. The parser creates a list with the * individual frames. Some frames are StorX frames (SATSTX), some are from * external sensors (ISUS: 'SATNLB', 'SATNDB'; SBE CTD: 'SATSBE') * * @param fileBuffer - the binary data file as a ByteBuffer *///w w w . j a v a 2 s . co m public void parse(ByteBuffer fileBuffer) throws Exception { logger.debug("StorXParser.parse() called."); this.fileBuffer = fileBuffer; //logger.debug(this.fileBuffer.toString()); try { // Create a buffer that will store a single frame of the file ByteBuffer frameBuffer = ByteBuffer.allocate(1024); // create four byte placeholders used to evaluate up to a four-byte // window. The FIFO layout looks like: // --------------------------- // in ---> | Four | Three | Two | One | ---> out // --------------------------- byte byteOne = 0x00, // set initial placeholder values byteTwo = 0x00, byteThree = 0x00, byteFour = 0x00; int frameByteCount = 0; // keep track of bytes per frame int frameCount = 0; // keep track of frames this.fileBuffer.position(0); this.fileBuffer.limit(this.fileBuffer.capacity()); while (this.fileBuffer.hasRemaining()) { // load the next byte into the FIFO window byteOne = fileBuffer.get(); // show the byte stream coming in //logger.debug("b1: " + new String(Hex.encodeHex(new byte[]{byteOne})) + "\t" + // "b2: " + new String(Hex.encodeHex(new byte[]{byteTwo})) + "\t" + // "b3: " + new String(Hex.encodeHex(new byte[]{byteThree})) + "\t" + // "b4: " + new String(Hex.encodeHex(new byte[]{byteFour})) + "\t" + // "st: " + Integer.toString(this.state) + "\t" + // "po: " + this.fileBuffer.position() + "\t" + // "cp: " + this.fileBuffer.capacity() // ); // evaluate the bytes, separate the file frame by frame (SAT ...) switch (this.state) { case 0: // find a frame beginning (SAT) 53 41 54 if (byteOne == 0x54 && byteTwo == 0x41 && byteThree == 0x53) { // found a line, add the beginning to the line buffer frameBuffer.put(byteThree); frameBuffer.put(byteTwo); frameBuffer.put(byteOne); frameByteCount = frameByteCount + 3; this.state = 1; break; } else { break; } case 1: // find the next frame beginning (SAT) 53 41 54 if ((byteOne == 0x54 && byteTwo == 0x41 && byteThree == 0x53) || fileBuffer.position() == fileBuffer.capacity()) { // we have a line ending. store the line in the arrayList frameBuffer.put(byteOne); frameByteCount++; frameBuffer.flip(); byte[] frameArray = frameBuffer.array(); ByteBuffer currentFrameBuffer; if (fileBuffer.position() == fileBuffer.capacity()) { // create a true copy of the byte array subset (no trailing 'SAT') byte[] frameCopy = new byte[frameByteCount]; System.arraycopy(frameArray, 0, frameCopy, 0, frameByteCount); currentFrameBuffer = ByteBuffer.wrap(frameCopy); } else { // create a true copy of the byte array subset (less the 'SAT') byte[] frameCopy = new byte[frameByteCount - 3]; System.arraycopy(frameArray, 0, frameCopy, 0, frameByteCount - 3); currentFrameBuffer = ByteBuffer.wrap(frameCopy); } // parse the current frame and add it to the frameMap frameCount++; // create a map to store frames as they are encountered BasicHierarchicalMap frameMap = new BasicHierarchicalMap(); // peek at the first six header bytes as a string byte[] sixBytes = new byte[6]; currentFrameBuffer.get(sixBytes); currentFrameBuffer.position(0); String frameHeader = new String(sixBytes, "US-ASCII"); // determine the frame type based on the header if (frameHeader.matches(this.STOR_X_HEADER_ID)) { frameMap.put("rawFrame", currentFrameBuffer); frameMap.put("id", frameHeader); frameMap.put("type", frameHeader.substring(3, 6)); frameMap.put("serialNumber", null); frameMap.put("date", null); String headerString = new String(currentFrameBuffer.array()); // trim trailing null characters and line endings int nullIndex = headerString.indexOf(0); headerString = headerString.substring(0, nullIndex).trim(); frameMap.put("parsedFrameObject", headerString); // Add the frame to the frames map this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone()); frameMap.removeAll("frame"); currentFrameBuffer.clear(); } else if (frameHeader.matches(this.STOR_X_FRAME_ID)) { // test if the frame is complete if (currentFrameBuffer.capacity() == this.STOR_X_FRAME_SIZE) { // convert the frame buffer to a StorXFrame StorXFrame storXFrame = new StorXFrame(currentFrameBuffer); frameMap.put("rawFrame", currentFrameBuffer); frameMap.put("id", frameHeader); frameMap.put("type", frameHeader.substring(3, 6)); frameMap.put("serialNumber", storXFrame.getSerialNumber()); frameMap.put("date", parseTimestamp(storXFrame.getTimestamp())); frameMap.put("parsedFrameObject", storXFrame); // Add the frame to the frames map this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone()); frameMap.removeAll("frame"); currentFrameBuffer.clear(); } else { logger.debug(frameHeader + " frame " + frameCount + " length is " + currentFrameBuffer.capacity() + " not " + this.STOR_X_FRAME_SIZE); } } else if (frameHeader.matches(this.SBE_CTD_FRAME_ID)) { // convert the frame buffer to a CTDFrame CTDFrame ctdFrame = new CTDFrame(currentFrameBuffer); // add in a sample if it matches a general data sample pattern if (ctdFrame.getSample().matches(" [0-9].*[0-9]\r\n")) { // extract the sample bytes from the frame frameMap.put("rawFrame", currentFrameBuffer); frameMap.put("id", frameHeader); frameMap.put("type", frameHeader.substring(3, 6)); frameMap.put("serialNumber", ctdFrame.getSerialNumber()); frameMap.put("date", parseTimestamp(ctdFrame.getTimestamp())); frameMap.put("parsedFrameObject", ctdFrame); // Add the frame to the frames map this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone()); } else { logger.debug("This CTD frame is not a data sample." + " Skipping it. The string is: " + ctdFrame.getSample()); } frameMap.removeAll("frame"); currentFrameBuffer.clear(); } else if (frameHeader.matches(this.ISUS_DARK_FRAME_ID)) { // test if the frame is complete if (currentFrameBuffer.capacity() == this.ISUS_FRAME_SIZE) { // convert the frame buffer to a ISUSFrame ISUSFrame isusFrame = new ISUSFrame(currentFrameBuffer); frameMap.put("rawFrame", currentFrameBuffer); frameMap.put("id", frameHeader); frameMap.put("type", frameHeader.substring(3, 6)); frameMap.put("serialNumber", isusFrame.getSerialNumber()); frameMap.put("date", parseTimestamp(isusFrame.getTimestamp())); frameMap.put("parsedFrameObject", isusFrame); // Add the frame to the frames map this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone()); frameMap.removeAll("frame"); currentFrameBuffer.clear(); } else { logger.debug(frameHeader + " frame " + frameCount + " length is " + currentFrameBuffer.capacity() + " not " + this.ISUS_FRAME_SIZE); } currentFrameBuffer.clear(); } else if (frameHeader.matches(this.ISUS_LIGHT_FRAME_ID)) { // test if the frame is complete if (currentFrameBuffer.capacity() == this.ISUS_FRAME_SIZE) { // convert the frame buffer to a ISUSFrame ISUSFrame isusFrame = new ISUSFrame(currentFrameBuffer); frameMap.put("rawFrame", currentFrameBuffer); frameMap.put("id", frameHeader); frameMap.put("type", frameHeader.substring(3, 6)); frameMap.put("serialNumber", isusFrame.getSerialNumber()); frameMap.put("date", parseTimestamp(isusFrame.getTimestamp())); frameMap.put("parsedFrameObject", isusFrame); // Add the frame to the frames map this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone()); frameMap.removeAll("frame"); currentFrameBuffer.clear(); } else { logger.debug(frameHeader + " frame " + frameCount + " length is " + currentFrameBuffer.capacity() + " not " + this.ISUS_FRAME_SIZE); } currentFrameBuffer.clear(); } else { logger.info("The current frame type is not recognized. " + "Discarding it. The header was: " + frameHeader); currentFrameBuffer.clear(); } // reset the frame buffer for the next frame, but add the 'SAT' // bytes already encountered frameBuffer.clear(); frameByteCount = 0; this.fileBuffer.position(this.fileBuffer.position() - 3); this.state = 0; break; } else { // no full line yet, keep adding bytes frameBuffer.put(byteOne); frameByteCount++; break; } } // end switch() // shift the bytes in the FIFO window byteFour = byteThree; byteThree = byteTwo; byteTwo = byteOne; } // end while() logger.debug(this.framesMap.toXMLString(1000)); } catch (Exception e) { logger.debug("Failed to parse the data file. The error message was:" + e.getMessage()); e.printStackTrace(); } }
From source file:net.librec.data.convertor.TextDataConvertor.java
/** * Read data from the data file. Note that we didn't take care of the * duplicated lines.//from www . j a va2 s .co m * * @param dataColumnFormat * the format of input data file * @param inputDataPath * the path of input data file * @param binThold * the threshold to binarize a rating. If a rating is greater * than the threshold, the value will be 1; otherwise 0. To * disable this appender, i.e., keep the original rating value, * set the threshold a negative value * @throws IOException * if the <code>inputDataPath</code> is not valid. */ private void readData(String dataColumnFormat, String inputDataPath, double binThold) throws IOException { LOG.info(String.format("Dataset: %s", StringUtil.last(inputDataPath, 38))); // Table {row-id, col-id, rate} Table<Integer, Integer, Double> dataTable = HashBasedTable.create(); // Table {row-id, col-id, timestamp} Table<Integer, Integer, Long> timeTable = null; // Map {col-id, multiple row-id}: used to fast build a rating matrix Multimap<Integer, Integer> colMap = HashMultimap.create(); // BiMap {raw id, inner id} userIds, itemIds if (this.userIds == null) { this.userIds = HashBiMap.create(); } if (this.itemIds == null) { this.itemIds = HashBiMap.create(); } final List<File> files = new ArrayList<File>(); final ArrayList<Long> fileSizeList = new ArrayList<Long>(); SimpleFileVisitor<Path> finder = new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { fileSizeList.add(file.toFile().length()); files.add(file.toFile()); return super.visitFile(file, attrs); } }; Files.walkFileTree(Paths.get(inputDataPath), finder); LOG.info("All dataset files " + files.toString()); long allFileSize = 0; for (Long everyFileSize : fileSizeList) { allFileSize = allFileSize + everyFileSize.longValue(); } LOG.info("All dataset files size " + Long.toString(allFileSize)); int readingFileCount = 0; long loadAllFileByte = 0; // loop every dataFile collecting from walkFileTree for (File dataFile : files) { LOG.info("Now loading dataset file " + dataFile.toString().substring( dataFile.toString().lastIndexOf(File.separator) + 1, dataFile.toString().lastIndexOf("."))); readingFileCount += 1; loadFilePathRate = readingFileCount / (float) files.size(); long readingOneFileByte = 0; FileInputStream fis = new FileInputStream(dataFile); FileChannel fileRead = fis.getChannel(); ByteBuffer buffer = ByteBuffer.allocate(BSIZE); int len; String bufferLine = new String(); byte[] bytes = new byte[BSIZE]; while ((len = fileRead.read(buffer)) != -1) { readingOneFileByte += len; loadDataFileRate = readingOneFileByte / (float) fileRead.size(); loadAllFileByte += len; loadAllFileRate = loadAllFileByte / (float) allFileSize; buffer.flip(); buffer.get(bytes, 0, len); bufferLine = bufferLine.concat(new String(bytes, 0, len)); bufferLine = bufferLine.replaceAll("\r", "\n"); String[] bufferData = bufferLine.split("(\n)+"); boolean isComplete = bufferLine.endsWith("\n"); int loopLength = isComplete ? bufferData.length : bufferData.length - 1; for (int i = 0; i < loopLength; i++) { String line = new String(bufferData[i]); String[] data = line.trim().split("[ \t,]+"); String user = data[0]; String item = data[1]; Double rate = ((dataColumnFormat.equals("UIR") || dataColumnFormat.equals("UIRT")) && data.length >= 3) ? Double.valueOf(data[2]) : 1.0; // binarize the rating for item recommendation task if (binThold >= 0) { rate = rate > binThold ? 1.0 : 0.0; } // inner id starting from 0 int row = userIds.containsKey(user) ? userIds.get(user) : userIds.size(); userIds.put(user, row); int col = itemIds.containsKey(item) ? itemIds.get(item) : itemIds.size(); itemIds.put(item, col); dataTable.put(row, col, rate); colMap.put(col, row); // record rating's issuing time if (StringUtils.equals(dataColumnFormat, "UIRT") && data.length >= 4) { if (timeTable == null) { timeTable = HashBasedTable.create(); } // convert to million-seconds long mms = 0L; try { mms = Long.parseLong(data[3]); // cannot format // 9.7323480e+008 } catch (NumberFormatException e) { mms = (long) Double.parseDouble(data[3]); } long timestamp = timeUnit.toMillis(mms); timeTable.put(row, col, timestamp); } } if (!isComplete) { bufferLine = bufferData[bufferData.length - 1]; } buffer.clear(); } fileRead.close(); fis.close(); } int numRows = numUsers(), numCols = numItems(); // build rating matrix preferenceMatrix = new SparseMatrix(numRows, numCols, dataTable, colMap); if (timeTable != null) datetimeMatrix = new SparseMatrix(numRows, numCols, timeTable, colMap); // release memory of data table dataTable = null; timeTable = null; }
From source file:hivemall.recommend.SlimUDTF.java
private void runIterativeTraining() throws HiveException { final ByteBuffer buf = this._inputBuf; final NioStatefulSegment dst = this._fileIO; assert (buf != null); assert (dst != null); final Reporter reporter = getReporter(); final Counters.Counter iterCounter = (reporter == null) ? null : reporter.getCounter("hivemall.recommend.slim$Counter", "iteration"); try {/*from w ww .jav a 2s .c om*/ if (dst.getPosition() == 0L) {// run iterations w/o temporary file if (buf.position() == 0) { return; // no training example } buf.flip(); for (int iter = 2; iter < numIterations; iter++) { _cvState.next(); reportProgress(reporter); setCounterValue(iterCounter, iter); while (buf.remaining() > 0) { int recordBytes = buf.getInt(); assert (recordBytes > 0) : recordBytes; replayTrain(buf); } buf.rewind(); if (_cvState.isConverged(_observedTrainingExamples)) { break; } } logger.info("Performed " + _cvState.getCurrentIteration() + " iterations of " + NumberUtils.formatNumber(_observedTrainingExamples) + " training examples on memory (thus " + NumberUtils.formatNumber(_observedTrainingExamples * _cvState.getCurrentIteration()) + " training updates in total) "); } else { // read training examples in the temporary file and invoke train for each example // write KNNi in buffer to a temporary file if (buf.remaining() > 0) { writeBuffer(buf, dst); } try { dst.flush(); } catch (IOException e) { throw new HiveException("Failed to flush a file: " + dst.getFile().getAbsolutePath(), e); } if (logger.isInfoEnabled()) { File tmpFile = dst.getFile(); logger.info("Wrote KNN entries of axis items to a temporary file for iterative training: " + tmpFile.getAbsolutePath() + " (" + FileUtils.prettyFileSize(tmpFile) + ")"); } // run iterations for (int iter = 2; iter < numIterations; iter++) { _cvState.next(); setCounterValue(iterCounter, iter); buf.clear(); dst.resetPosition(); while (true) { reportProgress(reporter); // load a KNNi to a buffer in the temporary file final int bytesRead; try { bytesRead = dst.read(buf); } catch (IOException e) { throw new HiveException("Failed to read a file: " + dst.getFile().getAbsolutePath(), e); } if (bytesRead == 0) { // reached file EOF break; } assert (bytesRead > 0) : bytesRead; // reads training examples from a buffer buf.flip(); int remain = buf.remaining(); if (remain < SizeOf.INT) { throw new HiveException("Illegal file format was detected"); } while (remain >= SizeOf.INT) { int pos = buf.position(); int recordBytes = buf.getInt(); remain -= SizeOf.INT; if (remain < recordBytes) { buf.position(pos); break; } replayTrain(buf); remain -= recordBytes; } buf.compact(); } if (_cvState.isConverged(_observedTrainingExamples)) { break; } } logger.info("Performed " + _cvState.getCurrentIteration() + " iterations of " + NumberUtils.formatNumber(_observedTrainingExamples) + " training examples on memory and KNNi data on secondary storage (thus " + NumberUtils.formatNumber(_observedTrainingExamples * _cvState.getCurrentIteration()) + " training updates in total) "); } } catch (Throwable e) { throw new HiveException("Exception caused in the iterative training", e); } finally { // delete the temporary file and release resources try { dst.close(true); } catch (IOException e) { throw new HiveException("Failed to close a file: " + dst.getFile().getAbsolutePath(), e); } this._inputBuf = null; this._fileIO = null; } }
From source file:com.healthmarketscience.jackcess.Column.java
/** * Write an LVAL column into a ByteBuffer inline if it fits, otherwise in * other data page(s).//from w w w . j av a 2 s . c om * @param value Value of the LVAL column * @return A buffer containing the LVAL definition and (possibly) the column * value (unless written to other pages) * @usage _advanced_method_ */ public ByteBuffer writeLongValue(byte[] value, int remainingRowLength) throws IOException { if (value.length > getType().getMaxSize()) { throw new IOException( "value too big for column, max " + getType().getMaxSize() + ", got " + value.length); } // determine which type to write byte type = 0; int lvalDefLen = getFormat().SIZE_LONG_VALUE_DEF; if (((getFormat().SIZE_LONG_VALUE_DEF + value.length) <= remainingRowLength) && (value.length <= getFormat().MAX_INLINE_LONG_VALUE_SIZE)) { type = LONG_VALUE_TYPE_THIS_PAGE; lvalDefLen += value.length; } else if (value.length <= getFormat().MAX_LONG_VALUE_ROW_SIZE) { type = LONG_VALUE_TYPE_OTHER_PAGE; } else { type = LONG_VALUE_TYPE_OTHER_PAGES; } ByteBuffer def = getPageChannel().createBuffer(lvalDefLen); // take length and apply type to first byte int lengthWithFlags = value.length | (type << 24); def.putInt(lengthWithFlags); if (type == LONG_VALUE_TYPE_THIS_PAGE) { // write long value inline def.putInt(0); def.putInt(0); //Unknown def.put(value); } else { TempPageHolder lvalBufferH = getTable().getLongValueBuffer(); ByteBuffer lvalPage = null; int firstLvalPageNum = PageChannel.INVALID_PAGE_NUMBER; byte firstLvalRow = 0; // write other page(s) switch (type) { case LONG_VALUE_TYPE_OTHER_PAGE: lvalPage = getLongValuePage(value.length, lvalBufferH); firstLvalPageNum = lvalBufferH.getPageNumber(); firstLvalRow = (byte) Table.addDataPageRow(lvalPage, value.length, getFormat(), 0); lvalPage.put(value); getPageChannel().writePage(lvalPage, firstLvalPageNum); break; case LONG_VALUE_TYPE_OTHER_PAGES: ByteBuffer buffer = ByteBuffer.wrap(value); int remainingLen = buffer.remaining(); buffer.limit(0); lvalPage = getLongValuePage(getFormat().MAX_LONG_VALUE_ROW_SIZE, lvalBufferH); firstLvalPageNum = lvalBufferH.getPageNumber(); int lvalPageNum = firstLvalPageNum; ByteBuffer nextLvalPage = null; int nextLvalPageNum = 0; while (remainingLen > 0) { lvalPage.clear(); // figure out how much we will put in this page (we need 4 bytes for // the next page pointer) int chunkLength = Math.min(getFormat().MAX_LONG_VALUE_ROW_SIZE - 4, remainingLen); // figure out if we will need another page, and if so, allocate it if (chunkLength < remainingLen) { // force a new page to be allocated lvalBufferH.clear(); nextLvalPage = getLongValuePage(getFormat().MAX_LONG_VALUE_ROW_SIZE, lvalBufferH); nextLvalPageNum = lvalBufferH.getPageNumber(); } else { nextLvalPage = null; nextLvalPageNum = 0; } // add row to this page byte lvalRow = (byte) Table.addDataPageRow(lvalPage, chunkLength + 4, getFormat(), 0); // write next page info (we'll always be writing into row 0 for // newly created pages) lvalPage.put((byte) 0); // row number ByteUtil.put3ByteInt(lvalPage, nextLvalPageNum); // page number // write this page's chunk of data buffer.limit(buffer.limit() + chunkLength); lvalPage.put(buffer); remainingLen -= chunkLength; // write new page to database getPageChannel().writePage(lvalPage, lvalPageNum); if (lvalPageNum == firstLvalPageNum) { // save initial row info firstLvalRow = lvalRow; } else { // check assertion that we wrote to row 0 for all subsequent pages if (lvalRow != (byte) 0) { throw new IllegalStateException("Expected row 0, but was " + lvalRow); } } // move to next page lvalPage = nextLvalPage; lvalPageNum = nextLvalPageNum; } break; default: throw new IOException("Unrecognized long value type: " + type); } // update def def.put(firstLvalRow); ByteUtil.put3ByteInt(def, firstLvalPageNum); def.putInt(0); //Unknown } def.flip(); return def; }
From source file:com.castis.sysComp.PoisConverterSysComp.java
public void parseRegionFile(File file) throws Exception { String line = ""; FileInputStream in = null;//from w ww . ja v a2s. c o m Reader isReader = null; LineNumberReader bufReader = null; FileOutputStream fos = null; String fileName = file.getName(); int index = fileName.indexOf("-"); if (index != -1) { fileName = fileName.substring(index + 1, fileName.length()); } String dir = filePolling.getValidFileDirectory(resultDir); String tempDir = dir + "/temp/"; File targetDirectory = new File(CiFileUtil.getReplaceFullPath(tempDir)); if (!targetDirectory.isDirectory()) { CiFileUtil.createDirectory(tempDir); } fos = new FileOutputStream(tempDir + fileName); int byteSize = 2048; ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteSize); GatheringByteChannel outByteCh = fos.getChannel(); try { in = new FileInputStream(file); isReader = new InputStreamReader(in, "UTF-16LE"); bufReader = new LineNumberReader(isReader); boolean first = true; while ((line = bufReader.readLine()) != null) { byte[] utf8 = line.getBytes("UTF-8"); String string = new String(utf8, "UTF-8"); String data[] = string.split("\t"); if (first == true) { first = false; if (data[0] == null || data[0].contains("region") == false) { throw new DataParsingException("data parsing error(not formatted)"); } continue; } if (data[0] == null || data[0].equals("")) { throw new DataParsingException("data parsing error(region id)"); } if (data[1] == null || data[1].equals("")) { throw new DataParsingException("data parsing error(region name)"); } if (data[2] == null || data[2].equals("")) { throw new DataParsingException("data parsing error(parent id)"); } StringBuffer strBuffer = new StringBuffer(); strBuffer.append(data[0]); strBuffer.append("\t"); strBuffer.append(data[1]); strBuffer.append("\t"); strBuffer.append(data[2]); strBuffer.append("\r\n"); byte[] outByte = null; try { outByte = strBuffer.toString().getBytes("UTF-8"); } catch (UnsupportedEncodingException e2) { e2.printStackTrace(); } byteBuffer.put(outByte); byteBuffer.flip(); try { outByteCh.write(byteBuffer); } catch (IOException e) { } byteBuffer.clear(); } fos.close(); index = fileName.indexOf("_"); String targetDir = resultDir; File sourceFile = new File(tempDir + fileName); if (index != -1) { String directory = fileName.substring(0, index); targetDir += "/" + directory; } try { File resultTargetDir = new File(CiFileUtil.getReplaceFullPath(targetDir)); if (!resultTargetDir.isDirectory()) { CiFileUtil.createDirectory(targetDir); } CiFileUtil.renameFile(sourceFile, targetDir, fileName); } catch (Exception e) { log.error(e.getMessage()); } } catch (Exception e) { String errorMsg = "Fail to parsing Line.[current line(" + bufReader.getLineNumber() + ") :" + line + "] : "; log.error(errorMsg, e); throw new DataParsingException(errorMsg, e); //throw(e); } finally { if (in != null) in.close(); if (isReader != null) isReader.close(); if (bufReader != null) bufReader.close(); } }