List of usage examples for java.io RandomAccessFile write
public void write(byte b[], int off, int len) throws IOException
From source file:org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.java
static private void truncateBlock(File blockFile, File metaFile, long oldlen, long newlen) throws IOException { LOG.info("truncateBlock: blockFile=" + blockFile + ", metaFile=" + metaFile + ", oldlen=" + oldlen + ", newlen=" + newlen); if (newlen == oldlen) { return;//from w w w .j av a 2s .c om } if (newlen > oldlen) { throw new IOException( "Cannot truncate block to from oldlen (=" + oldlen + ") to newlen (=" + newlen + ")"); } DataChecksum dcs = BlockMetadataHeader.readHeader(metaFile).getChecksum(); int checksumsize = dcs.getChecksumSize(); int bpc = dcs.getBytesPerChecksum(); long n = (newlen - 1) / bpc + 1; long newmetalen = BlockMetadataHeader.getHeaderSize() + n * checksumsize; long lastchunkoffset = (n - 1) * bpc; int lastchunksize = (int) (newlen - lastchunkoffset); byte[] b = new byte[Math.max(lastchunksize, checksumsize)]; RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw"); try { //truncate blockFile blockRAF.setLength(newlen); //read last chunk blockRAF.seek(lastchunkoffset); blockRAF.readFully(b, 0, lastchunksize); } finally { blockRAF.close(); } //compute checksum dcs.update(b, 0, lastchunksize); dcs.writeValue(b, 0, false); //update metaFile RandomAccessFile metaRAF = new RandomAccessFile(metaFile, "rw"); try { metaRAF.setLength(newmetalen); metaRAF.seek(newmetalen - checksumsize); metaRAF.write(b, 0, checksumsize); } finally { metaRAF.close(); } }
From source file:org.sbs.util.download.MultiThreadDownload.java
/** * // ww w . j a v a 2 s . c o m * @param url * @param path * @param fileName * @return * @throws DownLoadException */ @SuppressWarnings("unchecked") public File downLoad(final URL url, String path, String fileName) throws DownLoadException { // ?? this.downLen = -1; this.contentLen = 0; this.date = new Date(); this.finished = false; try { URLConnection con = url.openConnection(); //? this.contentLen = con.getContentLength(); //?? if (StringUtils.isBlank(fileName)) { fileName = StringUtils.substringAfterLast(url.getPath(), "/"); } // File _path = new File(path); if (!_path.exists()) { _path.mkdirs(); } final File file = new File(path + File.separator + fileName); if (file.exists()) file.delete(); if (this.threadNum == 0 && this.blockSize > 0) { this.threadNum = (int) (contentLen / blockSize); if (this.threadNum == 0) { this.threadNum = 1; } } long subLen = contentLen / threadNum; List<Future<DownLoadBean>> result = Lists.newArrayList(); for (int i = 0; i < threadNum; i++) { final int pos = (int) (i * subLen); final int end = (int) ((i + 1) * subLen) - 1; final int current = pos; Future<DownLoadBean> f = (Future<DownLoadBean>) pool.submit(new Callable<DownLoadBean>() { int $pos = pos; int $end = end; int $current = current; @Override public DownLoadBean call() throws Exception { //buff BufferedInputStream bis = null; RandomAccessFile fos = null; byte[] buf = new byte[BUFFER_SIZE]; URLConnection con = null; try { con = url.openConnection(); con.setAllowUserInteraction(true); //???startPosendPos con.setRequestProperty("Range", "bytes=" + $pos + "-" + $end); fos = new RandomAccessFile(file, "rw"); //startPos fos.seek($pos); //????curPos???endPos //endPos? bis = new BufferedInputStream(con.getInputStream()); while ($current < $end) { int len = bis.read(buf, 0, BUFFER_SIZE); if (len == -1) { break; } fos.write(buf, 0, len); $current = $current + len; if ($current - 1 > $end) { throw new DownLoadException( "????"); } addLen(len); } bis.close(); fos.close(); } catch (IOException ex) { /* ????? StringBuffer sb = new StringBuffer(); sb.append($pos).append("\t").append($current).append("\t").append($end).append("\n"); FileUtils.write(new File(file.getAbsolutePath()+".pos"), sb, true); */ throw new RuntimeException(ex); } log.debug(this.hashCode() + ":??[" + $pos + "," + $end + "]"); return new DownLoadBean($pos, $end, $current); } }); result.add(f); } Long resultTotal = 0L; for (Future<DownLoadBean> f : result) { DownLoadBean dInfo = f.get(); resultTotal += dInfo.getCurrent() - dInfo.getPos(); } // ? if (contentLen > resultTotal + 1) { // ??? FileUtils.write(new File(down_error_log_file), url.toString() + "\n", true); throw new DownLoadException("?"); } else { finished = true; return file; } } catch (IOException | InterruptedException | ExecutionException e) { // try { FileUtils.write(new File(down_error_log_file), url.toString() + "\n", true); } catch (IOException e1) { e1.printStackTrace(); } e.printStackTrace(); } return null; }
From source file:org.commoncrawl.service.crawler.CrawlSegmentLog.java
public static int reconcileLogFile(FileSystem fs, Path logFilePath, int listId, int segmentId, CrawlSegmentFPMap segment, File consolidationFile) throws IOException { RandomAccessFile consolidationStream = null; int consolidationFileItemCount = 0; if (consolidationFile != null) { consolidationStream = new RandomAccessFile(consolidationFile, "rw"); consolidationFileItemCount = readerHeader(consolidationFile); consolidationStream.seek(consolidationStream.length()); }/* ww w . j av a 2 s . c o m*/ int processedItemCount = 0; FSDataInputStream hdfsInputStream = null; try { // get the file size on disk long fileSize = fs.getFileStatus(logFilePath).getLen(); // allocate an array that can hold up to the list size of items ... byte[] buffer = new byte[DEFAULT_LOGITEM_LIST_SIZE * LogItem.ItemSize_Bytes]; // calcuate item count int totalItemCount = (int) ((fileSize - getHeaderSize()) / LogItem.ItemSize_Bytes); // get a reader ... hdfsInputStream = fs.open(logFilePath); int headerItemCount = readHeader(hdfsInputStream); if (headerItemCount != totalItemCount) { LOG.warn("CrawlSegmentLog - header item count for log file:" + logFilePath.toString() + " is:" + headerItemCount + " file size indicates:" + totalItemCount); totalItemCount = headerItemCount; } int remainingItemCount = totalItemCount; LogItemBuffer itemList = new LogItemBuffer(listId, segmentId); while (remainingItemCount != 0) { int blockItemCount = Math.min(remainingItemCount, DEFAULT_LOGITEM_LIST_SIZE); // and read the data hdfsInputStream.read(buffer, 0, (int) blockItemCount * LogItem.ItemSize_Bytes); // and if consolidation stream is valid ... if (consolidationStream != null) { // add entries to that stream ... consolidationStream.write(buffer, 0, (int) blockItemCount * LogItem.ItemSize_Bytes); } // if not a dry run... if (segment != null) { // populate the item list itemList.loadFromStream(buffer, blockItemCount); // reconcile the list against the segment processedItemCount += reconcileItemList(itemList, segment); } // reduce item count remainingItemCount -= blockItemCount; } // finally if consolidation stream is valid ... if (consolidationStream != null) { // update the file's header .. writeHeader(consolidationFile, consolidationFileItemCount + totalItemCount); } } finally { if (consolidationStream != null) { consolidationStream.close(); } if (hdfsInputStream != null) { hdfsInputStream.close(); } } return processedItemCount; }
From source file:com.polyvi.xface.extension.advancedfiletransfer.XFileDownloader.java
@Override public void transfer(XCallbackContext callbackCtx) { initDownloadInfo();//from w w w . ja v a2 s .c o m if (mState == DOWNLOADING) { return; } mCallbackCtx = callbackCtx; if (null == mDownloadInfo) { onError(CONNECTION_ERR); } else { setState(DOWNLOADING); new Thread(new Runnable() { @Override public void run() { HttpURLConnection connection = null; RandomAccessFile randomAccessFile = null; InputStream is = null; int retry = RETRY; //TODO:????? do { int completeSize = mDownloadInfo.getCompleteSize(); try { URL url = new URL(mUrl); connection = (HttpURLConnection) url.openConnection(); connection.setConnectTimeout(TIME_OUT_MILLISECOND); connection.setRequestMethod("GET"); // ?Rangebytes x-; connection.setRequestProperty("Range", "bytes=" + completeSize + "-"); //cookie setCookieProperty(connection, mUrl); // ?.temp randomAccessFile = new RandomAccessFile(mLocalFilePath + TEMP_FILE_SUFFIX, "rwd"); randomAccessFile.seek(completeSize); // ??? is = connection.getInputStream(); byte[] buffer = new byte[mBufferSize]; int length = -1; while ((length = is.read(buffer)) != -1) { try { randomAccessFile.write(buffer, 0, length); } catch (Exception e) { retry = -1; break; } completeSize += length; onProgressUpdated(completeSize, 0); if (PAUSE == mState) { break; } } if (mDownloadInfo.isDownloadCompleted()) { // ??.temp renameFile(mLocalFilePath + TEMP_FILE_SUFFIX, mLocalFilePath); onSuccess(); break; } } catch (IOException e) { if (retry <= 0) { onError(CONNECTION_ERR); XLog.e(CLASS_NAME, e.getMessage()); } // ,?1? try { Thread.sleep(RETRY_INTERVAL); } catch (InterruptedException ex) { XLog.e(CLASS_NAME, "sleep be interrupted", ex); } } finally { try { if (null != is) { is.close(); } if (null != randomAccessFile) { // new URL??randomAccessFilenull randomAccessFile.close(); } if (null != connection) { // new URL??connectionnull connection.disconnect(); } } catch (IOException e) { XLog.e(CLASS_NAME, e.getMessage()); } } } while ((DOWNLOADING == mState) && (0 < retry--)); } }).start(); } }
From source file:com.polyvi.xface.extension.advancedfiletransfer.FileDownloader.java
@Override public void transfer(CallbackContext callbackCtx) { initDownloadInfo();/*from w w w .jav a 2 s .com*/ if (mState == DOWNLOADING) { return; } mCallbackCtx = callbackCtx; if (null == mDownloadInfo) { onError(CONNECTION_ERR); } else { setState(DOWNLOADING); new Thread(new Runnable() { @Override public void run() { HttpURLConnection connection = null; RandomAccessFile randomAccessFile = null; InputStream is = null; int retry = RETRY; // TODO:????? do { int completeSize = mDownloadInfo.getCompleteSize(); try { URL url = new URL(mUrl); connection = (HttpURLConnection) url.openConnection(); connection.setConnectTimeout(TIME_OUT_MILLISECOND); connection.setRequestMethod("GET"); // ?Rangebytes x-; connection.setRequestProperty("Range", "bytes=" + completeSize + "-"); // cookie setCookieProperty(connection, mUrl); // ?.temp randomAccessFile = new RandomAccessFile(mLocalFilePath + TEMP_FILE_SUFFIX, "rwd"); randomAccessFile.seek(completeSize); // ??? is = connection.getInputStream(); byte[] buffer = new byte[mBufferSize]; int length = -1; while ((length = is.read(buffer)) != -1) { try { randomAccessFile.write(buffer, 0, length); } catch (Exception e) { retry = -1; break; } completeSize += length; onProgressUpdated(completeSize, mDownloadInfo.getTotalSize()); mDownloadInfo.setCompleteSize(completeSize); if (PAUSE == mState) { break; } } if (mDownloadInfo.isDownloadCompleted()) { // ??.temp renameFile(mLocalFilePath + TEMP_FILE_SUFFIX, mLocalFilePath); onSuccess(); break; } } catch (FileNotFoundException e) { onError(FILE_NOT_FOUND_ERR); XLog.e(CLASS_NAME, e.getMessage()); } catch (IOException e) { if (retry <= 0) { onError(CONNECTION_ERR); XLog.e(CLASS_NAME, e.getMessage()); } // ,?1? try { Thread.sleep(RETRY_INTERVAL); } catch (InterruptedException ex) { XLog.e(CLASS_NAME, "sleep be interrupted", ex); } } finally { try { if (null != is) { is.close(); } if (null != randomAccessFile) { // new URL??randomAccessFilenull randomAccessFile.close(); } if (null != connection) { // new URL??connectionnull connection.disconnect(); } } catch (IOException e) { XLog.e(CLASS_NAME, e.getMessage()); } } } while ((DOWNLOADING == mState) && (0 < retry--)); } }).start(); } }
From source file:com.hly.component.download.DownloadTransaction.java
public void run() { // TODO ?Daemon InputStream is = null;/*from www. j a v a2s.com*/ HttpURLConnection conn = null; RandomAccessFile randomFile = null; File tmpFile = null; try { // ?uri tmpFile = new File(mTempLocalUri); File parentFile = tmpFile.getParentFile(); if (!parentFile.exists()) { parentFile.mkdirs(); } if (!tmpFile.exists()) { tmpFile.createNewFile(); } randomFile = new RandomAccessFile(mTempLocalUri, "rw"); long fileLength = randomFile.length(); completeSize = fileLength; if (isCancel) { return; } String connUrl = mUri; // ?uri???? // getRedirectUrl(connUrl); if (!T.ckIsEmpty(mRedirectUri)) { connUrl = mRedirectUri; } conn = getHttpConnetion(connUrl); conn.setRequestProperty("range", "bytes=" + fileLength + "-"); conn.connect(); int contentLength = conn.getContentLength(); totalSize = completeSize + contentLength; if (contentLength == -1 || contentLength > 0) { // randomFile.seek(fileLength); byte[] buffer = new byte[8192]; is = conn.getInputStream(); int length = -1; while ((length = is.read(buffer)) != -1) { if (isCancel) { return; } randomFile.write(buffer, 0, length); completeSize += length; notifyProgress(length); } } mTransactionState.setState(TransactionState.SUCCESS); } catch (Throwable t) { Log.w(TAG, Log.getStackTraceString(t)); } finally { isRunning = false; isCancel = false; try { if (randomFile != null) { randomFile.close(); } } catch (IOException e) { e.printStackTrace(); } if (is != null) { try { is.close(); } catch (IOException e) { e.printStackTrace(); } } if (conn != null) { conn.disconnect(); } if (mTransactionState.getState() != TransactionState.SUCCESS) { mTransactionState.setState(TransactionState.FAILED); Log.e(TAG, "Delivery failed."); } else { if (tmpFile == null) { mTransactionState.setState(TransactionState.FAILED); } else { File localFile = new File(this.mLocalUri); boolean flag = tmpFile.renameTo(localFile); if (flag) { Log.d(TAG, "rename pic succ" + this.mLocalUri); } else { mTransactionState.setState(TransactionState.FAILED); Log.d(TAG, "rename pic failed" + this.mLocalUri); } } } notifyObservers(); } }
From source file:FileBaseDataMap.java
/** * put Method.<br>//from www. j ava2 s.c om * * @param key * @param value * @param hashCode This is a key value hash code */ public void put(String key, String value, int hashCode) { try { File file = dataFileList[hashCode % numberOfDataFiles]; StringBuffer buf = new StringBuffer(this.fillCharacter(key, keyDataLength)); buf.append(this.fillCharacter(value, oneDataLength)); CacheContainer accessor = (CacheContainer) innerCache.get(file.getAbsolutePath()); RandomAccessFile raf = null; BufferedWriter wr = null; if (accessor == null || accessor.isClosed == true) { raf = new RandomAccessFile(file, "rwd"); wr = new BufferedWriter(new FileWriter(file, true)); accessor = new CacheContainer(); accessor.raf = raf; accessor.wr = wr; accessor.file = file; innerCache.put(file.getAbsolutePath(), accessor); } else { raf = accessor.raf; wr = accessor.wr; } // KeyData Write File for (int tryIdx = 0; tryIdx < 2; tryIdx++) { try { // Key?? long dataLineNo = this.getLinePoint(key, raf); if (dataLineNo == -1) { wr.write(buf.toString()); wr.flush(); // The size of an increment this.totalSize.getAndIncrement(); } else { // ?????1 boolean increMentFlg = false; if (this.get(key, hashCode) == null) increMentFlg = true; raf.seek(dataLineNo * (lineDataSize)); raf.write(buf.toString().getBytes(), 0, lineDataSize); if (increMentFlg) this.totalSize.getAndIncrement(); } break; } catch (IOException ie) { // IOException???1???? if (tryIdx == 1) throw ie; try { if (raf != null) raf.close(); if (wr != null) wr.close(); raf = new RandomAccessFile(file, "rwd"); wr = new BufferedWriter(new FileWriter(file, true)); accessor = new CacheContainer(); accessor.raf = raf; accessor.wr = wr; accessor.file = file; innerCache.put(file.getAbsolutePath(), accessor); } catch (Exception e) { throw e; } } } } catch (Exception e2) { e2.printStackTrace(); } }
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
/** * serialize metadata to disk /*ww w. ja va 2 s. com*/ * @throws IOException */ void writeSubDomainMetadataToDisk(CrawlListMetadata subDomainData) throws IOException { DataOutputBuffer outputBuffer = new DataOutputBuffer(CrawlListMetadata.Constants.FixedDataSize); subDomainData.serialize(outputBuffer, new BinaryProtocol()); if (outputBuffer.getLength() > CrawlListMetadata.Constants.FixedDataSize) { LOG.error("ListMetadata Serialize for List:" + subDomainData.getDomainName() + " > FixedDataSize!!!"); outputBuffer.reset(); subDomainData.setDomainName("<<CORRUPT>>"); subDomainData.serialize(outputBuffer, new BinaryProtocol()); } synchronized (_subDomainMetadataFile) { RandomAccessFile file = new RandomAccessFile(_subDomainMetadataFile, "rw"); try { if (subDomainData.getSubDomainDataOffset() == 0) { throw new IOException("Data Offset Zero during write!"); } file.seek(subDomainData.getSubDomainDataOffset()); file.write(outputBuffer.getData(), 0, outputBuffer.getLength()); } finally { file.close(); } } }
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
private void writeInitialOnDiskItem(URLFP fp, ProxyCrawlHistoryItem historyItem, DataOutputStream valueStreamOut, RandomAccessFile stringStream) throws IOException { OnDiskCrawlHistoryItem itemOut = onDiskItemFromHistoryItem(fp, historyItem); // update string offset ... itemOut._stringsOffset = stringStream.length(); // write out string data length WritableUtils.writeVInt(stringStream, _stringBuffer1.getLength()); // write strings to log file stringStream.write(_stringBuffer1.getData(), 0, _stringBuffer1.getLength()); // update timestamp ... itemOut._updateTimestamp = -1;/*from w ww.ja v a 2s. c om*/ // and write to disk itemOut.serialize(valueStreamOut); }
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
/** * update list state of a recently crawled item * /*from w w w. j av a 2 s .co m*/ * @param fingerprint - the fingerprint of the updated item * @param newData - the updated crawl history data for the given item * @throws IOException */ @Override public void updateItemState(URLFP fingerprint, ProxyCrawlHistoryItem newData) throws IOException { if (_listState == LoadState.LOADED) { // check for membership ... if (_bloomFilter.isPresent(fingerprint)) { //LOG.info("UpdateItemState Called for URL:" + newData.getOriginalURL() + " List:" + getListId()); //LOG.info("UpdateItemState Loading OnDisk Item for URL:" + newData.getOriginalURL() + " List:" + getListId()); // extract existing item from disk OnDiskCrawlHistoryItem originalItem = loadOnDiskItemForURLFP(fingerprint); //if present (null if false cache hit) if (originalItem != null) { // build an on disk item data structure for any potential changes ... OnDiskCrawlHistoryItem newItem = onDiskItemFromHistoryItem(fingerprint, newData); // set inital offset information newItem._fileOffset = originalItem._fileOffset; newItem._stringsOffset = originalItem._stringsOffset; // LOG.info("UpdateItemState Comparing OnDisk Item to New Item for URL:" + newData.getOriginalURL() + " List:" + getListId()); // compare the two items ... if (!newItem.equals(originalItem)) { //LOG.info("UpdateItemState Items Don't Match for URL:" + newData.getOriginalURL() + " List:" + getListId()); // ok items do not match ... figure out if strings are different ... if (newItem._stringsCRC != originalItem._stringsCRC) { RandomAccessFile stringsFile = new RandomAccessFile(_variableDataFile, "rw"); try { // seek to end stringsFile.seek(stringsFile.length()); // update offset info newItem._stringsOffset = stringsFile.length(); // write out string data length WritableUtils.writeVInt(stringsFile, _stringBuffer1.getLength()); // write strings to log file stringsFile.write(_stringBuffer1.getData(), 0, _stringBuffer1.getLength()); } finally { stringsFile.close(); } } // otherwise take the offset from old item else { newItem._stringsOffset = originalItem._stringsOffset; } //LOG.info("Opening Data File for OnDiskItem load for Fingerprint:" + newItem._urlFingerprint); // ok, different paths depending on wether this is an in memory update or not ... boolean wroteToMemory = false; synchronized (this) { if (_tempFixedDataBuffer != null) { wroteToMemory = true; // reset output buffer _tempOutputBuffer.reset(); // serizlie to output buffer newItem.serialize(_tempOutputBuffer); // copy to appropriate location System.arraycopy(_tempOutputBuffer.getData(), 0, _tempFixedDataBuffer, (int) originalItem._fileOffset, OnDiskCrawlHistoryItem.ON_DISK_SIZE); } } if (!wroteToMemory) { // write to disk RandomAccessFile file = new RandomAccessFile(_fixedDataFile, "rw"); try { while (true) { try { //LOG.info("*** TRYING UPDATE LOCK FOR OFFSET:" + originalItem._fileOffset); FileLock lock = file.getChannel().tryLock(originalItem._fileOffset, OnDiskCrawlHistoryItem.ON_DISK_SIZE, false); try { //LOG.info("*** GOT UPDATE LOCK FOR OFFSET:" + originalItem._fileOffset); file.seek(originalItem._fileOffset); newItem.serialize(file); //LOG.info("Updated Data File for OnDiskItem for Fingerprint:" + originalItem._urlFingerprint); break; } finally { //LOG.info("*** RELEASED UPDATE LOCK FOR OFFSET:" + originalItem._fileOffset); lock.release(); } } catch (OverlappingFileLockException e) { LOG.error("###LockConflict(RETRY):" + CCStringUtils.stringifyException(e)); } } } finally { file.close(); } } // ok now update metadata ... synchronized (_metadata) { int updateFlags = calculateUpdateFlags(originalItem, newItem); if (updateFlags != 0) { int metadataDirtyFlags = updateMetadata(newItem, _metadata, 0); // only write metadata to disk if temp data buffer is null if (metadataDirtyFlags != 0 && !wroteToMemory) { if ((metadataDirtyFlags & MetadataUpdateFlag_ModifiedCrawlStatus) != 0) { _metadata.setQueuedItemCount(_metadata.getQueuedItemCount() - 1); } writeMetadataToDisk(); } // if not writing to memory then update subdomain metadata if (!wroteToMemory) { synchronized (_subDomainMetadataFile) { CrawlListMetadata subDomainMetadata = getSubDomainMetadataByURL( newData.getOriginalURL()); int subDomainMetadataDirtyFlags = updateMetadata(newItem, subDomainMetadata, processFileOffsets); if (subDomainMetadataDirtyFlags != 0 && !wroteToMemory) { if ((subDomainMetadataDirtyFlags & MetadataUpdateFlag_ModifiedCrawlStatus) != 0) { subDomainMetadata.setQueuedItemCount( subDomainMetadata.getQueuedItemCount() - 1); } writeSubDomainMetadataToDisk(subDomainMetadata); } } } } } synchronized (this) { if (_eventListener != null) { _eventListener.itemUpdated(fingerprint); } } } } } } }