List of usage examples for java.io RandomAccessFile close
public void close() throws IOException
From source file:org.commoncrawl.service.crawler.CrawlList.java
public static void startDiskQueueingThread(final EventLoop serverEventLoop, final File baseStoragePath) { // figure out // and finally start the blocking writer thread ... _diskOperationThread = new Thread(new Runnable() { public void run() { for (;;) { try { DiskQueueEntry entry = _diskOperationQueue.take(); // if buffer item is null... this is considered an eof condition ... break out ... if (entry.getListItem() == null) { LOG.info(//from w ww . j av a2 s. c o m "### DiskThread:Received Null Item ... Shutting down CrawlDomain Disk Queue Thread"); // now matter what ... break out ... break; } // otherwise .. figure out what to do with the domain ... else { if (_diskOpThreadShuttingDown == false) { final CrawlList domain = entry.getListItem(); try { if (Environment.detailLogEnabled()) LOG.info("### DiskThread: Got List:" + domain.getListName()); // build a hierarchichal path for the given domain id ... File logFilePath = null; String listName = null; synchronized (domain) { logFilePath = FileUtils.buildHierarchicalPathForId(baseStoragePath, domain.getUniqueListId()); listName = domain.getListName(); } // get the immediate parent directory ... File parentDirectory = logFilePath.getParentFile(); // and recursively create the directory chain (if necessary). parentDirectory.mkdirs(); IntrusiveList<CrawlTarget> flushList = null; int desiredLoadAmount = 0; boolean truncateFile = false; synchronized (domain) { if (domain._offlineTargetCount == 0) { truncateFile = true; } } if (truncateFile && logFilePath.exists()) { if (Environment.detailLogEnabled()) LOG.info("### DiskThread: Truncating Existing Log File for List:" + listName); LogFileHeader header = new LogFileHeader(); RandomAccessFile file = new RandomAccessFile(logFilePath, "rw"); try { writeLogFileHeader(file, header); } finally { file.close(); } } // now lock access to the domain's pending queue synchronized (domain) { // if a disk request was pending ... if (domain._diskRequestPending) { // reset disk request pending flag here to prevent race condition ... domain._diskRequestPending = false; // figure out what action to take with respect to the domain ... // if list count exceeds flush threshold if (domain._pending.size() >= DISK_FLUSH_THRESHOLD || domain._queued.size() != 0) { if (domain._queued.size() == 0) { LinkedList<CrawlTarget> candidates = new LinkedList<CrawlTarget>(); for (CrawlTarget candidate : domain._pending) { if ((candidate.getFlags() & CrawlURL.Flags.IsHighPriorityURL) == 0) { // add candidates in proper order ... candidates.add(candidate); } } // if there are low priority candidates we can flush ... if (candidates.size() != 0) { // create a new flush list ... flushList = new IntrusiveList<CrawlTarget>(); // reverse candidate list and start removing items from pending for (CrawlTarget candidate : Lists.reverse(candidates)) { domain._pending.removeElement(candidate); flushList.addHead(candidate); // if we are back to ideal target count bail ... if (domain._pending.size() <= IDEAL_TARGET_COUNT) break; } if (Environment.detailLogEnabled()) LOG.info("### DiskThread: List:" + domain.getListName() + " Created FetchList FROM PENDING of Size:" + flushList.size()); // increment offline target count ... domain._offlineTargetCount += flushList.size(); } } else { flushList = domain._queued.detach(domain._queued.getHead()); if (Environment.detailLogEnabled()) LOG.info("### DiskThread: List:" + domain.getListName() + " Created FetchList FROM QUEUED of Size:" + flushList.size()); // increment offline target count ... domain._offlineTargetCount += flushList.size(); } /* // walk one past IDEAL target item count... int i=0; CrawlTarget target = domain._pending.getHead(); while (i<IDEAL_TARGET_COUNT) { target = target.getNext(); ++i; } // and extract a sub-list starting at the target ... flushList = domain._pending.detach(target); */ //and immediately update offline target count in domain ... //domain._offlineTargetCount += flushList.size(); } // otherwise ... else { // check queued size ... if (domain._queued.size() != 0) { // if pending size <= DISK_LOAD_THRESHOLD if (domain._offlineTargetCount == 0 && domain._pending.size() <= DISK_LOAD_THRESHOLD) { if (Environment.detailLogEnabled()) LOG.info( "### DiskThread: Moving Items from Queued List to Pending List for CrawlList:" + domain.getListName()); // move over items from queued to pending while (domain._queued.getHead() != null) { domain._pending.addTail(domain._queued.removeHead()); if (domain._pending .size() == (DISK_FLUSH_THRESHOLD - 1)) break; } } //now if domain queue exceeds flush threshold ... if (domain._queued.size() >= IDEAL_TARGET_COUNT) { if (Environment.detailLogEnabled()) LOG.info( "### DiskThread: Queued Size Exceed Flush Threshold. Flushing to Disk for CrawlList:" + domain.getListName()); // extract a sub-list starting at head of queued list flushList = domain._queued.detach(domain._queued.getHead()); //and immediately update offline target count in domain ... domain._offlineTargetCount += flushList.size(); } } // check to see if a load is desired ... if (domain._pending.size() <= DISK_LOAD_THRESHOLD) { // calculate load amount ... desiredLoadAmount = IDEAL_TARGET_COUNT - domain._pending.size(); } } } else { if (Environment.detailLogEnabled()) LOG.info("### DiskThread: Skipping List:" + domain.getListName()); } } // now figure out what to do ... if (flushList != null) { if (Environment.detailLogEnabled()) LOG.info("### DiskThread: Flushing" + flushList.size() + " Items To Disk for Domain:" + domain.getListName()); // flush crawl targets to disk ... appendTargetsToLogFile(logFilePath, flushList); // clear list ... flushList.removeAll(); } // ... if load is desired ... if (desiredLoadAmount != 0) { IntrusiveList<CrawlTarget> loadList = new IntrusiveList<CrawlTarget>(); int loadCount = readTargetsFromLogFile(domain, logFilePath, desiredLoadAmount, loadList); // if (Environment.detailLogEnabled()) LOG.info("### DiskThread:Disk Queue Loaded: " + loadCount + "Items To Disk for Domain:" + domain.getListName()); if (loadCount != 0) { // time to lock domain again ... synchronized (domain) { // and reduce offline count ... domain._offlineTargetCount -= loadList.size(); // load new items into domain's list ... domain._pending.attach(loadList); } } } } catch (IOException e) { LOG.error("### DiskThread:" + CCStringUtils.stringifyException(e)); } } } } catch (InterruptedException e) { } catch (Exception e) { LOG.fatal("### DiskThread: Encountered Unhandled Exception:" + CCStringUtils.stringifyException(e)); } } LOG.info("### DiskThread: Exiting CrawlDomain Disk Queue Thread"); } }); // launch the writer thread ... _diskOperationThread.start(); }
From source file:com.owncloud.android.lib.resources.files.ChunkedUploadRemoteFileOperation.java
@Override protected int uploadFile(OwnCloudClient client) throws HttpException, IOException { int status = -1; FileChannel channel = null;//from ww w . ja va 2 s . c om RandomAccessFile raf = null; try { File file = new File(mLocalPath); raf = new RandomAccessFile(file, "r"); channel = raf.getChannel(); mEntity = new ChunkFromFileChannelRequestEntity(channel, mMimeType, CHUNK_SIZE, file); //((ProgressiveDataTransferer)mEntity).addDatatransferProgressListeners(getDataTransferListeners()); synchronized (mDataTransferListeners) { ((ProgressiveDataTransferer) mEntity).addDatatransferProgressListeners(mDataTransferListeners); } long offset = 0; String uriPrefix = client.getWebdavUri() + WebdavUtils.encodePath(mRemotePath) + "-chunking-" + Math.abs((new Random()).nextInt(9000) + 1000) + "-"; long chunkCount = (long) Math.ceil((double) file.length() / CHUNK_SIZE); for (int chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++, offset += CHUNK_SIZE) { if (mPutMethod != null) { mPutMethod.releaseConnection(); // let the connection available for other methods } mPutMethod = new PutMethod(uriPrefix + chunkCount + "-" + chunkIndex); mPutMethod.addRequestHeader(OC_CHUNKED_HEADER, OC_CHUNKED_HEADER); ((ChunkFromFileChannelRequestEntity) mEntity).setOffset(offset); mPutMethod.setRequestEntity(mEntity); status = client.executeMethod(mPutMethod); client.exhaustResponse(mPutMethod.getResponseBodyAsStream()); Log_OC.d(TAG, "Upload of " + mLocalPath + " to " + mRemotePath + ", chunk index " + chunkIndex + ", count " + chunkCount + ", HTTP result status " + status); if (!isSuccess(status)) break; } } finally { if (channel != null) channel.close(); if (raf != null) raf.close(); if (mPutMethod != null) mPutMethod.releaseConnection(); // let the connection available for other methods } return status; }
From source file:org.apache.hadoop.hdfs.TestFileAppend4.java
/** * Corrupt all of the blocks in the blocksBeingWritten dir * for the specified datanode number. The corruption is * specifically the last checksum chunk of the file being * modified by writing random data into it. *//*w w w . j a va2 s.c o m*/ private void corruptDataNode(int dnNumber, CorruptionType type) throws Exception { // get the FS data of the specified datanode File data_dir = new File(System.getProperty("test.build.data"), "dfs/data/data" + Integer.toString(dnNumber * 2 + 1) + "/blocksBeingWritten"); int corrupted = 0; for (File block : data_dir.listFiles()) { // only touch the actual data, not the metadata (with CRC) if (block.getName().startsWith("blk_") && !block.getName().endsWith("meta")) { if (type == CorruptionType.CORRUPT_LAST_CHUNK) { RandomAccessFile file = new RandomAccessFile(block, "rw"); FileChannel channel = file.getChannel(); Random r = new Random(); long lastBlockSize = channel.size() % 512; long position = channel.size() - lastBlockSize; int length = r.nextInt((int) (channel.size() - position + 1)); byte[] buffer = new byte[length]; r.nextBytes(buffer); channel.write(ByteBuffer.wrap(buffer), position); System.out.println("Deliberately corrupting file " + block.getName() + " at offset " + position + " length " + length); file.close(); } else if (type == CorruptionType.TRUNCATE_BLOCK_TO_ZERO) { LOG.info("Truncating block file at " + block); RandomAccessFile blockFile = new RandomAccessFile(block, "rw"); blockFile.setLength(0); blockFile.close(); RandomAccessFile metaFile = new RandomAccessFile(FSDataset.findMetaFile(block), "rw"); metaFile.setLength(0); metaFile.close(); } else if (type == CorruptionType.TRUNCATE_BLOCK_HALF) { FSDatasetTestUtil.truncateBlockFile(block, block.length() / 2); } else { assert false; } ++corrupted; } } assertTrue("Should have some data in bbw to corrupt", corrupted > 0); }
From source file:com.vincestyling.netroid.request.FileDownloadRequest.java
/** * In this method, we got the Content-Length, with the TemporaryFile length, * we can calculate the actually size of the whole file, if TemporaryFile not exists, * we'll take the store file length then compare to actually size, and if equals, * we consider this download was already done. * We used {@link RandomAccessFile} to continue download, when download success, * the TemporaryFile will be rename to StoreFile. */// w w w . j a v a 2 s . c o m @Override public byte[] handleResponse(HttpResponse response, Delivery delivery) throws IOException, ServerError { // Content-Length might be negative when use HttpURLConnection because it default header Accept-Encoding is gzip, // we can force set the Accept-Encoding as identity in prepare() method to slove this problem but also disable gzip response. HttpEntity entity = response.getEntity(); long fileSize = entity.getContentLength(); if (fileSize <= 0) { NetroidLog.d("Response doesn't present Content-Length!"); } long downloadedSize = mTemporaryFile.length(); boolean isSupportRange = HttpUtils.isSupportRange(response); if (isSupportRange) { fileSize += downloadedSize; // Verify the Content-Range Header, to ensure temporary file is part of the whole file. // Sometime, temporary file length add response content-length might greater than actual file length, // in this situation, we consider the temporary file is invalid, then throw an exception. String realRangeValue = HttpUtils.getHeader(response, "Content-Range"); // response Content-Range may be null when "Range=bytes=0-" if (!TextUtils.isEmpty(realRangeValue)) { String assumeRangeValue = "bytes " + downloadedSize + "-" + (fileSize - 1); if (TextUtils.indexOf(realRangeValue, assumeRangeValue) == -1) { throw new IllegalStateException("The Content-Range Header is invalid Assume[" + assumeRangeValue + "] vs Real[" + realRangeValue + "], " + "please remove the temporary file [" + mTemporaryFile + "]."); } } } // Compare the store file size(after download successes have) to server-side Content-Length. // temporary file will rename to store file after download success, so we compare the // Content-Length to ensure this request already download or not. if (fileSize > 0 && mStoreFile.length() == fileSize) { // Rename the store file to temporary file, mock the download success. ^_^ mStoreFile.renameTo(mTemporaryFile); // Deliver download progress. delivery.postDownloadProgress(this, fileSize, fileSize); return null; } RandomAccessFile tmpFileRaf = new RandomAccessFile(mTemporaryFile, "rw"); // If server-side support range download, we seek to last point of the temporary file. if (isSupportRange) { tmpFileRaf.seek(downloadedSize); } else { // If not, truncate the temporary file then start download from beginning. tmpFileRaf.setLength(0); downloadedSize = 0; } InputStream in = null; try { in = entity.getContent(); // Determine the response gzip encoding, support for HttpClientStack download. if (HttpUtils.isGzipContent(response) && !(in instanceof GZIPInputStream)) { in = new GZIPInputStream(in); } byte[] buffer = new byte[6 * 1024]; // 6K buffer int offset; while ((offset = in.read(buffer)) != -1) { tmpFileRaf.write(buffer, 0, offset); downloadedSize += offset; delivery.postDownloadProgress(this, fileSize, downloadedSize); if (isCanceled()) { delivery.postCancel(this); break; } } } finally { try { // Close the InputStream if (in != null) in.close(); } catch (Exception e) { NetroidLog.v("Error occured when calling InputStream.close"); } try { // release the resources by "consuming the content". entity.consumeContent(); } catch (Exception e) { // This can happen if there was an exception above that left the entity in // an invalid state. NetroidLog.v("Error occured when calling consumingContent"); } tmpFileRaf.close(); } return null; }
From source file:org.commoncrawl.service.listcrawler.CacheManager.java
/** * updateLogFileHeader - update the log file header * called via the log file writer thread ... * @throws IOException//from www . j a va 2 s. c o m */ void updateLogFileHeader(File logFileName, long newlyAddedItemsCount, long newItemsFileSize) throws IOException { RandomAccessFile file = new RandomAccessFile(logFileName, "rw"); try { synchronized (_header) { // update cached header ... _header._fileSize += newItemsFileSize; _header._itemCount += newlyAddedItemsCount; // set the position at zero .. file.seek(0); // and write header to disk ... _header.writeHeader(file); } } finally { // major bottle neck.. // file.getFD().sync(); file.close(); } }
From source file:com.googlecode.psiprobe.Utils.java
public static void sendFile(HttpServletRequest request, HttpServletResponse response, File file) throws IOException { OutputStream out = response.getOutputStream(); RandomAccessFile raf = new RandomAccessFile(file, "r"); try {/*from w w w . ja v a2 s .c o m*/ long fileSize = raf.length(); long rangeStart = 0; long rangeFinish = fileSize - 1; // accept attempts to resume download (if any) String range = request.getHeader("Range"); if (range != null && range.startsWith("bytes=")) { String pureRange = range.replaceAll("bytes=", ""); int rangeSep = pureRange.indexOf("-"); try { rangeStart = Long.parseLong(pureRange.substring(0, rangeSep)); if (rangeStart > fileSize || rangeStart < 0) { rangeStart = 0; } } catch (NumberFormatException e) { // ignore the exception, keep rangeStart unchanged } if (rangeSep < pureRange.length() - 1) { try { rangeFinish = Long.parseLong(pureRange.substring(rangeSep + 1)); if (rangeFinish < 0 || rangeFinish >= fileSize) { rangeFinish = fileSize - 1; } } catch (NumberFormatException e) { // ignore the exception } } } // set some headers response.setContentType("application/x-download"); response.setHeader("Content-Disposition", "attachment; filename=" + file.getName()); response.setHeader("Accept-Ranges", "bytes"); response.setHeader("Content-Length", Long.toString(rangeFinish - rangeStart + 1)); response.setHeader("Content-Range", "bytes " + rangeStart + "-" + rangeFinish + "/" + fileSize); // seek to the requested offset raf.seek(rangeStart); // send the file byte[] buffer = new byte[4096]; long len; int totalRead = 0; boolean nomore = false; while (true) { len = raf.read(buffer); if (len > 0 && totalRead + len > rangeFinish - rangeStart + 1) { // read more then required? // adjust the length len = rangeFinish - rangeStart + 1 - totalRead; nomore = true; } if (len > 0) { out.write(buffer, 0, (int) len); totalRead += len; if (nomore) { break; } } else { break; } } } finally { raf.close(); } }
From source file:com.linkedin.helix.store.file.FilePropertyStore.java
@Override public boolean compareAndSet(String key, T expected, T update, Comparator<T> comparator, boolean createIfAbsent) { String path = getPath(key);//from w ww. ja v a 2 s .c o m File file = new File(path); // FileInputStream fin = null; // FileOutputStream fout = null; RandomAccessFile raFile = null; FileLock fLock = null; try { _readWriteLock.writeLock().lock(); if (createIfAbsent) { file.createNewFile(); } // fin = new FileInputStream(file); // FileChannel fChannel = fin.getChannel(); raFile = new RandomAccessFile(file, "rw"); FileChannel fChannel = raFile.getChannel(); fLock = fChannel.lock(); T current = getProperty(key); if (comparator.compare(current, expected) == 0) { // fout = new FileOutputStream(file); // // byte[] bytes = _serializer.serialize(update); // fout.write(bytes); setProperty(key, update); return true; } return false; } catch (FileNotFoundException e) { logger.error("fail to compareAndSet. path:" + path, e); return false; } catch (Exception e) { logger.error("fail to compareAndSet. path:" + path, e); return false; } finally { _readWriteLock.writeLock().unlock(); try { if (fLock != null && fLock.isValid()) { fLock.release(); } if (raFile != null) { raFile.close(); } // if (fin != null) // { // fin.close(); // } // // if (fout != null) // { // fout.close(); // } } catch (IOException e) { logger.error("fail to close file. path:" + path, e); } } }
From source file:com.android.volley.toolbox.UploadNetwork.java
@Override public NetworkResponse performRequest(Request<?> request) throws VolleyError { long requestStart = SystemClock.elapsedRealtime(); while (true) { HttpResponse httpResponse = null; byte[] responseContents = null; Map<String, String> responseHeaders = Collections.emptyMap(); RandomAccessFile acessfile = null; File file = null;/* w w w .j av a2 s. co m*/ try { if (!(request instanceof DownOrUpRequest)) { throw new IllegalArgumentException("request object mast be DownOrUpRequest???"); } DownOrUpRequest requestDown = (DownOrUpRequest) request; // Gather headers. Map<String, String> headers = new HashMap<String, String>(); // Download have no cache String url = requestDown.getUrl(); String name = url.substring(url.lastIndexOf('/'), url.length()); String path = requestDown.getDownloadPath(); String filePath = ""; if (path.endsWith("/")) { filePath = path + name; } else { path = path + "/"; filePath = path + "/" + name; } File dir = new File(path); dir.mkdirs(); file = File.createTempFile(path, null, dir); acessfile = new RandomAccessFile(file, "rws"); long length = acessfile.length(); acessfile.seek(length); // Range: bytes=5275648- headers.put("Range", "bytes=" + length + "-");// httpResponse = mHttpStack.performRequest(requestDown, headers); StatusLine statusLine = httpResponse.getStatusLine(); int statusCode = statusLine.getStatusCode(); responseHeaders = convertHeaders(httpResponse.getAllHeaders()); // if the request is slow, log it. long requestLifetime = SystemClock.elapsedRealtime() - requestStart; logSlowRequests(requestLifetime, requestDown, responseContents, statusLine); if (statusCode < 200 || statusCode > 299) { acessfile.close(); throw new IOException(); } // Some responses such as 204s do not have content. We must check. if (httpResponse.getEntity() != null) { responseContents = entityToBytes(httpResponse.getEntity(), requestDown, acessfile); } else { // Add 0 byte response as a way of honestly representing a // no-content request. responseContents = new byte[0]; } acessfile.close(); file.renameTo(new File(filePath)); return new NetworkResponse(statusCode, responseContents, responseHeaders, false, SystemClock.elapsedRealtime() - requestStart); } catch (SocketTimeoutException e) { attemptRetryOnException("socket", request, new TimeoutError()); } catch (ConnectTimeoutException e) { attemptRetryOnException("connection", request, new TimeoutError()); } catch (MalformedURLException e) { throw new RuntimeException("Bad URL " + request.getUrl(), e); } catch (IOException e) { if (acessfile != null) { try { acessfile.close(); file.delete(); } catch (IOException e1) { e1.printStackTrace(); } } int statusCode = 0; NetworkResponse networkResponse = null; if (httpResponse != null) { statusCode = httpResponse.getStatusLine().getStatusCode(); } else { throw new NoConnectionError(e); } VolleyLog.e("Unexpected response code %d for %s", statusCode, request.getUrl()); throw new NetworkError(networkResponse); } } }
From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java
/** * updateLogFileHeader - update the log file header called via the log file * writer thread .../* ww w . ja v a 2s . c o m*/ * * @throws IOException */ void updateLogFileHeader(File logFileName, long newlyAddedItemsCount, long newItemsFileSize) throws IOException { RandomAccessFile file = new RandomAccessFile(logFileName, "rw"); try { synchronized (_header) { // update cached header ... _header._fileSize += newItemsFileSize; _header._itemCount += newlyAddedItemsCount; // set the position at zero .. file.seek(0); // and write header to disk ... _header.writeHeader(file); } } finally { // major bottle neck.. // file.getFD().sync(); file.close(); } }
From source file:org.apache.jackrabbit.oak.plugins.segment.file.TarReader.java
private static TarReader openFirstFileWithValidIndex(List<File> files, boolean memoryMapping) { for (File file : files) { String name = file.getName(); try {/*from w w w . j a v a 2 s . com*/ RandomAccessFile access = new RandomAccessFile(file, "r"); try { ByteBuffer index = loadAndValidateIndex(access, name); if (index == null) { log.info("No index found in tar file {}, skipping...", name); } else { // found a file with a valid index, drop the others for (File other : files) { if (other != file) { log.info("Removing unused tar file {}", other.getName()); other.delete(); } } if (memoryMapping) { try { FileAccess mapped = new FileAccess.Mapped(access); // re-read the index, now with memory mapping int indexSize = index.remaining(); index = mapped.read(mapped.length() - indexSize - 16 - 1024, indexSize); return new TarReader(file, mapped, index); } catch (IOException e) { log.warn("Failed to mmap tar file " + name + ". Falling back to normal file IO," + " which will negatively impact" + " repository performance. This" + " problem may have been caused by" + " restrictions on the amount of" + " virtual memory available to the" + " JVM. Please make sure that a" + " 64-bit JVM is being used and" + " that the process has access to" + " unlimited virtual memory" + " (ulimit option -v).", e); } } FileAccess random = new FileAccess.Random(access); // prevent the finally block from closing the file // as the returned TarReader will take care of that access = null; return new TarReader(file, random, index); } } finally { if (access != null) { access.close(); } } } catch (IOException e) { log.warn("Could not read tar file " + name + ", skipping...", e); } } return null; }