List of usage examples for org.apache.commons.io LineIterator next
public Object next()
Reader. From source file:dk.netarkivet.archive.arcrepositoryadmin.ReplicaCacheDatabase.java
/** * Given the output of a checksum job, add the results to the database. * <p>//from w w w. j a va2 s . c o m * The following fields in the table are updated for each corresponding entry in the replicafileinfo table: <br/> * - checksum = the given checksum. <br/> * - filelist_status = ok. <br/> * - filelist_checkdatetime = now. <br/> * - checksum_checkdatetime = now. * * @param checksumOutputFile The output of a checksum job in a file * @param replica The replica this checksum job is for. */ @Override public void addChecksumInformation(File checksumOutputFile, Replica replica) { // validate arguments ArgumentNotValid.checkNotNull(checksumOutputFile, "File checksumOutputFile"); ArgumentNotValid.checkNotNull(replica, "Replica replica"); // Sort the checksumOutputFile file. File sortedResult = new File(checksumOutputFile.getParent(), checksumOutputFile.getName() + ".sorted"); FileUtils.sortFile(checksumOutputFile, sortedResult); final long datasize = FileUtils.countLines(sortedResult); Set<Long> missingReplicaRFIs = null; Connection con = ArchiveDBConnection.get(); LineIterator lineIterator = null; try { // Make sure, that the replica exists in the database. if (!ReplicaCacheHelpers.existsReplicaInDB(replica, con)) { String msg = "Cannot add checksum information, since the replica '" + replica.toString() + "' does not exist within the database."; log.warn(msg); throw new IOFailure(msg); } log.info("Starting processing of {} checksum entries for replica {}", datasize, replica.getId()); // retrieve the list of files already known by this cache. // TODO This does not scale! Should the datastructure // (missingReplicaRFIs) be disk-bound in some way, or optimized // in some way, e.g. using it.unimi.dsi.fastutil.longs.LongArrayList missingReplicaRFIs = ReplicaCacheHelpers.retrieveReplicaFileInfoGuidsForReplica(replica.getId(), con); // Initialize the String iterator lineIterator = new LineIterator(new FileReader(sortedResult)); String lastFilename = ""; String lastChecksum = ""; int i = 0; while (lineIterator.hasNext()) { String line = lineIterator.next(); // log that it is in progress every so often. if ((i % LOGGING_ENTRY_INTERVAL) == 0) { log.info("Processed checksum list entry number {} for replica {}", i, replica); // Close connection, and open another one // to avoid memory-leak (NAS-2003) ArchiveDBConnection.release(con); con = ArchiveDBConnection.get(); log.debug("Databaseconnection has now been renewed"); } ++i; // parse the input. final KeyValuePair<String, String> entry = ChecksumJob.parseLine(line); final String filename = entry.getKey(); final String checksum = entry.getValue(); // check for duplicates if (filename.equals(lastFilename)) { // if different checksums, then if (!checksum.equals(lastChecksum)) { // log and send notification String errMsg = "Unidentical duplicates of file '" + filename + "' with the checksums '" + lastChecksum + "' and '" + checksum + "'. First instance used."; log.warn(errMsg); NotificationsFactory.getInstance().notify(errMsg, NotificationType.WARNING); } else { // log about duplicate identical log.debug("Duplicates of the file '{}' found with the same checksum '{}'.", filename, checksum); } // avoid overhead of inserting duplicates twice. continue; } // set these value to be the old values in next iteration. lastFilename = filename; lastChecksum = checksum; // Process the current (filename + checksum) combo for this replica // Remove the returned replicafileinfo guid from the missing entries. missingReplicaRFIs .remove(ReplicaCacheHelpers.processChecksumline(filename, checksum, replica, con)); } } catch (IOException e) { throw new IOFailure("Unable to read checksum entries from file", e); } finally { ArchiveDBConnection.release(con); LineIterator.closeQuietly(lineIterator); } con = ArchiveDBConnection.get(); try { // go through the not found replicafileinfo for this replica to change // their filelist_status to missing. if (missingReplicaRFIs.size() > 0) { log.warn("Found {} missing files for replica '{}'.", missingReplicaRFIs.size(), replica); for (long rfi : missingReplicaRFIs) { // set the replicafileinfo in the database to missing. ReplicaCacheHelpers.updateReplicaFileInfoMissingFromFilelist(rfi, con); } } // update the checksum updated date for this replica. ReplicaCacheHelpers.updateChecksumDateForReplica(replica, con); ReplicaCacheHelpers.updateFilelistDateForReplica(replica, con); log.info("Finished processing of {} checksum entries for replica {}", datasize, replica.getId()); } finally { ArchiveDBConnection.release(con); } }
From source file:org.aludratest.service.gitclient.GitClient.java
/** Provides the status. * @param data/*w w w.j ava 2 s . c om*/ * @return a reference to this */ public GitClient status(StatusData data) { // clear status object for supporting data object reuse data.getUntrackedFiles().clear(); data.getUnmodifiedFiles().clear(); data.getModifiedFiles().clear(); data.getAddedFiles().clear(); data.getDeletedFiles().clear(); data.getRenamedFiles().clear(); data.getCopiedFiles().clear(); data.getUpdatedFiles().clear(); // invoke git String output = invokeGenerically(GIT_STATUS_PROCESS_NAME, true, "status", "--short", "--branch"); LineIterator iterator = new LineIterator(new StringReader(output)); while (iterator.hasNext()) { String line = iterator.next(); if (line.startsWith("##")) { data.setCurrentBranch(line.substring(3)); } else { StringData filePath = new StringData(line.substring(3)); char statusCode = line.substring(0, 2).trim().charAt(0); switch (statusCode) { case '?': data.getUntrackedFiles().add(filePath); break; case '\'': data.getUnmodifiedFiles().add(filePath); break; case 'M': data.getModifiedFiles().add(filePath); break; case 'A': data.getAddedFiles().add(filePath); break; case 'D': data.getDeletedFiles().add(filePath); break; case 'R': data.getRenamedFiles().add(parseRename(filePath.getValue())); break; case 'C': data.getCopiedFiles().add(filePath); break; case 'U': data.getUpdatedFiles().add(filePath); break; default: throw new TechnicalException("Unknown status '" + statusCode + "' in git output: " + line); } } } return this; }
From source file:org.aludratest.service.gitclient.GitClient.java
/** Lists branches. * @param data/*from w w w . ja v a2 s. c o m*/ * @return a reference to this */ public GitClient listBranches(BranchListData data) { String output = invokeGenerically(GIT_LIST_BRANCHES_PROCESS_NAME, true, "branch", "--list"); LineIterator iterator = new LineIterator(new StringReader(output)); while (iterator.hasNext()) { String line = iterator.next(); boolean current = line.startsWith("*"); String branch = line.substring(2).trim(); data.getBranches().add(new StringData(branch)); if (current) { data.setCurrentBranch(branch); } } return this; }
From source file:org.aludratest.service.gitclient.GitClient.java
private void parseLogItem(LineIterator iterator, LogData data) { String commit = null;//from www . j av a2s.c o m String merge = null; String author = null; String date = null; // parse headers while (iterator.hasNext()) { String line = iterator.next(); if (line.startsWith("commit ")) { commit = parseKeyValuePair("commit ", line); } else if (line.startsWith("Merge: ")) { merge = parseKeyValuePair("Merge: ", line); } else if (line.startsWith("Author: ")) { author = parseKeyValuePair("Author: ", line); } else if (line.startsWith("Date: ")) { date = parseKeyValuePair("Date: ", line); } else if (StringUtil.isEmpty(line)) { break; } else { LOGGER.warn("Unrecognized log line: " + line); } } // parse message String message = parseMessage(iterator); // add log item LogItemData item = new LogItemData(commit, merge, author, date, message); data.getItems().add(item); }
From source file:org.aludratest.service.gitclient.GitClient.java
private String parseMessage(LineIterator iterator) { if (!iterator.hasNext()) { throw new TechnicalException("Expected commit message not available"); }//from www. j a v a 2 s. c om StringBuilder message = new StringBuilder(); while (iterator.hasNext()) { String line = iterator.next(); if (!StringUtil.isEmpty(line)) { if (message.length() > 0) { message.append(LF); } message.append(line.trim()); } else { return message.toString(); } } return message.toString(); }
From source file:org.apache.http.nio.client.methods.TestZeroCopy.java
@Test public void testTwoWayZeroCopy() throws Exception { this.serverBootstrap.registerHandler("*", new BasicAsyncRequestHandler(new TestHandler(false))); final HttpHost target = start(); final File tmpdir = FileUtils.getTempDirectory(); this.tmpfile = new File(tmpdir, "dst.test"); final TestZeroCopyPost httppost = new TestZeroCopyPost(target.toURI() + "/bounce", false); final TestZeroCopyConsumer consumer = new TestZeroCopyConsumer(this.tmpfile); final Future<Integer> future = this.httpclient.execute(httppost, consumer, null); final Integer status = future.get(); Assert.assertNotNull(status);//from ww w . j a v a 2 s . c om Assert.assertEquals(HttpStatus.SC_OK, status.intValue()); final InputStream instream = new FileInputStream(this.tmpfile); try { final LineIterator it = IOUtils.lineIterator(instream, ASCII.name()); int count = 0; while (it.hasNext()) { final String line = it.next(); final int i = count % TEXT.length; final String expected = TEXT[i]; Assert.assertEquals(expected, line); count++; } } finally { instream.close(); } }
From source file:org.apache.http.nio.client.methods.TestZeroCopy.java
@Test public void testZeroCopyFallback() throws Exception { this.serverBootstrap.registerHandler("*", new BasicAsyncRequestHandler(new TestHandler(true))); final HttpHost target = start(); final File tmpdir = FileUtils.getTempDirectory(); this.tmpfile = new File(tmpdir, "dst.test"); final TestZeroCopyPost httppost = new TestZeroCopyPost(target.toURI() + "/bounce", true); final TestZeroCopyConsumer consumer = new TestZeroCopyConsumer(this.tmpfile); final Future<Integer> future = this.httpclient.execute(httppost, consumer, null); final Integer status = future.get(); Assert.assertNotNull(status);/* w w w . ja v a 2 s. c o m*/ Assert.assertEquals(HttpStatus.SC_OK, status.intValue()); final InputStream instream = new FileInputStream(this.tmpfile); try { final LineIterator it = IOUtils.lineIterator(instream, ASCII.name()); int count = 0; while (it.hasNext()) { final String line = it.next(); final int i = count % TEXT.length; final String expected = TEXT[i]; Assert.assertEquals(expected, line); count++; } } finally { instream.close(); } }
From source file:org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.java
/** * Sweep phase of gc candidate deletion. * <p>//from ww w . java 2s. c o m * Performs the following steps depending upon the type of the blob store refer * {@link org.apache.jackrabbit.oak.plugins.blob.SharedDataStore.Type}: * * <ul> * <li>Shared</li> * <li> * <ul> * <li> Merge all marked references (from the mark phase run independently) available in the data store meta * store (from all configured independent repositories). * <li> Retrieve all blob ids available. * <li> Diffs the 2 sets above to retrieve list of blob ids not used. * <li> Deletes only blobs created after * (earliest time stamp of the marked references - #maxLastModifiedInterval) from the above set. * </ul> * </li> * * <li>Default</li> * <li> * <ul> * <li> Mark phase already run. * <li> Retrieve all blob ids available. * <li> Diffs the 2 sets above to retrieve list of blob ids not used. * <li> Deletes only blobs created after * (time stamp of the marked references - #maxLastModifiedInterval). * </ul> * </li> * </ul> * * @return the number of blobs deleted * @throws Exception the exception * @param fs the garbage collector file state * @param markStart the start time of mark to take as reference for deletion */ protected long sweep(GarbageCollectorFileState fs, long markStart) throws Exception { long earliestRefAvailTime; // Merge all the blob references available from all the reference files in the data store meta store // Only go ahead if merge succeeded try { earliestRefAvailTime = GarbageCollectionType.get(blobStore).mergeAllMarkedReferences(blobStore, fs); LOG.debug("Earliest reference available for timestamp [{}]", earliestRefAvailTime); earliestRefAvailTime = (earliestRefAvailTime < markStart ? earliestRefAvailTime : markStart); } catch (Exception e) { return 0; } // Find all blob references after iterating over the whole repository (new BlobIdRetriever(fs)).call(); // Calculate the references not used difference(fs); long count = 0; long deleted = 0; long lastMaxModifiedTime = getLastMaxModifiedTime(earliestRefAvailTime); LOG.debug("Starting sweep phase of the garbage collector"); LOG.debug("Sweeping blobs with modified time > than the configured max deleted time ({}). ", timestampToString(lastMaxModifiedTime)); ConcurrentLinkedQueue<String> exceptionQueue = new ConcurrentLinkedQueue<String>(); LineIterator iterator = FileUtils.lineIterator(fs.getGcCandidates(), Charsets.UTF_8.name()); List<String> ids = newArrayList(); while (iterator.hasNext()) { ids.add(iterator.next()); if (ids.size() >= getBatchCount()) { count += ids.size(); deleted += sweepInternal(ids, exceptionQueue, lastMaxModifiedTime); ids = newArrayList(); } } if (!ids.isEmpty()) { count += ids.size(); deleted += sweepInternal(ids, exceptionQueue, lastMaxModifiedTime); } BufferedWriter writer = null; try { if (!exceptionQueue.isEmpty()) { writer = Files.newWriter(fs.getGarbage(), Charsets.UTF_8); saveBatchToFile(newArrayList(exceptionQueue), writer); } } finally { LineIterator.closeQuietly(iterator); IOUtils.closeQuietly(writer); } if (!exceptionQueue.isEmpty()) { LOG.warn( "Unable to delete some blobs entries from the blob store. Details around such blob entries can " + "be found in [{}]", fs.getGarbage().getAbsolutePath()); } if (count != deleted) { LOG.warn( "Deleted only [{}] blobs entries from the [{}] candidates identified. This may happen if blob " + "modified time is > " + "than the max deleted time ({})", deleted, count, timestampToString(lastMaxModifiedTime)); } // Remove all the merged marked references GarbageCollectionType.get(blobStore).removeAllMarkedReferences(blobStore); LOG.debug("Ending sweep phase of the garbage collector"); return deleted; }
From source file:org.apache.marmotta.platform.core.services.prefix.PrefixCC.java
@Override public String getNamespace(final String prefix) { HttpGet get = new HttpGet(URI + prefix + ".file.txt"); HttpRequestUtil.setUserAgentString(get, USER_AGENT); get.setHeader(ACCEPT, "text/plain"); try {//from w w w. j a v a 2 s.c om return httpClientService.execute(get, new ResponseHandler<String>() { @Override public String handleResponse(HttpResponse response) throws ClientProtocolException, IOException { if (200 == response.getStatusLine().getStatusCode()) { HttpEntity entity = response.getEntity(); final LineIterator it = IOUtils.lineIterator(entity.getContent(), Charset.defaultCharset()); try { while (it.hasNext()) { final String l = it.next(); if (l.startsWith(prefix + "\t")) { return l.substring(prefix.length() + 1); } } } finally { it.close(); } } log.error("Error: prefix '" + prefix + "' not found at prefix.cc"); return null; } }); } catch (Exception e) { log.error("Error retrieving prefix '" + prefix + "' from prefix.cc: " + e.getMessage()); return null; } }
From source file:org.apache.marmotta.platform.core.services.prefix.PrefixCC.java
@Override public String getPrefix(final String namespace) { try {//from www . j av a2s . c o m HttpGet get = new HttpGet(URI + "reverse?format=txt&uri=" + URLEncoder.encode(namespace, "utf-8")); HttpRequestUtil.setUserAgentString(get, USER_AGENT); get.setHeader(ACCEPT, "text/plain"); return httpClientService.execute(get, new ResponseHandler<String>() { @Override public String handleResponse(HttpResponse response) throws ClientProtocolException, IOException { if (200 == response.getStatusLine().getStatusCode()) { HttpEntity entity = response.getEntity(); final LineIterator it = IOUtils.lineIterator(entity.getContent(), Charset.defaultCharset()); try { while (it.hasNext()) { final String l = it.next(); if (l.endsWith("\t" + namespace)) { return l.substring(0, l.indexOf("\t")); } } } finally { it.close(); } } log.error("Error: reverse namespace lookup for '" + namespace + "' not found at prefix.cc"); return null; } }); } catch (Exception e) { log.error("Error trying to retrieve prefic.cc reverse lookup for namespace '" + namespace + "': " + e.getMessage()); return null; } }