List of usage examples for org.apache.commons.collections IteratorUtils collatedIterator
public static Iterator collatedIterator(Comparator comparator, Collection iterators)
From source file:com.bigdata.dastor.db.ColumnFamilyStore.java
private ColumnFamily getTopLevelColumns(QueryFilter filter, int gcBefore) throws IOException { // we are querying top-level columns, do a merging fetch with indexes. List<ColumnIterator> iterators = new ArrayList<ColumnIterator>(); try {//w w w .j a v a2s .c om final ColumnFamily returnCF; ColumnIterator iter; /* add the current memtable */ Table.flusherLock.readLock().lock(); try { iter = filter.getMemColumnIterator(memtable_, getComparator()); // TODO this is a little subtle: the Memtable ColumnIterator has to be a shallow clone of the source CF, // with deletion times set correctly, so we can use it as the "base" CF to add query results to. // (for sstable ColumnIterators we do not care if it is a shallow clone or not.) returnCF = iter.getColumnFamily(); } finally { Table.flusherLock.readLock().unlock(); } iterators.add(iter); /* add the memtables being flushed */ for (Memtable memtable : getMemtablesPendingFlush()) { iter = filter.getMemColumnIterator(memtable, getComparator()); returnCF.delete(iter.getColumnFamily()); iterators.add(iter); } /* add the SSTables on disk */ for (SSTableReader sstable : ssTables_) { iter = filter.getSSTableColumnIterator(sstable); if (iter.getColumnFamily() != null) { returnCF.delete(iter.getColumnFamily()); iterators.add(iter); } } Comparator<IColumn> comparator = filter.getColumnComparator(getComparator()); Iterator collated = IteratorUtils.collatedIterator(comparator, iterators); filter.collectCollatedColumns(returnCF, collated, gcBefore); return removeDeleted(returnCF, gcBefore); } finally { /* close all cursors */ for (ColumnIterator ci : iterators) { try { ci.close(); } catch (Throwable th) { logger_.error("error closing " + ci, th); } } } }
From source file:com.bigdata.dastor.db.ColumnFamilyStore.java
/** * @param range: either a Bounds, which includes start key, or a Range, which does not. * @param maxResults// w ww. ja v a 2 s . c o m * @return list of keys between startWith and stopAt TODO refactor better. this is just getKeyRange w/o the deletion check, for the benefit of range_slice. still opens one randomaccessfile per key, which sucks. something like compactioniterator would be better. */ private void getKeyRange(List<String> keys, final AbstractBounds range, int maxResults) throws IOException, ExecutionException, InterruptedException { final DecoratedKey startWith = new DecoratedKey(range.left, null); final DecoratedKey stopAt = new DecoratedKey(range.right, null); // create a CollatedIterator that will return unique keys from different sources // (current memtable, historical memtables, and SSTables) in the correct order. final List<Iterator<DecoratedKey>> iterators = new ArrayList<Iterator<DecoratedKey>>(); // we iterate through memtables with a priority queue to avoid more sorting than necessary. // this predicate throws out the keys before the start of our range. Predicate<DecoratedKey> p = new Predicate<DecoratedKey>() { public boolean apply(DecoratedKey key) { return startWith.compareTo(key) <= 0 && (stopAt.isEmpty() || key.compareTo(stopAt) <= 0); } }; // current memtable keys. have to go through the CFS api for locking. iterators.add(Iterators.filter(memtableKeyIterator(startWith), p)); // historical memtables for (Memtable memtable : memtablesPendingFlush) { iterators.add(Iterators.filter(memtable.getKeyIterator(startWith), p)); } // sstables for (SSTableReader sstable : ssTables_) { final SSTableScanner scanner = sstable.getScanner(KEY_RANGE_FILE_BUFFER_SIZE); scanner.seekTo(startWith); Iterator<DecoratedKey> iter = new CloseableIterator<DecoratedKey>() { public boolean hasNext() { return scanner.hasNext(); } public DecoratedKey next() { return scanner.next().getKey(); } public void remove() { throw new UnsupportedOperationException(); } public void close() throws IOException { scanner.close(); } }; assert iter instanceof Closeable; // otherwise we leak FDs iterators.add(iter); } Iterator<DecoratedKey> collated = IteratorUtils.collatedIterator(DecoratedKey.comparator, iterators); Iterable<DecoratedKey> reduced = new ReducingIterator<DecoratedKey, DecoratedKey>(collated) { DecoratedKey current; public void reduce(DecoratedKey current) { this.current = current; } protected DecoratedKey getReduced() { return current; } }; try { // pull keys out of the CollatedIterator boolean first = true; for (DecoratedKey current : reduced) { if (!stopAt.isEmpty() && stopAt.compareTo(current) < 0) { return; } if (range instanceof Bounds || !first || !current.equals(startWith)) { if (logger_.isDebugEnabled()) logger_.debug("scanned " + current); keys.add(current.key); } first = false; if (keys.size() >= maxResults) { return; } } } finally { for (Iterator iter : iterators) { if (iter instanceof Closeable) { ((Closeable) iter).close(); } } } }
From source file:org.apache.cassandra.db.ColumnFamilyStore.java
private ColumnFamily getTopLevelColumns(QueryFilter filter, int gcBefore) { // we are querying top-level columns, do a merging fetch with indexes. List<IColumnIterator> iterators = new ArrayList<IColumnIterator>(); final ColumnFamily returnCF = ColumnFamily.create(metadata); try {//from w w w . j av a 2 s . co m IColumnIterator iter; int sstablesToIterate = 0; DataTracker.View currentView = data.getView(); /* add the current memtable */ iter = filter.getMemtableColumnIterator(currentView.memtable, getComparator()); if (iter != null) { returnCF.delete(iter.getColumnFamily()); iterators.add(iter); } /* add the memtables being flushed */ for (Memtable memtable : currentView.memtablesPendingFlush) { iter = filter.getMemtableColumnIterator(memtable, getComparator()); if (iter != null) { returnCF.delete(iter.getColumnFamily()); iterators.add(iter); } } /* add the SSTables on disk */ for (SSTableReader sstable : currentView.sstables) { iter = filter.getSSTableColumnIterator(sstable); if (iter.getColumnFamily() != null) { returnCF.delete(iter.getColumnFamily()); iterators.add(iter); sstablesToIterate++; } } recentSSTablesPerRead.add(sstablesToIterate); sstablesPerRead.add(sstablesToIterate); Comparator<IColumn> comparator = filter.filter.getColumnComparator(getComparator()); Iterator collated = IteratorUtils.collatedIterator(comparator, iterators); filter.collectCollatedColumns(returnCF, collated, gcBefore); // Caller is responsible for final removeDeletedCF. This is important for cacheRow to work correctly: // we need to distinguish between "there is no data at all for this row" (BF will let us rebuild that efficiently) // and "there used to be data, but it's gone now" (we should cache the empty CF so we don't need to rebuild that slower) return returnCF; } finally { /* close all cursors */ for (IColumnIterator ci : iterators) { try { ci.close(); } catch (Throwable th) { logger.error("error closing " + ci, th); } } } }
From source file:org.apache.cassandra.db.RowIteratorFactory.java
/** * Get a row iterator over the provided memtables and sstables, between the provided keys * and filtered by the queryfilter.//from w ww . j a v a2 s . com * @param memtables Memtables pending flush. * @param sstables SStables to scan through. * @param startWith Start at this key * @param stopAt Stop and this key * @param filter Used to decide which columns to pull out * @param comparator * @return A row iterator following all the given restrictions */ public static RowIterator getIterator(final Collection<Memtable> memtables, final Collection<SSTableReader> sstables, final DecoratedKey startWith, final DecoratedKey stopAt, final QueryFilter filter, final AbstractType comparator, final ColumnFamilyStore cfs) { // fetch data from current memtable, historical memtables, and SSTables in the correct order. final List<Iterator<IColumnIterator>> iterators = new ArrayList<Iterator<IColumnIterator>>(); // we iterate through memtables with a priority queue to avoid more sorting than necessary. // this predicate throws out the rows before the start of our range. Predicate<IColumnIterator> p = new Predicate<IColumnIterator>() { public boolean apply(IColumnIterator row) { return startWith.compareTo(row.getKey()) <= 0 && (stopAt.isEmpty() || row.getKey().compareTo(stopAt) <= 0); } }; // memtables for (Memtable memtable : memtables) { iterators.add(Iterators.filter(Iterators.transform(memtable.getEntryIterator(startWith), new ConvertToColumnIterator(filter, comparator)), p)); } // sstables for (SSTableReader sstable : sstables) { final SSTableScanner scanner = sstable.getScanner(RANGE_FILE_BUFFER_SIZE, filter); scanner.seekTo(startWith); assert scanner instanceof Closeable; // otherwise we leak FDs iterators.add(scanner); } Iterator<IColumnIterator> collated = IteratorUtils.collatedIterator(COMPARE_BY_KEY, iterators); // reduce rows from all sources into a single row ReducingIterator<IColumnIterator, Row> reduced = new ReducingIterator<IColumnIterator, Row>(collated) { private final int gcBefore = (int) (System.currentTimeMillis() / 1000) - cfs.metadata.getGcGraceSeconds(); private final List<IColumnIterator> colIters = new ArrayList<IColumnIterator>(); private DecoratedKey key; private ColumnFamily returnCF; @Override protected void onKeyChange() { this.returnCF = ColumnFamily.create(cfs.metadata); } public void reduce(IColumnIterator current) { this.colIters.add(current); this.key = current.getKey(); this.returnCF.delete(current.getColumnFamily()); } @Override protected boolean isEqual(IColumnIterator o1, IColumnIterator o2) { return COMPARE_BY_KEY.compare(o1, o2) == 0; } protected Row getReduced() { Comparator<IColumn> colComparator = filter.filter.getColumnComparator(comparator); Iterator<IColumn> colCollated = IteratorUtils.collatedIterator(colComparator, colIters); // First check if this row is in the rowCache. If it is we can skip the rest ColumnFamily cached = cfs.getRawCachedRow(key); if (cached != null) { QueryFilter keyFilter = new QueryFilter(key, filter.path, filter.filter); returnCF = cfs.filterColumnFamily(cached, keyFilter, gcBefore); } else if (colCollated.hasNext()) { filter.collectCollatedColumns(returnCF, colCollated, gcBefore); } Row rv = new Row(key, returnCF); colIters.clear(); key = null; return rv; } }; return new RowIterator(reduced, iterators); }